Merge

dbms/include/DB/AggregateFunctions/AggregateFunctionQuantileTiming.h

@@ -15,6 +15,7 @@
 #include <DB/Columns/ColumnArray.h>
 
 #include <stats/IntHash.h>
+#include <statdaemons/ext/range.hpp>
 
 
 namespace DB
@@ -234,9 +235,16 @@ namespace detail
 		template <typename ResultType>
 		void getMany(const double * levels, size_t size, ResultType * result) const
 		{
-			const double * levels_end = levels + size;
-			const double * level = levels;
-			UInt64 pos = count * *level;
+			std::size_t indices[size];
+			std::copy(ext::range_iterator<size_t>{}, ext::make_range_iterator(size), indices);
+			std::sort(indices, indices + size, [levels] (auto i1, auto i2) {
+				return levels[i1] < levels[i2];
+			});
+
+			const auto indices_end = indices + size;
+			auto index = indices;
+
+			UInt64 pos = count * levels[*index];
 
 			UInt64 accumulated = 0;
 
@@ -251,15 +259,14 @@ namespace detail
 
 				if (i < SMALL_THRESHOLD)
 				{
-					*result = i;
+					result[*index] = i;
 
-					++level;
-					++result;
+					++index;
 
-					if (level == levels_end)
+					if (index == indices_end)
 						return;
 
-					pos = count * *level;
+					pos = count * levels[*index];
 				}
 			}
 
@@ -274,24 +281,22 @@ namespace detail
 
 				if (i < BIG_SIZE)
 				{
-					*result = indexInBigToValue(i);
+					result[*index] = indexInBigToValue(i);
 
-					++level;
-					++result;
+					++index;
 
-					if (level == levels_end)
+					if (index == indices_end)
 						return;
 
-					pos = count * *level;
+					pos = count * levels[*index];
 				}
 			}
 
-			while (level < levels_end)
+			while (index < indices_end)
 			{
-				*result = BIG_THRESHOLD;
+				result[*index] = BIG_THRESHOLD;
 
-				++level;
-				++result;
+				++index;
 			}
 		}
 
@@ -466,7 +471,7 @@ public:
 
 				large = new detail::QuantileTimingLarge;
 			}
-			
+
 			large->merge(detail::QuantileTimingLarge(buf));
 		}
 		else
@@ -712,11 +717,11 @@ public:
 
 		size_t size = levels.size();
 		offsets_to.push_back((offsets_to.size() == 0 ? 0 : offsets_to.back()) + size);
-		
+
 		typename ColumnFloat32::Container_t & data_to = static_cast<ColumnFloat32 &>(arr_to.getData()).getData();
 		size_t old_size = data_to.size();
 		data_to.resize(data_to.size() + size);
-			
+
 		this->data(place).getManyFloat(&levels[0], size, &data_to[old_size]);
 	}
 };

dbms/include/DB/AggregateFunctions/AggregateFunctionSequenceMatch.h

@@ -124,7 +124,7 @@ struct AggregateFunctionSequenceMatchData final
 /// Max number of iterations to match the pattern against a sequence, exception thrown when exceeded
 constexpr auto sequence_match_max_iterations = 1000000;
 
-class AggregateFunctionSequenceMatch final : public IAggregateFunctionHelper<AggregateFunctionSequenceMatchData>
+class AggregateFunctionSequenceMatch : public IAggregateFunctionHelper<AggregateFunctionSequenceMatchData>
 {
 public:
 	static bool sufficientArgs(const std::size_t arg_count) { return arg_count >= 3; }
@@ -218,7 +218,14 @@ public:
 	void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
 	{
 		const_cast<Data &>(data(place)).sort();
-		static_cast<ColumnUInt8 &>(to).getData().push_back(match(place));
+
+		const auto & data_ref = data(place);
+
+		const auto events_begin = std::begin(data_ref.eventsList);
+		const auto events_end = std::end(data_ref.eventsList);
+		auto events_it = events_begin;
+
+		static_cast<ColumnUInt8 &>(to).getData().push_back(match(events_it, events_end));
 	}
 
 private:
@@ -233,21 +240,6 @@ private:
 		TimeGreater
 	};
 
-	static std::string to_string(const PatternActionType type)
-	{
-		static const std::map<PatternActionType, std::string> map{
-			{ PatternActionType::SpecificEvent, "SpecificEvent" },
-			{ PatternActionType::AnyEvent, "AnyEvent" },
-			{ PatternActionType::KleeneStar, "KleeneStar" },
-			{ PatternActionType::TimeLessOrEqual, "TimeLessOrEqual" },
-			{ PatternActionType::TimeLess, "TimeLess", },
-			{ PatternActionType::TimeGreaterOrEqual, "TimeGreaterOrEqual" },
-			{ PatternActionType::TimeGreater, "TimeGreater" }
-		};
-
-		return map.find(type)->second;
-	}
-
 	struct PatternAction final
 	{
 		PatternActionType type;
@@ -353,18 +345,15 @@ private:
 		this->actions = std::move(actions);
 	}
 
-	bool match(const ConstAggregateDataPtr & place) const
+protected:
+	template <typename T1, typename T2>
+	bool match(T1 & events_it, const T2 events_end) const
 	{
 		const auto action_begin = std::begin(actions);
 		const auto action_end = std::end(actions);
 		auto action_it = action_begin;
 
-		const auto & data_ref = data(place);
-		const auto events_begin = std::begin(data_ref.eventsList);
-		const auto events_end = std::end(data_ref.eventsList);
-		auto events_it = events_begin;
-
-		auto base_it = events_begin;
+		auto base_it = events_it;
 
 		/// an iterator to action plus an iterator to row in events list plus timestamp at the start of sequence
 		using backtrack_info = std::tuple<decltype(action_it), decltype(events_it), decltype(base_it)>;
@@ -392,11 +381,6 @@ private:
 		std::size_t i = 0;
 		while (action_it != action_end && events_it != events_end)
 		{
-//			std::cout << "start_timestamp " << base_it->first << "; ";
-//			std::cout << "elapsed " << (events_it->first - base_it->first) << "; ";
-//			std::cout << "action " << (action_it - action_begin) << " { " << to_string(action_it->type) << ' ' << action_it->extra << " }; ";
-//			std::cout << "symbol " << (events_it - events_begin) << " { " << events_it->first << ' ' << events_it->second.to_ulong() << " }" << std::endl;
-
 			if (action_it->type == PatternActionType::SpecificEvent)
 			{
 				if (events_it->second.test(action_it->extra))
@@ -492,9 +476,40 @@ private:
 		return action_it == action_end;
 	}
 
+private:
 	std::string pattern;
 	std::size_t arg_count;
 	PatternActions actions;
 };
 
+class AggregateFunctionSequenceCount final : public AggregateFunctionSequenceMatch
+{
+public:
+	String getName() const override { return "sequenceCount"; }
+
+	DataTypePtr getReturnType() const override { return new DataTypeUInt64; }
+
+	void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
+	{
+		const_cast<Data &>(data(place)).sort();
+		static_cast<ColumnUInt64 &>(to).getData().push_back(count(place));
+	}
+
+private:
+	UInt64 count(const ConstAggregateDataPtr & place) const
+	{
+		const auto & data_ref = data(place);
+
+		const auto events_begin = std::begin(data_ref.eventsList);
+		const auto events_end = std::end(data_ref.eventsList);
+		auto events_it = events_begin;
+
+		std::size_t count = 0;
+		while (events_it != events_end && match(events_it, events_end))
+			++count;
+
+		return count;
+	}
+};
+
 }

dbms/include/DB/AggregateFunctions/AggregateFunctionUniq.h

@@ -20,6 +20,7 @@
 #include <DB/Columns/ColumnString.h>
 
 #include <DB/AggregateFunctions/IUnaryAggregateFunction.h>
+#include <DB/AggregateFunctions/UniqCombinedBiasData.h>
 
 
 namespace DB
@@ -29,7 +30,7 @@ namespace DB
 
 struct AggregateFunctionUniqUniquesHashSetData
 {
-	typedef UniquesHashSet<DefaultHash<UInt64>> Set;
+	typedef UniquesHashSet<DefaultHash<UInt64> > Set;
 	Set set;
 
 	static String getName() { return "uniq"; }
@@ -94,83 +95,79 @@ struct AggregateFunctionUniqExactData<String>
 	static String getName() { return "uniqExact"; }
 };
 
+template <typename T, HyperLogLogMode mode>
+struct BaseUniqCombinedData
+{
+	using Key = UInt64;
+	using Set = CombinedCardinalityEstimator<
+		Key,
+		HashSet<Key, DefaultHash<Key>, HashTableGrower<> >,
+		16,
+		14,
+		17,
+		DefaultHash<Key>,
+		UInt64,
+		HyperLogLogBiasEstimator<UniqCombinedBiasData>,
+		mode
+	>;
+
+	Set set;
+};
+
+template <HyperLogLogMode mode>
+struct BaseUniqCombinedData<String, mode>
+{
+	using Key = UInt64;
+	using Set = CombinedCardinalityEstimator<
+		Key,
+		HashSet<Key, TrivialHash, HashTableGrower<> >,
+		16,
+		14,
+		17,
+		TrivialHash,
+		UInt64,
+		HyperLogLogBiasEstimator<UniqCombinedBiasData>,
+		mode
+	>;
+
+	Set set;
+};
+
+/// Агрегатные функции uniqCombinedRaw, uniqCombinedLinearCounting, и uniqCombinedBiasCorrected
+/// предназначены для разработки новых версий функции uniqCombined.
+/// Пользователи должны использовать только uniqCombined.
+
+template <typename T>
+struct AggregateFunctionUniqCombinedRawData
+	: public BaseUniqCombinedData<T, HyperLogLogMode::Raw>
+{
+	static String getName() { return "uniqCombinedRaw"; }
+};
+
+template <typename T>
+struct AggregateFunctionUniqCombinedLinearCountingData
+	: public BaseUniqCombinedData<T, HyperLogLogMode::LinearCounting>
+{
+	static String getName() { return "uniqCombinedLinearCounting"; }
+};
+
+template <typename T>
+struct AggregateFunctionUniqCombinedBiasCorrectedData
+	: public BaseUniqCombinedData<T, HyperLogLogMode::BiasCorrected>
+{
+	static String getName() { return "uniqCombinedBiasCorrected"; }
+};
 
 template <typename T>
 struct AggregateFunctionUniqCombinedData
+	: public BaseUniqCombinedData<T, HyperLogLogMode::FullFeatured>
 {
-	using Key = UInt32;
-	using Set = CombinedCardinalityEstimator<Key, HashSet<Key, TrivialHash, HashTableGrower<> >, 16, 14, 17, TrivialHash>;
-	Set set;
-
-	static String getName() { return "uniqCombined"; }
-};
-
-template <>
-struct AggregateFunctionUniqCombinedData<String>
-{
-	using Key = UInt64;
-	using Set = CombinedCardinalityEstimator<Key, HashSet<Key, TrivialHash, HashTableGrower<> >, 16, 14, 17, TrivialHash>;
-	Set set;
-
 	static String getName() { return "uniqCombined"; }
 };
 
 namespace detail
 {
 
-/** Хэш-функция для uniqCombined.
-  */
-template<typename T, typename Enable = void>
-struct CombinedCardinalityTraits
-{
-	static UInt32 hash(T key)
-	{
-		return key;
-	}
-};
-
-template<typename T>
-struct CombinedCardinalityTraits<T, typename std::enable_if<std::is_same<T, Int64>::value>::type>
-{
-	using U = typename std::make_unsigned<T>::type;
-
-	static UInt32 hash(T key)
-	{
-		return intHash32<0>(static_cast<U>(key));
-	};
-};
-
-template<typename T>
-struct CombinedCardinalityTraits<T, typename std::enable_if<std::is_same<T, UInt64>::value>::type>
-{
-	static UInt32 hash(T key)
-	{
-		return intHash32<0>(key);
-	};
-};
-
-template<typename T>
-struct CombinedCardinalityTraits<T, typename std::enable_if<std::is_same<T, Float64>::value>::type>
-{
-	static UInt32 hash(T key)
-	{
-		UInt64 res = 0;
-		memcpy(reinterpret_cast<char *>(&res), reinterpret_cast<char *>(&key), sizeof(key));
-		return intHash32<0>(res);
-	}
-};
-
-template<typename T>
-struct CombinedCardinalityTraits<T, typename std::enable_if<std::is_same<T, Float32>::value>::type>
-{
-	static UInt32 hash(T key)
-	{
-		UInt32 res = 0;
-		memcpy(reinterpret_cast<char *>(&res), reinterpret_cast<char *>(&key), sizeof(key));
-		return res;
-	}
-};
-
 /** Хэш-функция для uniq.
   */
 template <typename T> struct AggregateFunctionUniqTraits
@@ -199,41 +196,51 @@ template <> struct AggregateFunctionUniqTraits<Float64>
 };
 
 /** Структура для делегации работы по добавлению одного элемента в агрегатные функции uniq.
-	* Используется для частичной специализации для добавления строк.
-	*/
-template<typename T, typename Data>
-struct OneAdder
-{
-	static void addOne(Data & data, const IColumn & column, size_t row_num)
-	{
-		data.set.insert(AggregateFunctionUniqTraits<T>::hash(static_cast<const ColumnVector<T> &>(column).getData()[row_num]));
-	}
-};
+  * Используется для частичной специализации для добавления строк.
+  */
+template <typename T, typename Data, typename Enable = void>
+struct OneAdder;
 
-template<typename Data>
-struct OneAdder<String, Data>
+template <typename T, typename Data>
+struct OneAdder<T, Data, typename std::enable_if<
+	std::is_same<Data, AggregateFunctionUniqUniquesHashSetData>::value ||
+	std::is_same<Data, AggregateFunctionUniqHLL12Data<T> >::value ||
+	std::is_same<Data, AggregateFunctionUniqCombinedRawData<T> >::value ||
+	std::is_same<Data, AggregateFunctionUniqCombinedLinearCountingData<T> >::value ||
+	std::is_same<Data, AggregateFunctionUniqCombinedBiasCorrectedData<T> >::value ||
+	std::is_same<Data, AggregateFunctionUniqCombinedData<T> >::value>::type>
 {
-	static void addOne(Data & data, const IColumn & column, size_t row_num)
+	template <typename T2 = T>
+	static void addOne(Data & data, const IColumn & column, size_t row_num,
+		typename std::enable_if<!std::is_same<T2, String>::value>::type * = nullptr)
+	{
+		const auto & value = static_cast<const ColumnVector<T2> &>(column).getData()[row_num];
+		data.set.insert(AggregateFunctionUniqTraits<T2>::hash(value));
+	}
+
+	template <typename T2 = T>
+	static void addOne(Data & data, const IColumn & column,	size_t row_num,
+		typename std::enable_if<std::is_same<T2, String>::value>::type * = nullptr)
 	{
-		/// Имейте ввиду, что вычисление приближённое.
 		StringRef value = column.getDataAt(row_num);
 		data.set.insert(CityHash64(value.data, value.size));
 	}
 };
 
-template<typename T>
-struct OneAdder<T, AggregateFunctionUniqExactData<T> >
+template <typename T, typename Data>
+struct OneAdder<T, Data, typename std::enable_if<
+	std::is_same<Data, AggregateFunctionUniqExactData<T> >::value>::type>
 {
-	static void addOne(AggregateFunctionUniqExactData<T> & data, const IColumn & column, size_t row_num)
+	template <typename T2 = T>
+	static void addOne(Data & data, const IColumn & column, size_t row_num,
+		typename std::enable_if<!std::is_same<T2, String>::value>::type * = nullptr)
 	{
-		data.set.insert(static_cast<const ColumnVector<T> &>(column).getData()[row_num]);
+		data.set.insert(static_cast<const ColumnVector<T2> &>(column).getData()[row_num]);
 	}
-};
 
-template<>
-struct OneAdder<String, AggregateFunctionUniqExactData<String> >
-{
-	static void addOne(AggregateFunctionUniqExactData<String> & data, const IColumn & column, size_t row_num)
+	template <typename T2 = T>
+	static void addOne(Data & data, const IColumn & column, size_t row_num,
+		typename std::enable_if<std::is_same<T2, String>::value>::type * = nullptr)
 	{
 		StringRef value = column.getDataAt(row_num);
 
@@ -246,26 +253,6 @@ struct OneAdder<String, AggregateFunctionUniqExactData<String> >
 	}
 };
 
-template<typename T>
-struct OneAdder<T, AggregateFunctionUniqCombinedData<T> >
-{
-	static void addOne(AggregateFunctionUniqCombinedData<T> & data, const IColumn & column, size_t row_num)
-	{
-		const auto & value = static_cast<const ColumnVector<T> &>(column).getData()[row_num];
-		data.set.insert(CombinedCardinalityTraits<T>::hash(value));
-	}
-};
-
-template<>
-struct OneAdder<String, AggregateFunctionUniqCombinedData<String> >
-{
-	static void addOne(AggregateFunctionUniqCombinedData<String> & data, const IColumn & column, size_t row_num)
-	{
-		StringRef value = column.getDataAt(row_num);
-		data.set.insert(CityHash64(value.data, value.size));
-	}
-};
-
 }
 
 

dbms/include/DB/AggregateFunctions/UniqCombinedBiasData.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include <array>
+
+namespace DB
+{
+
+/** Данные для HyperLogLogBiasEstimator в функции uniqCombined.
+  * Схема разработки следующая:
+  * 1. Собрать ClickHouse.
+  * 2. Запустить скрипт src/dbms/scripts/gen-bias-data.py, который возвращает один массив для getRawEstimates()
+  *    и другой массив для getBiases().
+  * 3. Обновить массивы raw_estimates и biases. Также обновить размер массивов в InterpolatedData.
+  * 4. Собрать ClickHouse.
+  * 5. Запустить скрипт src/dbms/scripts/linear-counting-threshold.py, который создаёт 3 файла:
+  *    - raw_graph.txt (1-й столбец: настоящее количество уникальных значений;
+  *      2-й столбец: относительная погрешность в случае HyperLogLog без применения каких-либо поправок)
+  *    - linear_counting_graph.txt (1-й столбец: настоящее количество уникальных значений;
+  *      2-й столбец: относительная погрешность в случае HyperLogLog с применением LinearCounting)
+  *    - bias_corrected_graph.txt (1-й столбец: настоящее количество уникальных значений;
+  *      2-й столбец: относительная погрешность в случае HyperLogLog с применением поправок из алгортима HyperLogLog++)
+  * 6. Сгенерить график с gnuplot на основе этих данных.
+  * 7. Определить минимальное количество уникальных значений, при котором лучше исправить погрешность
+  *    с помощью её оценки (т.е. по алгоритму HyperLogLog++), чем применить алгоритм LinearCounting.
+  * 7. Соответственно обновить константу в функции getThreshold()
+  * 8. Собрать ClickHouse.
+  */
+struct UniqCombinedBiasData
+{
+	using InterpolatedData = std::array<double, 178>;
+
+	static double getThreshold();
+	/// Оценки количества уникальных значений по алгоритму HyperLogLog без применения каких-либо поправок.
+	static const InterpolatedData & getRawEstimates();
+	/// Соответствующие оценки погрешности.
+	static const InterpolatedData & getBiases();
+};
+
+}

dbms/include/DB/Common/CombinedCardinalityEstimator.h

@@ -34,17 +34,32 @@ template
 	UInt8 medium_set_power2_max,
 	UInt8 K,
 	typename Hash = IntHash32<Key>,
+	typename HashValueType = UInt32,
+	typename BiasEstimator = TrivialBiasEstimator,
+	HyperLogLogMode mode = HyperLogLogMode::FullFeatured,
 	typename DenominatorType = double
 >
 class CombinedCardinalityEstimator
 {
 public:
-	using Self = CombinedCardinalityEstimator<Key, HashContainer, small_set_size_max, medium_set_power2_max, K, Hash, DenominatorType>;
+	using Self = CombinedCardinalityEstimator
+		<
+			Key,
+			HashContainer,
+			small_set_size_max,
+			medium_set_power2_max,
+			K,
+			Hash,
+			HashValueType,
+			BiasEstimator,
+			mode,
+			DenominatorType
+		>;
 
 private:
 	using Small = SmallSet<Key, small_set_size_max>;
 	using Medium = HashContainer;
-	using Large = HyperLogLogCounter<K, Hash, DenominatorType>;
+	using Large = HyperLogLogCounter<K, Hash, HashValueType, DenominatorType, BiasEstimator, mode>;
 
 public:
 	CombinedCardinalityEstimator()

dbms/include/DB/Common/HyperLogLogWithSmallSetOptimization.h

@@ -21,7 +21,7 @@ class HyperLogLogWithSmallSetOptimization
 {
 private:
 	using Small = SmallSet<Key, small_set_size>;
-	using Large = HyperLogLogCounter<K, Hash, DenominatorType>;
+	using Large = HyperLogLogCounter<K, Hash, UInt32, DenominatorType>;
 
 	Small small;
 	Large * large = nullptr;

dbms/include/DB/Core/Defines.h

@@ -45,7 +45,7 @@
   */
 #define DEFAULT_MERGE_BLOCK_SIZE 								8192
 
-#define DEFAULT_MAX_QUERY_SIZE 									65536
+#define DEFAULT_MAX_QUERY_SIZE 									262144
 #define SHOW_CHARS_ON_SYNTAX_ERROR 								160L
 #define DEFAULT_MAX_DISTRIBUTED_CONNECTIONS						1024
 #define DEFAULT_INTERACTIVE_DELAY								100000

dbms/include/DB/DataStreams/LimitBlockInputStream.h

@@ -16,7 +16,12 @@ using Poco::SharedPtr;
 class LimitBlockInputStream : public IProfilingBlockInputStream
 {
 public:
-	LimitBlockInputStream(BlockInputStreamPtr input_, size_t limit_, size_t offset_ = 0);
+	/** Если always_read_till_end = false (по-умолчанию), то после чтения достаточного количества данных,
+	  *  возвращает пустой блок, и это приводит к отмене выполнения запроса.
+	  * Если always_read_till_end = true - читает все данные до конца, но игнорирует их. Это нужно в редких случаях:
+	  *  когда иначе, из-за отмены запроса, мы бы не получили данные для GROUP BY WITH TOTALS с удалённого сервера.
+	  */
+	LimitBlockInputStream(BlockInputStreamPtr input_, size_t limit_, size_t offset_, bool always_read_till_end_ = false);
 
 	String getName() const override { return "Limit"; }
 
@@ -33,7 +38,8 @@ protected:
 private:
 	size_t limit;
 	size_t offset;
-	size_t pos;
+	size_t pos = 0;
+	bool always_read_till_end;
 };
 
 }

dbms/include/DB/Functions/FunctionsCoding.h

@@ -42,31 +42,9 @@ namespace DB
 const auto ipv4_bytes_length = 4;
 const auto ipv6_bytes_length = 16;
 
-class FunctionIPv6NumToString : public IFunction
+class IPv6Format
 {
-public:
-	static constexpr auto name = "IPv6NumToString";
-	static IFunction * create(const Context & context) { return new FunctionIPv6NumToString; }
-
-	String getName() const { return name; }
-
-	DataTypePtr getReturnType(const DataTypes & arguments) const
-	{
-		if (arguments.size() != 1)
-			throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
-			+ toString(arguments.size()) + ", should be 1.",
-			ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
-
-		const auto ptr = typeid_cast<const DataTypeFixedString *>(arguments[0].get());
-		if (!ptr || ptr->getN() != ipv6_bytes_length)
-			throw Exception("Illegal type " + arguments[0]->getName() +
-							" of argument of function " + getName() +
-							", expected FixedString(" + toString(ipv6_bytes_length) + ")",
-							ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
-
-		return new DataTypeString;
-	}
-
+private:
 	/// integer logarithm, return ceil(log(value, base)) (the smallest integer greater or equal  than log(value, base)
 	static constexpr uint32_t int_log(const uint32_t value, const uint32_t base, const bool carry = false)
 	{
@@ -99,23 +77,25 @@ public:
 	}
 
 	/// print IPv4 address as %u.%u.%u.%u
-	static void ipv4_format(const unsigned char * src, char *& dst)
+	static void ipv4_format(const unsigned char * src, char *& dst, UInt8 zeroed_tail_bytes_count)
 	{
-		constexpr auto size = sizeof(UInt32);
+		const auto limit = ipv4_bytes_length - zeroed_tail_bytes_count;
 
-		for (const auto i : ext::range(0, size))
+		for (const auto i : ext::range(0, ipv4_bytes_length))
 		{
-			print_integer<10, UInt8>(dst, src[i]);
+			UInt8 byte = (i < limit) ? src[i] : 0;
+			print_integer<10, UInt8>(dst, byte);
 
-			if (i != size - 1)
+			if (i != ipv4_bytes_length - 1)
 				*dst++ = '.';
 		}
 	}
 
+public:
 	/** rewritten inet_ntop6 from http://svn.apache.org/repos/asf/apr/apr/trunk/network_io/unix/inet_pton.c
 	 *	performs significantly faster than the reference implementation due to the absence of sprintf calls,
 	 *	bounds checking, unnecessary string copying and length calculation */
-	static const void ipv6_format(const unsigned char * src, char *& dst)
+	static const void apply(const unsigned char * src, char *& dst, UInt8 zeroed_tail_bytes_count = 0)
 	{
 		struct { int base, len; } best{-1}, cur{-1};
 		std::array<uint16_t, ipv6_bytes_length / sizeof(uint16_t)> words{};
@@ -123,7 +103,7 @@ public:
 		/** Preprocess:
 		 *	Copy the input (bytewise) array into a wordwise array.
 		 *	Find the longest run of 0x00's in src[] for :: shorthanding. */
-		for (const auto i : ext::range(0, ipv6_bytes_length))
+		for (const auto i : ext::range(0, ipv6_bytes_length - zeroed_tail_bytes_count))
 			words[i / 2] |= src[i] << ((1 - (i % 2)) << 3);
 
 		for (const auto i : ext::range(0, words.size()))
@@ -172,7 +152,7 @@ public:
 			/// Is this address an encapsulated IPv4?
 			if (i == 6 && best.base == 0 && (best.len == 6 || (best.len == 5 && words[5] == 0xffffu)))
 			{
-				ipv4_format(src + 12, dst);
+				ipv4_format(src + 12, dst, std::min(zeroed_tail_bytes_count, static_cast<UInt8>(ipv4_bytes_length)));
 				break;
 			}
 
@@ -185,6 +165,32 @@ public:
 
 		*dst++ = '\0';
 	}
+};
+
+class FunctionIPv6NumToString : public IFunction
+{
+public:
+	static constexpr auto name = "IPv6NumToString";
+	static IFunction * create(const Context & context) { return new FunctionIPv6NumToString; }
+
+	String getName() const { return name; }
+
+	DataTypePtr getReturnType(const DataTypes & arguments) const
+	{
+		if (arguments.size() != 1)
+			throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
+			+ toString(arguments.size()) + ", should be 1.",
+			ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+		const auto ptr = typeid_cast<const DataTypeFixedString *>(arguments[0].get());
+		if (!ptr || ptr->getN() != ipv6_bytes_length)
+			throw Exception("Illegal type " + arguments[0]->getName() +
+							" of argument of function " + getName() +
+							", expected FixedString(" + toString(ipv6_bytes_length) + ")",
+							ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+		return new DataTypeString;
+	}
 
 	void execute(Block & block, const ColumnNumbers & arguments, const size_t result)
 	{
@@ -216,7 +222,7 @@ public:
 
 			for (size_t offset = 0, i = 0; offset < vec_in.size(); offset += ipv6_bytes_length, ++i)
 			{
-				ipv6_format(&vec_in[offset], pos);
+				IPv6Format::apply(&vec_in[offset], pos);
 				offsets_res[i] = pos - begin;
 			}
 
@@ -236,7 +242,7 @@ public:
 
 			char buf[INET6_ADDRSTRLEN];
 			char * dst = buf;
-			ipv6_format(reinterpret_cast<const unsigned char *>(data_in.data()), dst);
+			IPv6Format::apply(reinterpret_cast<const unsigned char *>(data_in.data()), dst);
 
 			block.getByPosition(result).column = new ColumnConstString{col_in->size(), buf};
 		}
@@ -247,6 +253,172 @@ public:
 	}
 };
 
+class FunctionCutIPv6 : public IFunction
+{
+public:
+	static constexpr auto name = "cutIPv6";
+	static IFunction * create(const Context & context) { return new FunctionCutIPv6; }
+
+	String getName() const { return name; }
+
+	DataTypePtr getReturnType(const DataTypes & arguments) const
+	{
+		if (arguments.size() != 3)
+			throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
+			+ toString(arguments.size()) + ", should be 3.",
+			ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+		const auto ptr = typeid_cast<const DataTypeFixedString *>(arguments[0].get());
+		if (!ptr || ptr->getN() != ipv6_bytes_length)
+			throw Exception("Illegal type " + arguments[0]->getName() +
+							" of argument 1 of function " + getName() +
+							", expected FixedString(" + toString(ipv6_bytes_length) + ")",
+							ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+		if (!typeid_cast<const DataTypeUInt8 *>(arguments[1].get()))
+			throw Exception("Illegal type " + arguments[1]->getName() +
+							" of argument 2 of function " + getName(),
+							ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+		if (!typeid_cast<const DataTypeUInt8 *>(arguments[2].get()))
+			throw Exception("Illegal type " + arguments[2]->getName() +
+							" of argument 3 of function " + getName(),
+							ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+		return new DataTypeString;
+	}
+
+	void execute(Block & block, const ColumnNumbers & arguments, const size_t result)
+	{
+		const auto & col_name_type = block.getByPosition(arguments[0]);
+		const ColumnPtr & column = col_name_type.column;
+
+		const auto & col_ipv6_zeroed_tail_bytes_type = block.getByPosition(arguments[1]);
+		const auto & col_ipv6_zeroed_tail_bytes = col_ipv6_zeroed_tail_bytes_type.column;
+		const auto & col_ipv4_zeroed_tail_bytes_type = block.getByPosition(arguments[2]);
+		const auto & col_ipv4_zeroed_tail_bytes = col_ipv4_zeroed_tail_bytes_type.column;
+
+		if (const auto col_in = typeid_cast<const ColumnFixedString *>(column.get()))
+		{
+			if (col_in->getN() != ipv6_bytes_length)
+				throw Exception("Illegal type " + col_name_type.type->getName() +
+								" of column " + col_in->getName() +
+								" argument of function " + getName() +
+								", expected FixedString(" + toString(ipv6_bytes_length) + ")",
+								ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+			const auto ipv6_zeroed_tail_bytes = typeid_cast<const ColumnConst<UInt8> *>(col_ipv6_zeroed_tail_bytes.get());
+			if (!ipv6_zeroed_tail_bytes)
+				throw Exception("Illegal type " + col_ipv6_zeroed_tail_bytes_type.type->getName() +
+								" of argument 2 of function " + getName(),
+								ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+			UInt8 ipv6_zeroed_tail_bytes_count = ipv6_zeroed_tail_bytes->getData();
+			if (ipv6_zeroed_tail_bytes_count > ipv6_bytes_length)
+				throw Exception("Illegal value for argument 2 " + col_ipv6_zeroed_tail_bytes_type.type->getName() +
+								" of function " + getName(),
+								ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+			const auto ipv4_zeroed_tail_bytes = typeid_cast<const ColumnConst<UInt8> *>(col_ipv4_zeroed_tail_bytes.get());
+			if (!ipv4_zeroed_tail_bytes)
+				throw Exception("Illegal type " + col_ipv4_zeroed_tail_bytes_type.type->getName() +
+								" of argument 3 of function " + getName(),
+								ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+			UInt8 ipv4_zeroed_tail_bytes_count = ipv4_zeroed_tail_bytes->getData();
+			if (ipv4_zeroed_tail_bytes_count > ipv6_bytes_length)
+				throw Exception("Illegal value for argument 3 " + col_ipv4_zeroed_tail_bytes_type.type->getName() +
+								" of function " + getName(),
+								ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+			const auto size = col_in->size();
+			const auto & vec_in = col_in->getChars();
+
+			auto col_res = new ColumnString;
+			block.getByPosition(result).column = col_res;
+
+			ColumnString::Chars_t & vec_res = col_res->getChars();
+			ColumnString::Offsets_t & offsets_res = col_res->getOffsets();
+			vec_res.resize(size * INET6_ADDRSTRLEN);
+			offsets_res.resize(size);
+
+			auto begin = reinterpret_cast<char *>(&vec_res[0]);
+			auto pos = begin;
+
+			for (size_t offset = 0, i = 0; offset < vec_in.size(); offset += ipv6_bytes_length, ++i)
+			{
+				const auto address = &vec_in[offset];
+				UInt8 zeroed_tail_bytes_count = isIPv4Mapped(address) ? ipv4_zeroed_tail_bytes_count : ipv6_zeroed_tail_bytes_count;
+				cutAddress(address, pos, zeroed_tail_bytes_count);
+				offsets_res[i] = pos - begin;
+			}
+
+			vec_res.resize(pos - begin);
+		}
+		else if (const auto col_in = typeid_cast<const ColumnConst<String> *>(column.get()))
+		{
+			const auto data_type_fixed_string = typeid_cast<const DataTypeFixedString *>(col_in->getDataType().get());
+			if (!data_type_fixed_string || data_type_fixed_string->getN() != ipv6_bytes_length)
+				throw Exception("Illegal type " + col_name_type.type->getName() +
+								" of column " + col_in->getName() +
+								" argument of function " + getName() +
+								", expected FixedString(" + toString(ipv6_bytes_length) + ")",
+								ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+			const auto ipv6_zeroed_tail_bytes = typeid_cast<const ColumnConst<UInt8> *>(col_ipv6_zeroed_tail_bytes.get());
+			if (!ipv6_zeroed_tail_bytes)
+				throw Exception("Illegal type " + col_ipv6_zeroed_tail_bytes_type.type->getName() +
+								" of argument 2 of function " + getName(),
+								ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+			UInt8 ipv6_zeroed_tail_bytes_count = ipv6_zeroed_tail_bytes->getData();
+			if (ipv6_zeroed_tail_bytes_count > ipv6_bytes_length)
+				throw Exception("Illegal value for argument 2 " + col_ipv6_zeroed_tail_bytes_type.type->getName() +
+								" of function " + getName(),
+								ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+			const auto ipv4_zeroed_tail_bytes = typeid_cast<const ColumnConst<UInt8> *>(col_ipv4_zeroed_tail_bytes.get());
+			if (!ipv4_zeroed_tail_bytes)
+				throw Exception("Illegal type " + col_ipv4_zeroed_tail_bytes_type.type->getName() +
+								" of argument 3 of function " + getName(),
+								ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+			UInt8 ipv4_zeroed_tail_bytes_count = ipv4_zeroed_tail_bytes->getData();
+			if (ipv4_zeroed_tail_bytes_count > ipv6_bytes_length)
+				throw Exception("Illegal value for argument 3 " + col_ipv6_zeroed_tail_bytes_type.type->getName() +
+								" of function " + getName(),
+								ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+			const auto & data_in = col_in->getData();
+
+			char buf[INET6_ADDRSTRLEN];
+			char * dst = buf;
+
+			const auto address = reinterpret_cast<const unsigned char *>(data_in.data());
+			UInt8 zeroed_tail_bytes_count = isIPv4Mapped(address) ? ipv4_zeroed_tail_bytes_count : ipv6_zeroed_tail_bytes_count;
+			cutAddress(address, dst, zeroed_tail_bytes_count);
+
+			block.getByPosition(result).column = new ColumnConstString{col_in->size(), buf};
+		}
+		else
+			throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName()
+			+ " of argument of function " + getName(),
+			ErrorCodes::ILLEGAL_COLUMN);
+	}
+
+private:
+	bool isIPv4Mapped(const unsigned char * address) const
+	{
+		return (*reinterpret_cast<const UInt64 *>(&address[0]) == 0) &&
+			((*reinterpret_cast<const UInt64 *>(&address[8]) & 0x00000000FFFFFFFFull) == 0x00000000FFFF0000ull);
+	}
+
+	void cutAddress(const unsigned char * address, char *& dst, UInt8 zeroed_tail_bytes_count)
+	{
+		IPv6Format::apply(address, dst, zeroed_tail_bytes_count);
+	}
+};
+
 class FunctionIPv6StringToNum : public IFunction
 {
 public:
@@ -767,6 +939,69 @@ public:
 	}
 };
 
+class FunctionIPv4ToIPv6 : public IFunction
+{
+public:
+ 	static constexpr auto name = "IPv4ToIPv6";
+	static IFunction * create(const Context & context) { return new FunctionIPv4ToIPv6; }
+
+	String getName() const { return name; }
+
+	DataTypePtr getReturnType(const DataTypes & arguments) const
+	{
+		if (arguments.size() != 1)
+			throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
+			+ toString(arguments.size()) + ", should be 1.",
+			ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+		if (typeid_cast<const DataTypeUInt32 *>(arguments[0].get()) == nullptr)
+			throw Exception("Illegal type " + arguments[0]->getName() +
+							" of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+		return new DataTypeFixedString{16};
+	}
+
+	void execute(Block & block, const ColumnNumbers & arguments, const size_t result)
+	{
+		const auto & col_name_type = block.getByPosition(arguments[0]);
+		const ColumnPtr & column = col_name_type.column;
+
+		if (const auto col_in = typeid_cast<const ColumnVector<UInt32> *>(column.get()))
+		{
+			const auto col_res = new ColumnFixedString{ipv6_bytes_length};
+			block.getByPosition(result).column = col_res;
+
+			auto & vec_res = col_res->getChars();
+			vec_res.resize(col_in->size() * ipv6_bytes_length);
+
+			const auto & vec_in = col_in->getData();
+
+			for (size_t out_offset = 0, i = 0; out_offset < vec_res.size(); out_offset += ipv6_bytes_length, ++i)
+				mapIPv4ToIPv6(vec_in[i], &vec_res[out_offset]);
+		}
+		else if (const auto col_in = typeid_cast<const ColumnConst<UInt32> *>(column.get()))
+		{
+			std::string buf;
+			buf.resize(ipv6_bytes_length);
+			mapIPv4ToIPv6(col_in->getData(), reinterpret_cast<unsigned char *>(&buf[0]));
+
+			ColumnConstString * col_res = new ColumnConstString(ipv6_bytes_length, buf,
+				new DataTypeFixedString{ipv6_bytes_length});
+			block.getByPosition(result).column = col_res;
+		}
+		else
+			throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName()
+			+ " of argument of function " + getName(),
+			ErrorCodes::ILLEGAL_COLUMN);
+	}
+
+private:
+	void mapIPv4ToIPv6(UInt32 in, unsigned char * buf) const
+	{
+		*reinterpret_cast<UInt64 *>(&buf[0]) = 0;
+		*reinterpret_cast<UInt64 *>(&buf[8]) = 0x00000000FFFF0000ull | (static_cast<UInt64>(ntohl(in)) << 32);
+	}
+};
 
 class FunctionHex : public IFunction
 {
@@ -1447,6 +1682,7 @@ public:
 			};
 	}
 
+private:
 	template <typename T> bool execute(
 		Block & block, const ColumnNumbers & arguments, const size_t result,
 		const IColumn * const value_col_untyped)
@@ -1564,4 +1800,219 @@ public:
 	}
 };
 
+template <typename Impl>
+struct FunctionBitTestMany : public IFunction
+{
+public:
+	static constexpr auto name = Impl::name;
+	static IFunction * create(const Context &) { return new FunctionBitTestMany; }
+
+	String getName() const override { return name; }
+
+	DataTypePtr getReturnType(const DataTypes & arguments) const override
+	{
+		if (arguments.size() < 2)
+			throw Exception{
+				"Number of arguments for function " + getName() + " doesn't match: passed "
+				+ toString(arguments.size()) + ", should be at least 2.",
+				ErrorCodes::TOO_LESS_ARGUMENTS_FOR_FUNCTION
+			};
+
+		const auto first_arg = arguments.front().get();
+		if (!typeid_cast<const DataTypeUInt8 *>(first_arg) && !typeid_cast<const DataTypeUInt16 *>(first_arg) &&
+			!typeid_cast<const DataTypeUInt32 *>(first_arg) && !typeid_cast<const DataTypeUInt64 *>(first_arg) &&
+			!typeid_cast<const DataTypeInt8 *>(first_arg) && !typeid_cast<const DataTypeInt16 *>(first_arg) &&
+			!typeid_cast<const DataTypeInt32 *>(first_arg) && !typeid_cast<const DataTypeInt64 *>(first_arg))
+			throw Exception{
+				"Illegal type " + first_arg->getName() + " of first argument of function " + getName(),
+				ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT
+			};
+
+
+		for (const auto i : ext::range(1, arguments.size()))
+		{
+			const auto pos_arg = arguments[i].get();
+
+			if (!typeid_cast<const DataTypeUInt8 *>(pos_arg) && !typeid_cast<const DataTypeUInt16 *>(pos_arg) &&
+				!typeid_cast<const DataTypeUInt32 *>(pos_arg) && !typeid_cast<const DataTypeUInt64 *>(pos_arg))
+				throw Exception{
+					"Illegal type " + pos_arg->getName() + " of " + toString(i) + " argument of function " + getName(),
+					ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT
+				};
+		}
+
+		return new DataTypeUInt8;
+	}
+
+	void execute(Block & block, const ColumnNumbers & arguments, const size_t result) override
+	{
+		const auto value_col = block.getByPosition(arguments.front()).column.get();
+
+		if (!execute<UInt8>(block, arguments, result, value_col) &&
+			!execute<UInt16>(block, arguments, result, value_col) &&
+			!execute<UInt32>(block, arguments, result, value_col) &&
+			!execute<UInt64>(block, arguments, result, value_col) &&
+			!execute<Int8>(block, arguments, result, value_col) &&
+			!execute<Int16>(block, arguments, result, value_col) &&
+			!execute<Int32>(block, arguments, result, value_col) &&
+			!execute<Int64>(block, arguments, result, value_col))
+			throw Exception{
+				"Illegal column " + value_col->getName() + " of argument of function " + getName(),
+				ErrorCodes::ILLEGAL_COLUMN
+			};
+	}
+
+private:
+	template <typename T> bool execute(
+		Block & block, const ColumnNumbers & arguments, const size_t result,
+		const IColumn * const value_col_untyped)
+	{
+		if (const auto value_col = typeid_cast<const ColumnVector<T> *>(value_col_untyped))
+		{
+			const auto size = value_col->size();
+			bool is_const;
+			const auto mask = createConstMask<T>(size, block, arguments, is_const);
+			const auto & val = value_col->getData();
+
+			const auto out_col = new ColumnVector<UInt8>(size);
+			ColumnPtr out_col_ptr{out_col};
+			block.getByPosition(result).column = out_col_ptr;
+
+			auto & out = out_col->getData();
+
+			if (is_const)
+			{
+				for (const auto i : ext::range(0, size))
+					out[i] = Impl::combine(val[i], mask);
+			}
+			else
+			{
+				const auto mask = createMask<T>(size, block, arguments);
+
+				for (const auto i : ext::range(0, size))
+					out[i] = Impl::combine(val[i], mask[i]);
+			}
+
+			return true;
+		}
+		else if (const auto value_col = typeid_cast<const ColumnConst<T> *>(value_col_untyped))
+		{
+			const auto size = value_col->size();
+			bool is_const;
+			const auto mask = createConstMask<T>(size, block, arguments, is_const);
+			const auto val = value_col->getData();
+
+			if (is_const)
+			{
+				block.getByPosition(result).column = new ColumnConst<UInt8>{
+					size, Impl::combine(val, mask)
+				};
+			}
+			else
+			{
+				const auto mask = createMask<T>(size, block, arguments);
+				const auto out_col = new ColumnVector<UInt8>(size);
+				ColumnPtr out_col_ptr{out_col};
+				block.getByPosition(result).column = out_col_ptr;
+
+				auto & out = out_col->getData();
+
+				for (const auto i : ext::range(0, size))
+					out[i] = Impl::combine(val, mask[i]);
+			}
+
+			return true;
+		}
+
+		return false;
+	}
+
+	template <typename ValueType>
+	ValueType createConstMask(const std::size_t size, const Block & block, const ColumnNumbers & arguments, bool & is_const)
+	{
+		is_const = true;
+		ValueType mask{};
+
+		for (const auto i : ext::range(1, arguments.size()))
+		{
+			const auto pos_col = block.getByPosition(arguments[i]).column.get();
+
+			if (pos_col->isConst())
+			{
+				const auto pos = static_cast<const ColumnConst<ValueType> *>(pos_col)->getData();
+				mask = mask | 1 << pos;
+			}
+			else
+			{
+				is_const = false;
+				return {};
+			}
+		}
+
+		return mask;
+	}
+
+	template <typename ValueType>
+	PODArray<ValueType> createMask(const std::size_t size, const Block & block, const ColumnNumbers & arguments)
+	{
+		PODArray<ValueType> mask(size, ValueType{});
+
+		for (const auto i : ext::range(1, arguments.size()))
+		{
+			const auto pos_col = block.getByPosition(arguments[i]).column.get();
+
+			if (!addToMaskImpl<UInt8>(mask, pos_col) && !addToMaskImpl<UInt16>(mask, pos_col) &&
+				!addToMaskImpl<UInt32>(mask, pos_col) && !addToMaskImpl<UInt64>(mask, pos_col))
+				throw Exception{
+					"Illegal column " + pos_col->getName() + " of argument of function " + getName(),
+					ErrorCodes::ILLEGAL_COLUMN
+				};
+		}
+
+		return mask;
+	}
+
+	template <typename PosType, typename ValueType>
+	bool addToMaskImpl(PODArray<ValueType> & mask, const IColumn * const pos_col_untyped)
+	{
+		if (const auto pos_col = typeid_cast<const ColumnVector<PosType> *>(pos_col_untyped))
+		{
+			const auto & pos = pos_col->getData();
+
+			for (const auto i : ext::range(0, mask.size()))
+				mask[i] = mask[i] | (1 << pos[i]);
+
+			return true;
+		}
+		else if (const auto pos_col = typeid_cast<const ColumnConst<PosType> *>(pos_col_untyped))
+		{
+			const auto & pos = pos_col->getData();
+			const auto new_mask = 1 << pos;
+
+			for (const auto i : ext::range(0, mask.size()))
+				mask[i] = mask[i] | new_mask;
+
+			return true;
+		}
+
+		return false;
+	}
+};
+
+struct BitTestAnyImpl
+{
+	static constexpr auto name = "bitTestAny";
+	template <typename T> static UInt8 combine(const T val, const T mask) { return (val & mask) != 0; }
+};
+
+struct BitTestAllImpl
+{
+	static constexpr auto name = "bitTestAll";
+	template <typename T> static UInt8 combine(const T val, const T mask) { return (val & mask) == mask; }
+};
+
+using FunctionBitTestAny = FunctionBitTestMany<BitTestAnyImpl>;
+using FunctionBitTestAll = FunctionBitTestMany<BitTestAllImpl>;
+
+
 }

dbms/include/DB/Functions/FunctionsStringSearch.h

@@ -323,8 +323,13 @@ namespace Regexps
 		if (known_regexps.end() == it)
 			it = known_regexps.emplace(pattern, std::make_unique<Holder>()).first;
 
-		return it->second->get([&pattern] {
-			return new Regexp{createRegexp<like>(pattern, no_capture ? OptimizedRegularExpression::RE_NO_CAPTURE : 0)};
+		return it->second->get([&pattern]
+		{
+			int flags = OptimizedRegularExpression::RE_DOT_NL;
+			if (no_capture)
+				flags |= OptimizedRegularExpression::RE_NO_CAPTURE;
+
+			return new Regexp{createRegexp<like>(pattern, flags)};
 		});
 	}
 }

dbms/include/DB/IO/BufferWithOwnMemory.h

@@ -59,22 +59,27 @@ struct Memory : boost::noncopyable, Allocator
 
 	void resize(size_t new_size)
 	{
-		if (new_size < m_capacity)
+		if (0 == m_capacity)
+		{
+			m_size = m_capacity = new_size;
+			alloc();
+		}
+		else if (new_size < m_capacity)
 		{
 			m_size = new_size;
 			return;
 		}
 		else
 		{
-			new_size = align(new_size);
+			new_size = align(new_size, alignment);
+			/// @todo pointer to void can be converted to pointer to any type with static_cast by ISO C++, reinterpret_cast has no advantages
 			m_data = reinterpret_cast<char *>(Allocator::realloc(m_data, m_capacity, new_size, alignment));
 			m_capacity = new_size;
 			m_size = m_capacity;
 		}
 	}
 
-private:
-	size_t align(size_t value) const
+	static size_t align(const size_t value, const size_t alignment)
 	{
 		if (!alignment)
 			return value;
@@ -82,6 +87,7 @@ private:
 		return (value + alignment - 1) / alignment * alignment;
 	}
 
+private:
 	void alloc()
 	{
 		if (!m_capacity)
@@ -93,7 +99,8 @@ private:
 		ProfileEvents::increment(ProfileEvents::IOBufferAllocs);
 		ProfileEvents::increment(ProfileEvents::IOBufferAllocBytes, m_capacity);
 
-		size_t new_capacity = align(m_capacity);
+		size_t new_capacity = align(m_capacity, alignment);
+		/// @todo pointer to void can be converted to pointer to any type with static_cast by ISO C++, reinterpret_cast has no advantages
 		m_data = reinterpret_cast<char *>(Allocator::alloc(new_capacity, alignment));
 		m_capacity = new_capacity;
 		m_size = m_capacity;
@@ -104,6 +111,7 @@ private:
 		if (!m_data)
 			return;
 
+		/// @todo pointer to any type can be implicitly converted to pointer to void, no cast required
 		Allocator::free(reinterpret_cast<void *>(m_data), m_capacity);
 		m_data = nullptr;	/// Чтобы избежать double free, если последующий вызов alloc кинет исключение.
 	}

dbms/include/DB/IO/CachedCompressedReadBuffer.h

@@ -82,10 +82,11 @@ private:
 	}
 
 public:
-	CachedCompressedReadBuffer(const std::string & path_, UncompressedCache * cache_, size_t estimated_size_,
-							   size_t aio_threshold_, size_t buf_size_ = DBMS_DEFAULT_BUFFER_SIZE)
-		: ReadBuffer(nullptr, 0), path(path_), cache(cache_), buf_size(buf_size_),
-		estimated_size(estimated_size_), aio_threshold(aio_threshold_), file_pos(0)
+	CachedCompressedReadBuffer(
+		const std::string & path_, UncompressedCache * cache_, size_t estimated_size_, size_t aio_threshold_,
+		size_t buf_size_ = DBMS_DEFAULT_BUFFER_SIZE)
+		: ReadBuffer(nullptr, 0), path(path_), cache(cache_), buf_size(buf_size_), estimated_size(estimated_size_),
+		  aio_threshold(aio_threshold_), file_pos(0)
 	{
 	}
 

dbms/include/DB/IO/CompressedReadBufferFromFile.h

@@ -42,10 +42,11 @@ private:
 	}
 
 public:
-	CompressedReadBufferFromFile(const std::string & path, size_t estimated_size, size_t aio_threshold, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE)
+	CompressedReadBufferFromFile(
+		const std::string & path, size_t estimated_size, size_t aio_threshold, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE)
 		: BufferWithOwnMemory<ReadBuffer>(0),
-		p_file_in(createReadBufferFromFileBase(path, estimated_size, aio_threshold, buf_size)),
-		file_in(*p_file_in)
+		  p_file_in(createReadBufferFromFileBase(path, estimated_size, aio_threshold, buf_size)),
+		  file_in(*p_file_in)
 	{
 		compressed_in = &file_in;
 	}

dbms/include/DB/Interpreters/ExpressionAnalyzer.h

@@ -163,8 +163,8 @@ private:
 	  *  - в "левой" таблице, он будет доступен по имени expr(x), так как ещё не было выполнено действие Project.
 	  * Надо запомнить оба этих варианта.
 	  */
-	NameSet join_key_names_left_set;
-	NameSet join_key_names_right_set;
+	Names join_key_names_left;
+	Names join_key_names_right;
 
 	NamesAndTypesList columns_added_by_join;
 

dbms/include/DB/Interpreters/Settings.h

@@ -119,6 +119,9 @@ struct Settings
 	  * (Чтобы большие запросы не вымывали кэш.) */ \
 	M(SettingUInt64, merge_tree_max_rows_to_use_cache, (1024 * 1024)) \
 	\
+	/** Распределять чтение из MergeTree по потокам равномерно, обеспечивая стабильное среднее время исполнения каждого потока в пределах одного чтения. */ \
+	M(SettingBool, merge_tree_uniform_read_distribution, false) \
+	\
 	/** Минимальная длина выражения expr = x1 OR ... expr = xN для оптимизации */ \
 	M(SettingUInt64, optimize_min_equality_disjunction_chain_length, 3) \
 	\

dbms/include/DB/Storages/MergeTree/MarkRange.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <cstddef>
+
+
+namespace DB
+{
+
+
+/** Пара засечек, определяющая диапазон строк в куске. Именно, диапазон имеет вид [begin * index_granularity, end * index_granularity).
+  */
+struct MarkRange
+{
+	std::size_t begin;
+	std::size_t end;
+
+	MarkRange() = default;
+	MarkRange(const std::size_t begin, const std::size_t end) : begin{begin}, end{end} {}
+};
+
+using MarkRanges = std::vector<MarkRange>;
+
+
+}

dbms/include/DB/Storages/MergeTree/MergeTreeDataSelectExecutor.h

@@ -2,6 +2,7 @@
 
 #include <DB/Storages/MergeTree/MergeTreeData.h>
 #include <DB/Storages/MergeTree/MergeTreeReader.h>
+#include <DB/Storages/MergeTree/RangesInDataPart.h>
 
 
 namespace DB
@@ -33,22 +34,6 @@ private:
 
 	Logger * log;
 
-	struct RangesInDataPart
-	{
-		MergeTreeData::DataPartPtr data_part;
-		size_t part_index_in_query;
-		MarkRanges ranges;
-
-		RangesInDataPart() {}
-
-		RangesInDataPart(MergeTreeData::DataPartPtr data_part_, size_t part_index_in_query_)
-			: data_part(data_part_), part_index_in_query(part_index_in_query_)
-		{
-		}
-	};
-
-	typedef std::vector<RangesInDataPart> RangesInDataParts;
-
 	BlockInputStreams spreadMarkRangesAmongThreads(
 		RangesInDataParts parts,
 		size_t threads,

dbms/include/DB/Storages/MergeTree/MergeTreeReadPool.h

@@ -0,0 +1,435 @@
+#pragma once
+
+#include <DB/Core/NamesAndTypes.h>
+#include <DB/Storages/MergeTree/RangesInDataPart.h>
+#include <statdaemons/ext/range.hpp>
+#include <mutex>
+
+
+namespace DB
+{
+
+
+struct MergeTreeReadTask
+{
+	MergeTreeData::DataPartPtr data_part;
+	MarkRanges mark_ranges;
+	std::size_t part_index_in_query;
+	const Names & ordered_names;
+	const NameSet & column_name_set;
+	const NamesAndTypesList & columns;
+	const NamesAndTypesList & pre_columns;
+	const bool remove_prewhere_column;
+	const bool should_reorder;
+
+	MergeTreeReadTask(
+		const MergeTreeData::DataPartPtr & data_part, const MarkRanges & ranges, const std::size_t part_index_in_query,
+		const Names & ordered_names, const NameSet & column_name_set, const NamesAndTypesList & columns,
+		const NamesAndTypesList & pre_columns, const bool remove_prewhere_column, const bool should_reorder)
+		: data_part{data_part}, mark_ranges{ranges}, part_index_in_query{part_index_in_query},
+		  ordered_names{ordered_names}, column_name_set{column_name_set}, columns{columns}, pre_columns{pre_columns},
+		  remove_prewhere_column{remove_prewhere_column}, should_reorder{should_reorder}
+	{}
+};
+
+using MergeTreeReadTaskPtr = std::unique_ptr<MergeTreeReadTask>;
+
+class MergeTreeReadPool
+{
+public:
+	MergeTreeReadPool(
+		const std::size_t threads, const std::size_t sum_marks, const std::size_t min_marks_for_concurrent_read,
+		RangesInDataParts parts, MergeTreeData & data, const ExpressionActionsPtr & prewhere_actions,
+		const String & prewhere_column_name, const bool check_columns, const Names & column_names,
+		const bool do_not_steal_tasks = false)
+		: data{data}, column_names{column_names}, do_not_steal_tasks{do_not_steal_tasks}
+	{
+		const auto per_part_sum_marks = fillPerPartInfo(parts, prewhere_actions, prewhere_column_name, check_columns);
+		fillPerThreadInfo(threads, sum_marks, per_part_sum_marks, parts, min_marks_for_concurrent_read);
+	}
+
+	MergeTreeReadPool(const MergeTreeReadPool &) = delete;
+	MergeTreeReadPool & operator=(const MergeTreeReadPool &) = delete;
+
+	MergeTreeReadTaskPtr getTask(const std::size_t min_marks_to_read, const std::size_t thread)
+	{
+		const std::lock_guard<std::mutex> lock{mutex};
+
+		if (remaining_thread_tasks.empty())
+			return nullptr;
+
+		const auto tasks_remaining_for_this_thread = !threads_tasks[thread].sum_marks_in_parts.empty();
+		if (!tasks_remaining_for_this_thread && do_not_steal_tasks)
+			return nullptr;
+
+		const auto thread_idx = tasks_remaining_for_this_thread ? thread : *std::begin(remaining_thread_tasks);
+		auto & thread_tasks = threads_tasks[thread_idx];
+
+		auto & thread_task = thread_tasks.parts_and_ranges.back();
+		const auto part_idx = thread_task.part_idx;
+
+		auto & part = parts[part_idx];
+		auto & marks_in_part = thread_tasks.sum_marks_in_parts.back();
+
+		/// Берём весь кусок, если он достаточно мал
+		auto need_marks = std::min(marks_in_part, min_marks_to_read);
+
+		/// Не будем оставлять в куске слишком мало строк.
+		if (marks_in_part > need_marks &&
+			marks_in_part - need_marks < min_marks_to_read)
+			need_marks = marks_in_part;
+
+		MarkRanges ranges_to_get_from_part;
+
+		/// Возьмем весь кусок, если он достаточно мал.
+		if (marks_in_part <= need_marks)
+		{
+			const auto marks_to_get_from_range = marks_in_part;
+
+			/// Восстановим порядок отрезков.
+			std::reverse(thread_task.ranges.begin(), thread_task.ranges.end());
+
+			ranges_to_get_from_part = thread_task.ranges;
+
+			marks_in_part -= marks_to_get_from_range;
+
+			thread_tasks.parts_and_ranges.pop_back();
+			thread_tasks.sum_marks_in_parts.pop_back();
+
+			if (thread_tasks.sum_marks_in_parts.empty())
+				remaining_thread_tasks.erase(thread_idx);
+		}
+		else
+		{
+			/// Цикл по отрезкам куска.
+			while (need_marks > 0 && !thread_task.ranges.empty())
+			{
+				auto & range = thread_task.ranges.back();
+
+				const std::size_t marks_in_range = range.end - range.begin;
+				const std::size_t marks_to_get_from_range = std::min(marks_in_range, need_marks);
+
+				ranges_to_get_from_part.emplace_back(range.begin, range.begin + marks_to_get_from_range);
+				range.begin += marks_to_get_from_range;
+				if (range.begin == range.end)
+				{
+					std::swap(range, thread_task.ranges.back());
+					thread_task.ranges.pop_back();
+				}
+
+				marks_in_part -= marks_to_get_from_range;
+				need_marks -= marks_to_get_from_range;
+			}
+		}
+
+		return std::make_unique<MergeTreeReadTask>(
+			part.data_part, ranges_to_get_from_part, part.part_index_in_query, column_names,
+			per_part_column_name_set[part_idx], per_part_columns[part_idx], per_part_pre_columns[part_idx],
+			per_part_remove_prewhere_column[part_idx], per_part_should_reorder[part_idx]);
+	}
+
+public:
+	std::vector<std::size_t> fillPerPartInfo(
+		RangesInDataParts & parts, const ExpressionActionsPtr & prewhere_actions, const String & prewhere_column_name,
+		const bool check_columns)
+	{
+		std::vector<std::size_t> per_part_sum_marks;
+
+		for (const auto i : ext::range(0, parts.size()))
+		{
+			auto & part = parts[i];
+
+			/// Посчитаем засечки для каждого куска.
+			size_t sum_marks = 0;
+			/// Отрезки уже перечислены справа налево, reverse в MergeTreeDataSelectExecutor.
+			for (const auto & range : part.ranges)
+				sum_marks += range.end - range.begin;
+
+			per_part_sum_marks.push_back(sum_marks);
+
+			per_part_columns_lock.push_back(std::make_unique<Poco::ScopedReadRWLock>(
+				part.data_part->columns_lock));
+
+			/// inject column names required for DEFAULT evaluation in current part
+			auto required_column_names = column_names;
+
+			const auto injected_columns = injectRequiredColumns(part.data_part, required_column_names);
+			auto should_reoder = !injected_columns.empty();
+
+			Names required_pre_column_names;
+
+			if (prewhere_actions)
+			{
+				/// collect columns required for PREWHERE evaluation
+				required_pre_column_names = prewhere_actions->getRequiredColumns();
+
+				/// there must be at least one column required for PREWHERE
+				if (required_pre_column_names.empty())
+					required_pre_column_names.push_back(required_column_names[0]);
+
+				/// PREWHERE columns may require some additional columns for DEFAULT evaluation
+				const auto injected_pre_columns = injectRequiredColumns(part.data_part, required_pre_column_names);
+				if (!injected_pre_columns.empty())
+					should_reoder = true;
+
+				/// will be used to distinguish between PREWHERE and WHERE columns when applying filter
+				const NameSet pre_name_set{
+					std::begin(required_pre_column_names), std::end(required_pre_column_names)
+				};
+				/** Если выражение в PREWHERE - не столбец таблицы, не нужно отдавать наружу столбец с ним
+				 *	(от storage ожидают получить только столбцы таблицы). */
+				per_part_remove_prewhere_column.push_back(0 == pre_name_set.count(prewhere_column_name));
+
+				Names post_column_names;
+				for (const auto & name : required_column_names)
+					if (!pre_name_set.count(name))
+						post_column_names.push_back(name);
+
+				required_column_names = post_column_names;
+			}
+			else
+				per_part_remove_prewhere_column.push_back(false);
+
+			per_part_column_name_set.emplace_back(std::begin(required_column_names), std::end(required_column_names));
+
+			if (check_columns)
+			{
+				/** Под part->columns_lock проверим, что все запрошенные столбцы в куске того же типа, что в таблице.
+				 *	Это может быть не так во время ALTER MODIFY. */
+				if (!required_pre_column_names.empty())
+					data.check(part.data_part->columns, required_pre_column_names);
+				if (!required_column_names.empty())
+					data.check(part.data_part->columns, required_column_names);
+
+				per_part_pre_columns.push_back(data.getColumnsList().addTypes(required_pre_column_names));
+				per_part_columns.push_back(data.getColumnsList().addTypes(required_column_names));
+			}
+			else
+			{
+				per_part_pre_columns.push_back(part.data_part->columns.addTypes(required_pre_column_names));
+				per_part_columns.push_back(part.data_part->columns.addTypes(required_column_names));
+			}
+
+			per_part_should_reorder.push_back(should_reoder);
+
+			this->parts.push_back({ part.data_part, part.part_index_in_query });
+		}
+
+		return per_part_sum_marks;
+	}
+
+	void fillPerThreadInfo(
+		const std::size_t threads, const std::size_t sum_marks, std::vector<std::size_t> per_part_sum_marks,
+		RangesInDataParts & parts, const std::size_t min_marks_for_concurrent_read)
+	{
+		threads_tasks.resize(threads);
+
+		const size_t min_marks_per_thread = (sum_marks - 1) / threads + 1;
+
+		for (std::size_t i = 0; i < threads && !parts.empty(); ++i)
+		{
+			auto need_marks = min_marks_per_thread;
+
+			while (need_marks > 0 && !parts.empty())
+			{
+				const auto part_idx = parts.size() - 1;
+				RangesInDataPart & part = parts.back();
+				size_t & marks_in_part = per_part_sum_marks.back();
+
+				/// Не будем брать из куска слишком мало строк.
+				if (marks_in_part >= min_marks_for_concurrent_read &&
+					need_marks < min_marks_for_concurrent_read)
+					need_marks = min_marks_for_concurrent_read;
+
+				/// Не будем оставлять в куске слишком мало строк.
+				if (marks_in_part > need_marks &&
+					marks_in_part - need_marks < min_marks_for_concurrent_read)
+					need_marks = marks_in_part;
+
+				MarkRanges ranges_to_get_from_part;
+				size_t marks_in_ranges = need_marks;
+
+				/// Возьмем весь кусок, если он достаточно мал.
+				if (marks_in_part <= need_marks)
+				{
+					/// Оставим отрезки перечисленными справа налево для удобства.
+					ranges_to_get_from_part = part.ranges;
+					marks_in_ranges = marks_in_part;
+
+					need_marks -= marks_in_part;
+					parts.pop_back();
+					per_part_sum_marks.pop_back();
+				}
+				else
+				{
+					/// Цикл по отрезкам куска.
+					while (need_marks > 0)
+					{
+						if (part.ranges.empty())
+							throw Exception("Unexpected end of ranges while spreading marks among threads", ErrorCodes::LOGICAL_ERROR);
+
+						MarkRange & range = part.ranges.back();
+
+						const size_t marks_in_range = range.end - range.begin;
+						const size_t marks_to_get_from_range = std::min(marks_in_range, need_marks);
+
+						ranges_to_get_from_part.emplace_back(range.begin, range.begin + marks_to_get_from_range);
+						range.begin += marks_to_get_from_range;
+						marks_in_part -= marks_to_get_from_range;
+						need_marks -= marks_to_get_from_range;
+						if (range.begin == range.end)
+							part.ranges.pop_back();
+					}
+
+					/// Вновь перечислим отрезки справа налево, чтобы .getTask() мог забирать их с помощью .pop_back().
+					std::reverse(std::begin(ranges_to_get_from_part), std::end(ranges_to_get_from_part));
+				}
+
+				threads_tasks[i].parts_and_ranges.push_back({ part_idx, ranges_to_get_from_part });
+				threads_tasks[i].sum_marks_in_parts.push_back(marks_in_ranges);
+				if (marks_in_ranges != 0)
+					remaining_thread_tasks.insert(i);
+			}
+		}
+	}
+
+	/** Если некоторых запрошенных столбцов нет в куске,
+	 *	то выясняем, какие столбцы может быть необходимо дополнительно прочитать,
+	 *	чтобы можно было вычислить DEFAULT выражение для этих столбцов.
+	 *	Добавляет их в columns. */
+	NameSet injectRequiredColumns(const MergeTreeData::DataPartPtr & part, Names & columns) const
+	{
+		NameSet required_columns{std::begin(columns), std::end(columns)};
+		NameSet injected_columns;
+
+		auto all_column_files_missing = true;
+
+		for (size_t i = 0; i < columns.size(); ++i)
+		{
+			const auto & column_name = columns[i];
+
+			/// column has files and hence does not require evaluation
+			if (part->hasColumnFiles(column_name))
+			{
+				all_column_files_missing = false;
+				continue;
+			}
+
+			const auto default_it = data.column_defaults.find(column_name);
+			/// columns has no explicit default expression
+			if (default_it == std::end(data.column_defaults))
+				continue;
+
+			/// collect identifiers required for evaluation
+			IdentifierNameSet identifiers;
+			default_it->second.expression->collectIdentifierNames(identifiers);
+
+			for (const auto & identifier : identifiers)
+			{
+				if (data.hasColumn(identifier))
+				{
+					/// ensure each column is added only once
+					if (required_columns.count(identifier) == 0)
+					{
+						columns.emplace_back(identifier);
+						required_columns.emplace(identifier);
+						injected_columns.emplace(identifier);
+					}
+				}
+			}
+		}
+
+		if (all_column_files_missing)
+		{
+			addMinimumSizeColumn(part, columns);
+			/// correctly report added column
+			injected_columns.insert(columns.back());
+		}
+
+		return injected_columns;
+	}
+
+	/** Добавить столбец минимального размера.
+	  * Используется в случае, когда ни один столбец не нужен, но нужно хотя бы знать количество строк.
+	  * Добавляет в columns.
+	  */
+	void addMinimumSizeColumn(const MergeTreeData::DataPartPtr & part, Names & columns) const
+	{
+		const auto get_column_size = [this, &part] (const String & name) {
+			const auto & files = part->checksums.files;
+
+			const auto escaped_name = escapeForFileName(name);
+			const auto bin_file_name = escaped_name + ".bin";
+			const auto mrk_file_name = escaped_name + ".mrk";
+
+			return files.find(bin_file_name)->second.file_size + files.find(mrk_file_name)->second.file_size;
+		};
+
+		const auto & storage_columns = data.getColumnsList();
+		const NameAndTypePair * minimum_size_column = nullptr;
+		auto minimum_size = std::numeric_limits<size_t>::max();
+
+		for (const auto & column : storage_columns)
+		{
+			if (!part->hasColumnFiles(column.name))
+				continue;
+
+			const auto size = get_column_size(column.name);
+			if (size < minimum_size)
+			{
+				minimum_size = size;
+				minimum_size_column = &column;
+			}
+		}
+
+		if (!minimum_size_column)
+			throw Exception{
+				"Could not find a column of minimum size in MergeTree",
+				ErrorCodes::LOGICAL_ERROR
+			};
+
+		columns.push_back(minimum_size_column->name);
+	}
+
+	std::vector<std::unique_ptr<Poco::ScopedReadRWLock>> per_part_columns_lock;
+	MergeTreeData & data;
+	Names column_names;
+	const bool do_not_steal_tasks;
+	std::vector<NameSet> per_part_column_name_set;
+	std::vector<NamesAndTypesList> per_part_columns;
+	std::vector<NamesAndTypesList> per_part_pre_columns;
+	/// @todo actually all of these values are either true or false for the whole query, thus no vector required
+	std::vector<bool> per_part_remove_prewhere_column;
+	std::vector<bool> per_part_should_reorder;
+
+	struct part_t
+	{
+		MergeTreeData::DataPartPtr data_part;
+		std::size_t part_index_in_query;
+	};
+
+	std::vector<part_t> parts;
+
+	struct thread_task_t
+	{
+		struct part_index_and_range_t
+		{
+			std::size_t part_idx;
+			MarkRanges ranges;
+		};
+
+		std::vector<part_index_and_range_t> parts_and_ranges;
+		std::vector<std::size_t> sum_marks_in_parts;
+	};
+
+	std::vector<thread_task_t> threads_tasks;
+
+	std::unordered_set<std::size_t> remaining_thread_tasks;
+
+	mutable std::mutex mutex;
+};
+
+using MergeTreeReadPoolPtr = std::shared_ptr<MergeTreeReadPool>;
+
+
+}

dbms/include/DB/Storages/MergeTree/MergeTreeReader.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <DB/Storages/MarkCache.h>
+#include <DB/Storages/MergeTree/MarkRange.h>
 #include <DB/Storages/MergeTree/MergeTreeData.h>
 #include <DB/DataTypes/IDataType.h>
 #include <DB/DataTypes/DataTypeNested.h>
@@ -17,19 +18,6 @@
 namespace DB
 {
 
-/** Пара засечек, определяющая диапазон строк в куске. Именно, диапазон имеет вид [begin * index_granularity, end * index_granularity).
-  */
-struct MarkRange
-{
-	size_t begin;
-	size_t end;
-
-	MarkRange() {}
-	MarkRange(size_t begin_, size_t end_) : begin(begin_), end(end_) {}
-};
-
-typedef std::vector<MarkRange> MarkRanges;
-
 
 /** Умеет читать данные между парой засечек из одного куска. При чтении последовательных отрезков не делает лишних seek-ов.
   * При чтении почти последовательных отрезков делает seek-и быстро, не выбрасывая содержимое буфера.
@@ -44,11 +32,23 @@ public:
 		UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_,
 		MergeTreeData & storage_, const MarkRanges & all_mark_ranges,
 		size_t aio_threshold_, size_t max_read_buffer_size_)
-		: path(path_), data_part(data_part), part_name(data_part->name), columns(columns_),
-		uncompressed_cache(uncompressed_cache_), mark_cache(mark_cache_),
-		storage(storage_), all_mark_ranges(all_mark_ranges),
-		aio_threshold(aio_threshold_), max_read_buffer_size(max_read_buffer_size_)
+		: uncompressed_cache(uncompressed_cache_), mark_cache(mark_cache_), storage(storage_),
+		  aio_threshold(aio_threshold_), max_read_buffer_size(max_read_buffer_size_)
 	{
+		reconf(path_, data_part, columns_, all_mark_ranges);
+	}
+
+	void reconf(
+		const String & path, const MergeTreeData::DataPartPtr & data_part, const NamesAndTypesList & columns,
+		const MarkRanges & all_mark_ranges)
+	{
+		this->path = path;
+		this->data_part = data_part;
+		this->part_name = data_part->name;
+		this->columns = columns;
+		this->all_mark_ranges = all_mark_ranges;
+		this->streams.clear();
+
 		try
 		{
 			if (!Poco::File(path).exists())
@@ -74,20 +74,14 @@ public:
 		{
 			size_t max_rows_to_read = (to_mark - from_mark) * storage.index_granularity;
 
-			/** Для некоторых столбцов файлы с данными могут отсутствовать.
-				* Это бывает для старых кусков, после добавления новых столбцов в структуру таблицы.
-				*/
-			auto has_missing_columns = false;
-
 			/// Указатели на столбцы смещений, общие для столбцов из вложенных структур данных
 			/// Если append, все значения nullptr, и offset_columns используется только для проверки, что столбец смещений уже прочитан.
 			OffsetColumns offset_columns;
-			const auto read_column = [&] (const NameAndTypePair & it) {
+
+			for (const NameAndTypePair & it : columns)
+			{
 				if (streams.end() == streams.find(it.name))
-				{
-					has_missing_columns = true;
-					return;
-				}
+					continue;
 
 				/// Все столбцы уже есть в блоке. Будем добавлять значения в конец.
 				bool append = res.has(it.name);
@@ -120,24 +114,12 @@ public:
 
 				if (!append && column.column->size())
 					res.insert(column);
-			};
-
-			for (const NameAndTypePair & it : columns)
-				read_column(it);
-
-			if (has_missing_columns && !res)
-			{
-				addMinimumSizeColumn();
-				/// minimum size column is necessarily at list's front
-				read_column(columns.front());
 			}
 		}
 		catch (const Exception & e)
 		{
 			if (e.code() != ErrorCodes::MEMORY_LIMIT_EXCEEDED)
-			{
 				storage.reportBrokenPart(part_name);
-			}
 
 			/// Более хорошая диагностика.
 			throw Exception(e.message() +  "\n(while reading from part " + path + " from mark " + toString(from_mark) + " to "
@@ -151,60 +133,13 @@ public:
 		}
 	}
 
-
-	/** Добавить столбец минимального размера.
-	  * Используется в случае, когда ни один столбец не нужен, но нужно хотя бы знать количество строк.
-	  * Добавляет в columns.
-	  */
-	void addMinimumSizeColumn()
-	{
-		const auto get_column_size = [this] (const String & name) {
-			const auto & files = data_part->checksums.files;
-
-			const auto escaped_name = escapeForFileName(name);
-			const auto bin_file_name = escaped_name + ".bin";
-			const auto mrk_file_name = escaped_name + ".mrk";
-
-			return files.find(bin_file_name)->second.file_size + files.find(mrk_file_name)->second.file_size;
-		};
-
-		const auto & storage_columns = storage.getColumnsList();
-		const NameAndTypePair * minimum_size_column = nullptr;
-		auto minimum_size = std::numeric_limits<size_t>::max();
-
-		for (const auto & column : storage_columns)
-		{
-			if (!data_part->hasColumnFiles(column.name))
-				continue;
-
-			const auto size = get_column_size(column.name);
-			if (size < minimum_size)
-			{
-				minimum_size = size;
-				minimum_size_column = &column;
-			}
-		}
-
-		if (!minimum_size_column)
-			throw Exception{
-				"could not find a column of minimum size in MergeTree",
-				ErrorCodes::LOGICAL_ERROR
-			};
-
-		addStream(minimum_size_column->name, *minimum_size_column->type, all_mark_ranges);
-		columns.emplace(std::begin(columns), *minimum_size_column);
-
-		added_minimum_size_column = &columns.front();
-	}
-
-
 	/** Добавляет в блок недостающие столбцы из ordered_names, состоящие из значений по-умолчанию.
 	  * Недостающие столбцы добавляются в позиции, такие же как в ordered_names.
 	  * Если был добавлен хотя бы один столбец - то все столбцы в блоке переупорядочиваются как в ordered_names.
 	  */
-	void fillMissingColumns(Block & res, const Names & ordered_names)
+	void fillMissingColumns(Block & res, const Names & ordered_names, const bool always_reorder = false)
 	{
-		fillMissingColumnsImpl(res, ordered_names, false);
+		fillMissingColumnsImpl(res, ordered_names, always_reorder);
 	}
 
 	/** То же самое, но всегда переупорядочивает столбцы в блоке, как в ordered_names
@@ -220,16 +155,14 @@ private:
 	{
 		MarkCache::MappedPtr marks;
 		ReadBuffer * data_buffer;
-		Poco::SharedPtr<CachedCompressedReadBuffer> cached_buffer;
-		Poco::SharedPtr<CompressedReadBufferFromFile> non_cached_buffer;
+		std::unique_ptr<CachedCompressedReadBuffer> cached_buffer;
+		std::unique_ptr<CompressedReadBufferFromFile> non_cached_buffer;
 		std::string path_prefix;
 		size_t max_mark_range;
 
-		/// Используется в качестве подсказки, чтобы уменьшить количество реаллокаций при создании столбца переменной длины.
-		double avg_value_size_hint = 0;
-
-		Stream(const String & path_prefix_, UncompressedCache * uncompressed_cache, MarkCache * mark_cache, const MarkRanges & all_mark_ranges,
-			   size_t aio_threshold, size_t max_read_buffer_size)
+		Stream(
+			const String & path_prefix_, UncompressedCache * uncompressed_cache, MarkCache * mark_cache,
+			const MarkRanges & all_mark_ranges, size_t aio_threshold, size_t max_read_buffer_size)
 			: path_prefix(path_prefix_)
 		{
 			loadMarks(mark_cache);
@@ -281,15 +214,15 @@ private:
 
 			if (uncompressed_cache)
 			{
-				cached_buffer = new CachedCompressedReadBuffer(path_prefix + ".bin", uncompressed_cache,
-															   estimated_size, aio_threshold, buffer_size);
-				data_buffer = &*cached_buffer;
+				cached_buffer = std::make_unique<CachedCompressedReadBuffer>(
+					path_prefix + ".bin", uncompressed_cache, estimated_size, aio_threshold, buffer_size);
+				data_buffer = cached_buffer.get();
 			}
 			else
 			{
-				non_cached_buffer = new CompressedReadBufferFromFile(path_prefix + ".bin", estimated_size,
-																	 aio_threshold, buffer_size);
-				data_buffer = &*non_cached_buffer;
+				non_cached_buffer = std::make_unique<CompressedReadBufferFromFile>(
+					path_prefix + ".bin", estimated_size, aio_threshold, buffer_size);
+				data_buffer = non_cached_buffer.get();
 			}
 		}
 
@@ -350,20 +283,21 @@ private:
 
 	typedef std::map<std::string, std::unique_ptr<Stream> > FileStreams;
 
+	/// Используется в качестве подсказки, чтобы уменьшить количество реаллокаций при создании столбца переменной длины.
+	std::map<std::string, double> avg_value_size_hints;
 	String path;
-	const MergeTreeData::DataPartPtr & data_part;
+	MergeTreeData::DataPartPtr data_part;
 	String part_name;
 	FileStreams streams;
 
-	/// Запрашиваемые столбцы. Возможно, с добавлением minimum_size_column.
+	/// Запрашиваемые столбцы.
 	NamesAndTypesList columns;
-	const NameAndTypePair * added_minimum_size_column = nullptr;
 
 	UncompressedCache * uncompressed_cache;
 	MarkCache * mark_cache;
 
 	MergeTreeData & storage;
-	const MarkRanges & all_mark_ranges;
+	MarkRanges all_mark_ranges;
 	size_t aio_threshold;
 	size_t max_read_buffer_size;
 
@@ -386,14 +320,16 @@ private:
 				+ ARRAY_SIZES_COLUMN_NAME_SUFFIX + toString(level);
 
 			if (!streams.count(size_name))
-				streams.emplace(size_name, std::unique_ptr<Stream>(new Stream(
-					path + escaped_size_name, uncompressed_cache, mark_cache, all_mark_ranges, aio_threshold, max_read_buffer_size)));
+				streams.emplace(size_name, std::make_unique<Stream>(
+					path + escaped_size_name, uncompressed_cache, mark_cache,
+					all_mark_ranges, aio_threshold, max_read_buffer_size));
 
 			addStream(name, *type_arr->getNestedType(), all_mark_ranges, level + 1);
 		}
 		else
-			streams[name].reset(new Stream(
-				path + escaped_column_name, uncompressed_cache, mark_cache, all_mark_ranges, aio_threshold, max_read_buffer_size));
+			streams.emplace(name, std::make_unique<Stream>(
+				path + escaped_column_name, uncompressed_cache, mark_cache,
+				all_mark_ranges, aio_threshold, max_read_buffer_size));
 	}
 
 
@@ -453,8 +389,9 @@ private:
 		else
 		{
 			Stream & stream = *streams[name];
+			double & avg_value_size_hint = avg_value_size_hints[name];
 			stream.seekToMark(from_mark);
-			type.deserializeBinary(column, *stream.data_buffer, max_rows_to_read, stream.avg_value_size_hint);
+			type.deserializeBinary(column, *stream.data_buffer, max_rows_to_read, avg_value_size_hint);
 
 			/// Вычисление подсказки о среднем размере значения.
 			size_t column_size = column.size();
@@ -463,10 +400,10 @@ private:
 				double current_avg_value_size = static_cast<double>(column.byteSize()) / column_size;
 
 				/// Эвристика, чтобы при изменениях, значение avg_value_size_hint быстро росло, но медленно уменьшалось.
-				if (current_avg_value_size > stream.avg_value_size_hint)
-					stream.avg_value_size_hint = current_avg_value_size;
-				else if (current_avg_value_size * 2 < stream.avg_value_size_hint)
-					stream.avg_value_size_hint = (current_avg_value_size + stream.avg_value_size_hint * 3) / 4;
+				if (current_avg_value_size > avg_value_size_hint)
+					avg_value_size_hint = current_avg_value_size;
+				else if (current_avg_value_size * 2 < avg_value_size_hint)
+					avg_value_size_hint = (current_avg_value_size + avg_value_size_hint * 3) / 4;
 			}
 		}
 	}
@@ -548,15 +485,6 @@ private:
 			if (should_evaluate_defaults)
 				evaluateMissingDefaults(res, columns, storage.column_defaults, storage.context);
 
-			/// remove added column to ensure same content among all blocks
-			if (added_minimum_size_column)
-			{
-				res.erase(0);
-				streams.erase(added_minimum_size_column->name);
-				columns.erase(std::begin(columns));
-				added_minimum_size_column = nullptr;
-			}
-
 			/// sort columns to ensure consistent order among all blocks
 			if (should_sort)
 			{
@@ -566,12 +494,6 @@ private:
 					if (res.has(name))
 						ordered_block.insert(res.getByName(name));
 
-				if (res.columns() != ordered_block.columns())
-					throw Exception{
-						"Ordered block has different number of columns than original one:\n" +
-							ordered_block.dumpNames() + "\nvs.\n" + res.dumpNames(),
-						ErrorCodes::LOGICAL_ERROR};
-
 				std::swap(res, ordered_block);
 			}
 		}

dbms/include/DB/Storages/MergeTree/MergeTreeSettings.h

@@ -66,7 +66,7 @@ struct MergeTreeSettings
 	size_t replicated_logs_to_keep = 100;
 
 	/// Настройки минимального количества битых данных, при котором отказываться автоматически их удалять.
-	size_t max_suspicious_broken_parts = 5;
+	size_t max_suspicious_broken_parts = 10;
 
 	/// Максимальное количество ошибок при загрузке кусков, при котором ReplicatedMergeTree соглашается запускаться.
 	size_t replicated_max_unexpected_parts = 3;

dbms/include/DB/Storages/MergeTree/MergeTreeThreadBlockInputStream.h

@@ -0,0 +1,342 @@
+#pragma once
+
+#include <DB/DataStreams/IProfilingBlockInputStream.h>
+#include <DB/Storages/MergeTree/MergeTreeData.h>
+#include <DB/Storages/MergeTree/PKCondition.h>
+#include <DB/Storages/MergeTree/MergeTreeReader.h>
+#include <DB/Storages/MergeTree/MergeTreeReadPool.h>
+
+
+namespace DB
+{
+
+
+class MergeTreeThreadBlockInputStream : public IProfilingBlockInputStream
+{
+	std::size_t thread;
+public:
+	MergeTreeThreadBlockInputStream(
+		const std::size_t thread,
+		const MergeTreeReadPoolPtr & pool, const std::size_t min_marks_to_read, const std::size_t block_size,
+		MergeTreeData & storage, const bool use_uncompressed_cache, const ExpressionActionsPtr & prewhere_actions,
+		const String & prewhere_column, const Settings & settings, const Names & virt_column_names)
+		: thread{thread}, pool{pool}, block_size_marks{block_size / storage.index_granularity},
+		  /// round min_marks_to_read up to nearest multiple of block_size expressed in marks
+		  min_marks_to_read{block_size
+							? (min_marks_to_read * storage.index_granularity + block_size - 1)
+							  / block_size * block_size / storage.index_granularity
+							: min_marks_to_read
+		  },
+		  storage{storage}, use_uncompressed_cache{use_uncompressed_cache}, prewhere_actions{prewhere_actions},
+		  prewhere_column{prewhere_column}, min_bytes_to_use_direct_io{settings.min_bytes_to_use_direct_io},
+		  max_read_buffer_size{settings.max_read_buffer_size}, virt_column_names{virt_column_names},
+		  log{&Logger::get("MergeTreeThreadBlockInputStream")}
+	{}
+
+	String getName() const override { return "MergeTreeThread"; }
+
+	String getID() const override
+	{
+		std::stringstream res;
+		/// @todo print some meaningful information
+//		res << "MergeTreeThread(columns";
+//
+//		for (const auto & column : columns)
+//			res << ", " << column.name;
+//
+//		if (prewhere_actions)
+//			res << ", prewhere, " << prewhere_actions->getID();
+//
+//		res << ", marks";
+//
+//		for (size_t i = 0; i < all_mark_ranges.size(); ++i)
+//			res << ", " << all_mark_ranges[i].begin << ", " << all_mark_ranges[i].end;
+//
+//		res << ")";
+		return res.str();
+	}
+
+protected:
+	/// Будем вызывать progressImpl самостоятельно.
+	void progress(const Progress & value) override {}
+
+	Block readImpl() override
+	{
+		Block res;
+
+		while (!res)
+		{
+			if (!task && !getNewTask())
+				break;
+
+			res = readFromPart();
+
+			if (res)
+				injectVirtualColumns(res);
+
+			if (task->mark_ranges.empty())
+				task = {};
+		}
+
+		return res;
+	}
+
+private:
+	bool getNewTask()
+	{
+		task = pool->getTask(min_marks_to_read, thread);
+
+		if (!task)
+		{
+			/** Закрываем файлы (ещё до уничтожения объекта).
+			 *	Чтобы при создании многих источников, но одновременном чтении только из нескольких,
+			 *	буферы не висели в памяти. */
+			reader = {};
+			pre_reader = {};
+			return false;
+		}
+
+		const auto path = storage.getFullPath() + task->data_part->name + '/';
+
+		if (!reader)
+		{
+			if (use_uncompressed_cache)
+				owned_uncompressed_cache = storage.context.getUncompressedCache();
+
+			owned_mark_cache = storage.context.getMarkCache();
+
+			reader = std::make_unique<MergeTreeReader>(
+				path, task->data_part, task->columns, owned_uncompressed_cache.get(), owned_mark_cache.get(),
+				storage, task->mark_ranges, min_bytes_to_use_direct_io, max_read_buffer_size);
+
+			if (prewhere_actions)
+				pre_reader = std::make_unique<MergeTreeReader>(
+					path, task->data_part, task->pre_columns, owned_uncompressed_cache.get(),
+					owned_mark_cache.get(), storage, task->mark_ranges, min_bytes_to_use_direct_io,
+					max_read_buffer_size);
+		}
+		else
+		{
+			reader->reconf(path, task->data_part, task->columns, task->mark_ranges);
+			if (prewhere_actions)
+				pre_reader->reconf(path, task->data_part, task->pre_columns, task->mark_ranges);
+		}
+
+		return true;
+	}
+
+	Block readFromPart()
+	{
+		Block res;
+
+		if (prewhere_actions)
+		{
+			do
+			{
+				/// Прочитаем полный блок столбцов, нужных для вычисления выражения в PREWHERE.
+				size_t space_left = std::max(1LU, block_size_marks);
+				MarkRanges ranges_to_read;
+				while (!task->mark_ranges.empty() && space_left)
+				{
+					auto & range = task->mark_ranges.back();
+
+					size_t marks_to_read = std::min(range.end - range.begin, space_left);
+					pre_reader->readRange(range.begin, range.begin + marks_to_read, res);
+
+					ranges_to_read.emplace_back(range.begin, range.begin + marks_to_read);
+					space_left -= marks_to_read;
+					range.begin += marks_to_read;
+					if (range.begin == range.end)
+						task->mark_ranges.pop_back();
+				}
+				progressImpl({ res.rowsInFirstColumn(), res.bytes() });
+				pre_reader->fillMissingColumns(res, task->ordered_names, task->should_reorder);
+
+				/// Вычислим выражение в PREWHERE.
+				prewhere_actions->execute(res);
+
+				ColumnPtr column = res.getByName(prewhere_column).column;
+				if (task->remove_prewhere_column)
+					res.erase(prewhere_column);
+
+				const auto pre_bytes = res.bytes();
+
+				/** Если фильтр - константа (например, написано PREWHERE 1),
+				  *  то либо вернём пустой блок, либо вернём блок без изменений.
+				  */
+				if (const auto column_const = typeid_cast<const ColumnConstUInt8 *>(column.get()))
+				{
+					if (!column_const->getData())
+					{
+						res.clear();
+						return res;
+					}
+
+					for (const auto & range : ranges_to_read)
+						reader->readRange(range.begin, range.end, res);
+
+					progressImpl({ 0, res.bytes() - pre_bytes });
+				}
+				else if (const auto column_vec = typeid_cast<const ColumnUInt8 *>(column.get()))
+				{
+					size_t index_granularity = storage.index_granularity;
+
+					const auto & pre_filter = column_vec->getData();
+					IColumn::Filter post_filter(pre_filter.size());
+
+					/// Прочитаем в нужных отрезках остальные столбцы и составим для них свой фильтр.
+					size_t pre_filter_pos = 0;
+					size_t post_filter_pos = 0;
+
+					for (const auto & range : ranges_to_read)
+					{
+						auto begin = range.begin;
+						auto pre_filter_begin_pos = pre_filter_pos;
+
+						for (auto mark = range.begin; mark <= range.end; ++mark)
+						{
+							UInt8 nonzero = 0;
+
+							if (mark != range.end)
+							{
+								const size_t limit = std::min(pre_filter.size(), pre_filter_pos + index_granularity);
+								for (size_t row = pre_filter_pos; row < limit; ++row)
+									nonzero |= pre_filter[row];
+							}
+
+							if (!nonzero)
+							{
+								if (mark > begin)
+								{
+									memcpy(
+										&post_filter[post_filter_pos],
+										&pre_filter[pre_filter_begin_pos],
+										pre_filter_pos - pre_filter_begin_pos);
+									post_filter_pos += pre_filter_pos - pre_filter_begin_pos;
+									reader->readRange(begin, mark, res);
+								}
+								begin = mark + 1;
+								pre_filter_begin_pos = std::min(pre_filter_pos + index_granularity, pre_filter.size());
+							}
+
+							if (mark < range.end)
+								pre_filter_pos = std::min(pre_filter_pos + index_granularity, pre_filter.size());
+						}
+					}
+
+					if (!post_filter_pos)
+					{
+						res.clear();
+						continue;
+					}
+
+					progressImpl({ 0, res.bytes() - pre_bytes });
+
+					post_filter.resize(post_filter_pos);
+
+					/// Отфильтруем столбцы, относящиеся к PREWHERE, используя pre_filter,
+					///  остальные столбцы - используя post_filter.
+					size_t rows = 0;
+					for (const auto i : ext::range(0, res.columns()))
+					{
+						auto & col = res.getByPosition(i);
+						if (col.name == prewhere_column && res.columns() > 1)
+							continue;
+						col.column =
+							col.column->filter(task->column_name_set.count(col.name) ? post_filter : pre_filter);
+						rows = col.column->size();
+					}
+
+					/// Заменим столбец со значением условия из PREWHERE на константу.
+					if (!task->remove_prewhere_column)
+						res.getByName(prewhere_column).column = new ColumnConstUInt8{rows, 1};
+				}
+				else
+					throw Exception{
+						"Illegal type " + column->getName() + " of column for filter. Must be ColumnUInt8 or ColumnConstUInt8.",
+						ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER
+					};
+
+				reader->fillMissingColumnsAndReorder(res, task->ordered_names);
+			}
+			while (!task->mark_ranges.empty() && !res && !isCancelled());
+		}
+		else
+		{
+			size_t space_left = std::max(1LU, block_size_marks);
+
+			while (!task->mark_ranges.empty() && space_left)
+			{
+				auto & range = task->mark_ranges.back();
+
+				const size_t marks_to_read = std::min(range.end - range.begin, space_left);
+				reader->readRange(range.begin, range.begin + marks_to_read, res);
+
+				space_left -= marks_to_read;
+				range.begin += marks_to_read;
+				if (range.begin == range.end)
+					task->mark_ranges.pop_back();
+			}
+
+			progressImpl({ res.rowsInFirstColumn(), res.bytes() });
+
+			reader->fillMissingColumns(res, task->ordered_names, task->should_reorder);
+		}
+
+		return res;
+	}
+
+	void injectVirtualColumns(Block & block)
+	{
+		const auto rows = block.rowsInFirstColumn();
+
+		/// add virtual columns
+		if (!virt_column_names.empty())
+		{
+			for (const auto & virt_column_name : virt_column_names)
+			{
+				if (virt_column_name == "_part")
+				{
+					block.insert(ColumnWithTypeAndName{
+						ColumnConst<String>{rows, task->data_part->name}.convertToFullColumn(),
+						new DataTypeString,
+						virt_column_name
+					});
+				}
+				else if (virt_column_name == "_part_index")
+				{
+					block.insert(ColumnWithTypeAndName{
+						ColumnConst<UInt64>{rows, task->part_index_in_query}.convertToFullColumn(),
+						new DataTypeUInt64,
+						virt_column_name
+					});
+				}
+			}
+		}
+	}
+
+	MergeTreeReadPoolPtr pool;
+	const std::size_t block_size_marks;
+	const std::size_t min_marks_to_read;
+	MergeTreeData & storage;
+	const bool use_uncompressed_cache;
+	ExpressionActionsPtr prewhere_actions;
+	const String prewhere_column;
+	const std::size_t min_bytes_to_use_direct_io;
+	const std::size_t max_read_buffer_size;
+	const Names virt_column_names;
+
+	Logger * log;
+
+	using MergeTreeReaderPtr = std::unique_ptr<MergeTreeReader>;
+
+	UncompressedCachePtr owned_uncompressed_cache;
+	MarkCachePtr owned_mark_cache;
+
+	MergeTreeReadTaskPtr task;
+	MergeTreeReaderPtr reader;
+	MergeTreeReaderPtr pre_reader;
+};
+
+
+}

dbms/include/DB/Storages/MergeTree/RangesInDataPart.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <DB/Storages/MergeTree/MergeTreeData.h>
+#include <DB/Storages/MergeTree/MarkRange.h>
+
+
+namespace DB
+{
+
+
+struct RangesInDataPart
+{
+	MergeTreeData::DataPartPtr data_part;
+	std::size_t part_index_in_query;
+	MarkRanges ranges;
+
+	RangesInDataPart() = default;
+
+	RangesInDataPart(const MergeTreeData::DataPartPtr & data_part, const std::size_t part_index_in_query,
+					 const MarkRanges & ranges = MarkRanges{})
+		: data_part{data_part}, part_index_in_query{part_index_in_query}, ranges{ranges}
+	{
+	}
+};
+
+using RangesInDataParts = std::vector<RangesInDataPart>;
+
+
+}

dbms/scripts/gen-bias-data.py

@@ -0,0 +1,262 @@
+#!/usr/bin/python3.4
+# -*- coding: utf-8 -*-
+
+import sys
+import argparse
+import tempfile
+import random
+import subprocess
+import bisect
+from copy import deepcopy
+
+# Псевдослучайный генератор уникальных чисел.
+# http://preshing.com/20121224/how-to-generate-a-sequence-of-unique-random-integers/
+class UniqueRandomGenerator:
+	prime = 4294967291
+
+	def __init__(self, seed_base, seed_offset):
+		self.index = self.permutePQR(self.permutePQR(seed_base) + 0x682f0161)
+		self.intermediate_offset = self.permutePQR(self.permutePQR(seed_offset) + 0x46790905)
+
+	def next(self):
+		val = self.permutePQR((self.permutePQR(self.index) + self.intermediate_offset) ^ 0x5bf03635)
+		self.index = self.index + 1
+		return val
+
+	def permutePQR(self, x):
+		if x >=self.prime:
+			return x
+		else:
+			residue = (x * x) % self.prime
+			if x <= self.prime/2:
+				return residue
+			else:
+				return self.prime - residue
+
+# Создать таблицу содержащую уникальные значения.
+def generate_data_source(host, port, http_port, min_cardinality, max_cardinality, count):
+	chunk_size = round((max_cardinality - min_cardinality) / float(count))
+	used_values = 0
+
+	cur_count = 0
+	next_size = 0
+
+	sup = 32768
+	n1 = random.randrange(0, sup)
+	n2 = random.randrange(0, sup)
+	urng = UniqueRandomGenerator(n1, n2)
+
+	is_first = True
+
+	with tempfile.TemporaryDirectory() as tmp_dir:
+		filename = tmp_dir + '/table.txt'
+		with open(filename, 'w+b') as file_handle:
+			while cur_count < count:
+
+				if is_first == True:
+					is_first = False
+					if min_cardinality != 0:
+						next_size = min_cardinality + 1
+					else:
+						next_size = chunk_size
+				else:
+					next_size += chunk_size
+
+				while used_values < next_size:
+					h = urng.next()
+					used_values = used_values + 1
+					out = str(h) + "\t" + str(cur_count) + "\n";
+					file_handle.write(bytes(out, 'UTF-8'));
+				cur_count = cur_count + 1
+
+		query = "DROP TABLE IF EXISTS data_source"
+		subprocess.check_output(["clickhouse-client", "--host", host, "--port", str(port), "--query", query])
+		query = "CREATE TABLE data_source(UserID UInt64, KeyID UInt64) ENGINE=TinyLog"
+		subprocess.check_output(["clickhouse-client", "--host", host, "--port", str(port), "--query", query])
+
+		cat = subprocess.Popen(("cat", filename), stdout=subprocess.PIPE)
+		subprocess.check_output(("POST", "http://{0}:{1}/?query=INSERT INTO data_source FORMAT TabSeparated".format(host, http_port)), stdin=cat.stdout)
+		cat.wait()
+
+def perform_query(host, port):
+    query  = "SELECT runningAccumulate(uniqExactState(UserID)) AS exact, "
+    query += "runningAccumulate(uniqCombinedRawState(UserID)) AS approx "
+    query += "FROM data_source GROUP BY KeyID"
+    return subprocess.check_output(["clickhouse-client", "--host", host, "--port", port, "--query", query])
+
+def parse_clickhouse_response(response):
+    parsed = []
+    lines = response.decode().split("\n")
+    for cur_line in lines:
+        rows = cur_line.split("\t")
+        if len(rows) == 2:
+            parsed.append([float(rows[0]), float(rows[1])])
+    return parsed
+
+def accumulate_data(accumulated_data, data):
+    if not accumulated_data:
+        accumulated_data = deepcopy(data)
+    else:
+        for row1, row2 in zip(accumulated_data, data):
+            row1[1] += row2[1];
+    return accumulated_data
+
+def generate_raw_result(accumulated_data, count):
+	expected_tab = []
+	bias_tab = []
+	for row in accumulated_data:
+		exact = row[0]
+		expected = row[1] / count
+		bias = expected - exact
+
+		expected_tab.append(expected)
+		bias_tab.append(bias)
+	return [ expected_tab, bias_tab ]
+
+def generate_sample(raw_estimates, biases, n_samples):
+	result = []
+
+	min_card = raw_estimates[0]
+	max_card = raw_estimates[len(raw_estimates) - 1]
+	step = (max_card - min_card) / (n_samples - 1)
+
+	for i in range(0, n_samples + 1):
+		x = min_card + i * step
+		j = bisect.bisect_left(raw_estimates, x)
+
+		if j == len(raw_estimates):
+			result.append((raw_estimates[j - 1], biases[j - 1]))
+		elif raw_estimates[j] == x:
+			result.append((raw_estimates[j], biases[j]))
+		else:
+			# Найти 6 ближайших соседей. Вычислить среднее арифметическое.
+
+			# 6 точек слева x [j-6 j-5 j-4 j-3 j-2 j-1]
+
+			begin = max(j - 6, 0) - 1
+			end = j - 1
+
+			T = []
+			for k in range(end, begin, -1):
+				T.append(x - raw_estimates[k])
+
+			# 6 точек справа x [j j+1 j+2 j+3 j+4 j+5]
+
+			begin = j
+			end = min(j + 5, len(raw_estimates) - 1) + 1
+
+			U = []
+			for k in range(begin, end):
+				U.append(raw_estimates[k] - x)
+
+			# Сливаем расстояния.
+
+			V = []
+
+			lim = min(len(T), len(U))
+			k1 = 0
+			k2 = 0
+
+			while k1 < lim and k2 < lim:
+				if T[k1] == U[k2]:
+					V.append(j - k1 - 1)
+					V.append(j + k2)
+					k1 = k1 + 1
+					k2 = k2 + 1
+				elif T[k1] < U[k2]:
+					V.append(j - k1 - 1)
+					k1 = k1 + 1
+				else:
+					V.append(j + k2)
+					k2 = k2 + 1
+
+			if k1 < len(T):
+				while k1 < len(T):
+					V.append(j - k1 - 1)
+					k1 = k1 + 1
+			elif k2 < len(U):
+				while k2 < len(U):
+					V.append(j + k2)
+					k2 = k2 + 1
+
+			# Выбираем 6 ближайших точек.
+			# Вычисляем средние.
+
+			begin = 0
+			end = min(len(V), 6)
+
+			sum = 0
+			bias = 0
+			for k in range(begin, end):
+				sum += raw_estimates[V[k]]
+				bias += biases[V[k]]
+			sum /= float(end)
+			bias /= float(end)
+
+			result.append((sum, bias))
+
+	# Пропустить последовательные результаты, чьи оценки одинаковые.
+	final_result = []
+	last = -1
+	for entry in result:
+		if entry[0] != last:
+			final_result.append((entry[0], entry[1]))
+			last = entry[0]
+
+	return final_result
+
+def dump_arrays(data):
+
+	print("Size of each array: {0}\n".format(len(data)))
+
+	is_first = True
+	sep = ''
+
+	print("raw_estimates = ")
+	print("{")
+	for row in data:
+		print("\t{0}{1}".format(sep, row[0]))
+		if is_first == True:
+			is_first = False
+			sep = ","
+	print("};")
+
+	is_first = True
+	sep = ""
+
+	print("\nbiases = ")
+	print("{")
+	for row in data:
+		print("\t{0}{1}".format(sep, row[1]))
+		if is_first == True:
+			is_first = False
+			sep = ","
+	print("};")
+
+def start():
+	parser = argparse.ArgumentParser(description = "Generate bias correction tables for HyperLogLog-based functions.")
+	parser.add_argument("-x", "--host", default="127.0.0.1", help="ClickHouse server host name");
+	parser.add_argument("-p", "--port", type=int, default=9000, help="ClickHouse server TCP port");
+	parser.add_argument("-t", "--http_port", type=int, default=8123, help="ClickHouse server HTTP port");
+	parser.add_argument("-i", "--iterations", type=int, default=5000, help="number of iterations");
+	parser.add_argument("-m", "--min_cardinality", type=int, default=16384, help="minimal cardinality");
+	parser.add_argument("-M", "--max_cardinality", type=int, default=655360, help="maximal cardinality");
+	parser.add_argument("-s", "--samples", type=int, default=200, help="number of sampled values");
+	args = parser.parse_args()
+
+	accumulated_data = []
+
+	for i in range(0, args.iterations):
+		print(i + 1)
+		sys.stdout.flush()
+
+		generate_data_source(args.host, str(args.port), str(args.http_port), args.min_cardinality, args.max_cardinality, 1000)
+		response = perform_query(args.host, str(args.port))
+		data = parse_clickhouse_response(response)
+		accumulated_data = accumulate_data(accumulated_data, data)
+
+	result = generate_raw_result(accumulated_data, args.iterations)
+	sampled_data = generate_sample(result[0], result[1], args.samples)
+	dump_arrays(sampled_data)
+
+if __name__ == "__main__": start()

dbms/scripts/linear-counting-threshold.py

@@ -0,0 +1,150 @@
+#!/usr/bin/python3.4
+# -*- coding: utf-8 -*-
+
+import sys
+import argparse
+import tempfile
+import random
+import subprocess
+import bisect
+from copy import deepcopy
+
+# Псевдослучайный генератор уникальных чисел.
+# http://preshing.com/20121224/how-to-generate-a-sequence-of-unique-random-integers/
+class UniqueRandomGenerator:
+	prime = 4294967291
+
+	def __init__(self, seed_base, seed_offset):
+		self.index = self.permutePQR(self.permutePQR(seed_base) + 0x682f0161)
+		self.intermediate_offset = self.permutePQR(self.permutePQR(seed_offset) + 0x46790905)
+
+	def next(self):
+		val = self.permutePQR((self.permutePQR(self.index) + self.intermediate_offset) ^ 0x5bf03635)
+		self.index = self.index + 1
+		return val
+
+	def permutePQR(self, x):
+		if x >=self.prime:
+			return x
+		else:
+			residue = (x * x) % self.prime
+			if x <= self.prime/2:
+				return residue
+			else:
+				return self.prime - residue
+
+# Создать таблицу содержащую уникальные значения.
+def generate_data_source(host, port, http_port, min_cardinality, max_cardinality, count):
+	chunk_size = round((max_cardinality - (min_cardinality + 1)) / float(count))
+	used_values = 0
+
+	cur_count = 0
+	next_size = 0
+
+	sup = 32768
+	n1 = random.randrange(0, sup)
+	n2 = random.randrange(0, sup)
+	urng = UniqueRandomGenerator(n1, n2)
+
+	is_first = True
+
+	with tempfile.TemporaryDirectory() as tmp_dir:
+		filename = tmp_dir + '/table.txt'
+		with open(filename, 'w+b') as file_handle:
+			while cur_count < count:
+
+				if is_first == True:
+					is_first = False
+					if min_cardinality != 0:
+						next_size = min_cardinality + 1
+					else:
+						next_size = chunk_size
+				else:
+					next_size += chunk_size
+
+				while used_values < next_size:
+					h = urng.next()
+					used_values = used_values + 1
+					out = str(h) + "\t" + str(cur_count) + "\n";
+					file_handle.write(bytes(out, 'UTF-8'));
+				cur_count = cur_count + 1
+
+		query = "DROP TABLE IF EXISTS data_source"
+		subprocess.check_output(["clickhouse-client", "--host", host, "--port", str(port), "--query", query])
+		query = "CREATE TABLE data_source(UserID UInt64, KeyID UInt64) ENGINE=TinyLog"
+		subprocess.check_output(["clickhouse-client", "--host", host, "--port", str(port), "--query", query])
+
+		cat = subprocess.Popen(("cat", filename), stdout=subprocess.PIPE)
+		subprocess.check_output(("POST", "http://{0}:{1}/?query=INSERT INTO data_source FORMAT TabSeparated".format(host, http_port)), stdin=cat.stdout)
+		cat.wait()
+
+def perform_query(host, port):
+    query  = "SELECT runningAccumulate(uniqExactState(UserID)) AS exact, "
+    query += "runningAccumulate(uniqCombinedRawState(UserID)) AS raw, "
+    query += "runningAccumulate(uniqCombinedLinearCountingState(UserID)) AS linear_counting, "
+    query += "runningAccumulate(uniqCombinedBiasCorrectedState(UserID)) AS bias_corrected "
+    query += "FROM data_source GROUP BY KeyID"
+    return subprocess.check_output(["clickhouse-client", "--host", host, "--port", port, "--query", query])
+
+def parse_clickhouse_response(response):
+    parsed = []
+    lines = response.decode().split("\n")
+    for cur_line in lines:
+        rows = cur_line.split("\t")
+        if len(rows) == 4:
+            parsed.append([float(rows[0]), float(rows[1]), float(rows[2]), float(rows[3])])
+    return parsed
+
+def accumulate_data(accumulated_data, data):
+    if not accumulated_data:
+        accumulated_data = deepcopy(data)
+    else:
+        for row1, row2 in zip(accumulated_data, data):
+            row1[1] += row2[1];
+            row1[2] += row2[2];
+            row1[3] += row2[3];
+    return accumulated_data
+
+def dump_graphs(data, count):
+	with open("raw_graph.txt", "w+b") as fh1, open("linear_counting_graph.txt", "w+b") as fh2, open("bias_corrected_graph.txt", "w+b") as fh3:
+		expected_tab = []
+		bias_tab = []
+		for row in data:
+			exact = row[0]
+			raw = row[1] / count;
+			linear_counting = row[2] / count;
+			bias_corrected = row[3] / count;
+
+			outstr = "{0}\t{1}\n".format(exact, abs(raw - exact) / exact)
+			fh1.write(bytes(outstr, 'UTF-8'))
+
+			outstr = "{0}\t{1}\n".format(exact, abs(linear_counting - exact) / exact)
+			fh2.write(bytes(outstr, 'UTF-8'))
+
+			outstr = "{0}\t{1}\n".format(exact, abs(bias_corrected - exact) / exact)
+			fh3.write(bytes(outstr, 'UTF-8'))
+
+def start():
+	parser = argparse.ArgumentParser(description = "Generate graphs that help to determine the linear counting threshold.")
+	parser.add_argument("-x", "--host", default="127.0.0.1", help="clickhouse host name");
+	parser.add_argument("-p", "--port", type=int, default=9000, help="clickhouse client TCP port");
+	parser.add_argument("-t", "--http_port", type=int, default=8123, help="clickhouse HTTP port");
+	parser.add_argument("-i", "--iterations", type=int, default=5000, help="number of iterations");
+	parser.add_argument("-m", "--min_cardinality", type=int, default=16384, help="minimal cardinality");
+	parser.add_argument("-M", "--max_cardinality", type=int, default=655360, help="maximal cardinality");
+	args = parser.parse_args()
+
+	accumulated_data = []
+
+	for i in range(0, args.iterations):
+		print(i + 1)
+		sys.stdout.flush()
+
+		generate_data_source(args.host, str(args.port), str(args.http_port), args.min_cardinality, args.max_cardinality, 1000)
+		response = perform_query(args.host, str(args.port))
+		data = parse_clickhouse_response(response)
+		accumulated_data = accumulate_data(accumulated_data, data)
+
+	dump_graphs(accumulated_data, args.iterations)
+
+if __name__ == "__main__": start()

dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp

@@ -351,6 +351,72 @@ AggregateFunctionPtr AggregateFunctionFactory::get(const String & name, const Da
 		else
 			throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
 	}
+	else if (name == "uniqCombinedRaw")
+	{
+		if (argument_types.size() != 1)
+			throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+		const IDataType & argument_type = *argument_types[0];
+
+		AggregateFunctionPtr res = createWithNumericType<AggregateFunctionUniq, AggregateFunctionUniqCombinedRawData>(*argument_types[0]);
+
+		if (res)
+			return res;
+		else if (typeid_cast<const DataTypeDate 	*>(&argument_type))
+			return new AggregateFunctionUniq<DataTypeDate::FieldType, AggregateFunctionUniqCombinedRawData<DataTypeDate::FieldType>>;
+		else if (typeid_cast<const DataTypeDateTime*>(&argument_type))
+			return new AggregateFunctionUniq<DataTypeDateTime::FieldType, AggregateFunctionUniqCombinedRawData<DataTypeDateTime::FieldType>>;
+		else if (typeid_cast<const DataTypeString*>(&argument_type) || typeid_cast<const DataTypeFixedString*>(&argument_type))
+			return new AggregateFunctionUniq<String, AggregateFunctionUniqCombinedRawData<String>>;
+		else
+			throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+	}
+	else if (name == "uniqCombinedLinearCounting")
+	{
+		if (argument_types.size() != 1)
+			throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+		const IDataType & argument_type = *argument_types[0];
+
+		AggregateFunctionPtr res = createWithNumericType<AggregateFunctionUniq,
+				AggregateFunctionUniqCombinedLinearCountingData>(*argument_types[0]);
+
+		if (res)
+			return res;
+		else if (typeid_cast<const DataTypeDate 	*>(&argument_type))
+			return new AggregateFunctionUniq<DataTypeDate::FieldType,
+				AggregateFunctionUniqCombinedLinearCountingData<DataTypeDate::FieldType>>;
+		else if (typeid_cast<const DataTypeDateTime*>(&argument_type))
+			return new AggregateFunctionUniq<DataTypeDateTime::FieldType,
+				AggregateFunctionUniqCombinedLinearCountingData<DataTypeDateTime::FieldType>>;
+		else if (typeid_cast<const DataTypeString*>(&argument_type) || typeid_cast<const DataTypeFixedString*>(&argument_type))
+			return new AggregateFunctionUniq<String, AggregateFunctionUniqCombinedLinearCountingData<String>>;
+		else
+			throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+	}
+	else if (name == "uniqCombinedBiasCorrected")
+	{
+		if (argument_types.size() != 1)
+			throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+		const IDataType & argument_type = *argument_types[0];
+
+		AggregateFunctionPtr res = createWithNumericType<AggregateFunctionUniq,
+			AggregateFunctionUniqCombinedBiasCorrectedData>(*argument_types[0]);
+
+		if (res)
+			return res;
+		else if (typeid_cast<const DataTypeDate 	*>(&argument_type))
+			return new AggregateFunctionUniq<DataTypeDate::FieldType,
+				AggregateFunctionUniqCombinedBiasCorrectedData<DataTypeDate::FieldType>>;
+		else if (typeid_cast<const DataTypeDateTime*>(&argument_type))
+			return new AggregateFunctionUniq<DataTypeDateTime::FieldType,
+				AggregateFunctionUniqCombinedBiasCorrectedData<DataTypeDateTime::FieldType>>;
+		else if (typeid_cast<const DataTypeString*>(&argument_type) || typeid_cast<const DataTypeFixedString*>(&argument_type))
+			return new AggregateFunctionUniq<String, AggregateFunctionUniqCombinedBiasCorrectedData<String>>;
+		else
+			throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+	}
 	else if (name == "uniqCombined")
 	{
 		if (argument_types.size() != 1)
@@ -562,6 +628,13 @@ AggregateFunctionPtr AggregateFunctionFactory::get(const String & name, const Da
 
 		return new AggregateFunctionSequenceMatch;
 	}
+	else if (name == "sequenceCount")
+	{
+		if (!AggregateFunctionSequenceCount::sufficientArgs(argument_types.size()))
+			throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+		return new AggregateFunctionSequenceCount;
+	}
 	else if (name == "varSamp")
 	{
 		if (argument_types.size() != 1)
@@ -726,6 +799,9 @@ const AggregateFunctionFactory::FunctionNames & AggregateFunctionFactory::getFun
 		"uniq",
 		"uniqHLL12",
 		"uniqExact",
+		"uniqCombinedRaw",
+		"uniqCombinedLinearCounting",
+		"uniqCombinedBiasCorrected",
 		"uniqCombined",
 		"uniqUpTo",
 		"groupArray",
@@ -743,6 +819,7 @@ const AggregateFunctionFactory::FunctionNames & AggregateFunctionFactory::getFun
 		"quantileDeterministic",
 		"quantilesDeterministic",
 		"sequenceMatch",
+		"sequenceCount",
 		"varSamp",
 		"varPop",
 		"stddevSamp",

dbms/src/AggregateFunctions/UniqCombinedBiasData.cpp

@@ -0,0 +1,390 @@
+#include <DB/AggregateFunctions/UniqCombinedBiasData.h>
+
+namespace DB
+{
+
+namespace
+{
+
+const UniqCombinedBiasData::InterpolatedData raw_estimates =
+{
+	700.0
+	,3850.0
+	,7350.0
+	,10850.0
+	,14350.0
+	,89003.5714
+	,103764.30343333333
+	,105572.1915
+	,109252.46533333334
+	,112638.20573333332
+	,116094.29566666669
+	,119619.81926666666
+	,123214.92233333334
+	,126469.06656666666
+	,130196.15093333334
+	,133566.85673333335
+	,136991.63890000002
+	,140470.0118666667
+	,144000.91686666667
+	,147585.44463333333
+	,151222.7466
+	,154447.75893333333
+	,158181.68399999998
+	,161492.41386666667
+	,164840.6352
+	,168713.9904
+	,172143.82656666666
+	,175611.2078
+	,179116.94873333335
+	,182658.0355
+	,186236.36723333332
+	,189332.1009
+	,192976.1847
+	,196654.62706666664
+	,199835.39103333335
+	,203575.92429999998
+	,206808.87086666666
+	,210611.72886666664
+	,213896.25913333334
+	,217759.63066666664
+	,221096.10933333333
+	,224456.31466666667
+	,227839.0366333333
+	,231242.72576666667
+	,235239.98256666667
+	,238688.95070000002
+	,242158.17593333332
+	,245649.42926666664
+	,249158.9859666667
+	,252689.67179999998
+	,256241.95376666667
+	,259214.9391666667
+	,262798.3925666667
+	,266399.8345666667
+	,270018.35863333335
+	,273653.1149
+	,276696.7119
+	,280366.51476666663
+	,284051.95540000004
+	,287133.5254333333
+	,290847.31173333334
+	,294579.5226
+	,297698.64109999995
+	,301454.39253333333
+	,305223.59123333334
+	,308375.3184666667
+	,312170.06
+	,315342.02996666665
+	,319162.8188666667
+	,322356.3565666666
+	,326199.5866
+	,329412.83396666666
+	,332634.3235666667
+	,336510.7596333333
+	,339747.7330333333
+	,343643.0385666667
+	,346896.77420000004
+	,350157.6729666667
+	,354079.3932333334
+	,357354.5196333334
+	,360638.3034333333
+	,364588.47873333335
+	,367886.05706666666
+	,371189.98006666667
+	,375161.95876666665
+	,378478.6737666666
+	,381801.6619
+	,385130.9645
+	,389131.7460333333
+	,392471.6233333333
+	,395817.1175
+	,399165.1003333333
+	,402518.7819333333
+	,406549.7624333333
+	,409916.016
+	,413289.0218666666
+	,416661.9977333333
+	,420040.4257333334
+	,424099.3186333333
+	,427485.4292000001
+	,430876.4814666666
+	,434269.4718
+	,437665.82826666674
+	,441066.7185
+	,444469.97226666665
+	,448561.9376666667
+	,451974.73750000005
+	,455389.1112
+	,458808.5816666667
+	,462230.8184666667
+	,465656.9889
+	,469081.3269
+	,472512.4878
+	,475944.4204333333
+	,480065.7132666667
+	,483502.04110000003
+	,486939.5075666667
+	,490379.7868333334
+	,493818.5365333333
+	,497259.08013333334
+	,500705.3513
+	,504155.6234666666
+	,507606.65499999997
+	,511060.7448666667
+	,514517.4004
+	,517973.35829999996
+	,521431.3761666666
+	,524891.7097333333
+	,529044.7593
+	,532507.0878999999
+	,535971.5070333333
+	,539436.2416999999
+	,542903.1470333333
+	,546370.3423
+	,549837.6947999999
+	,553307.0003666667
+	,556775.3770333333
+	,560247.6308333334
+	,563721.0700333334
+	,567196.7586333333
+	,570669.8439666666
+	,574146.018
+	,577622.2794666667
+	,581098.3862333334
+	,584575.8826666666
+	,588055.1468000001
+	,591538.0234
+	,595018.0103000001
+	,598504.5469333333
+	,601992.5697666666
+	,605475.5452
+	,608959.4645
+	,612444.0261
+	,615929.6436
+	,619412.3877333334
+	,622903.4263999999
+	,626391.3657333333
+	,629876.7359333333
+	,633364.2825999999
+	,636855.2673666667
+	,640344.4321000001
+	,643836.5543666667
+	,647327.3073999999
+	,650818.3525666667
+	,654312.2421666667
+	,657807.0899666668
+	,661301.4443666666
+	,664794.1040333334
+	,668288.1969666666
+	,671781.0196666667
+	,675272.7522333333
+	,678766.9045999999
+	,682259.3583666667
+	,685747.8148333334
+	,689238.7994666666
+	,692732.0478333334
+	,696224.6407
+	,700069.9224
+};
+
+const UniqCombinedBiasData::InterpolatedData biases =
+{
+	0.0
+	,0.0
+	,0.0
+	,0.0
+	,0.0
+	,71153.5714
+	,85214.30343333333
+	,83522.1915
+	,80202.46533333334
+	,77288.20573333332
+	,74444.29566666667
+	,71669.81926666667
+	,68964.92233333334
+	,66619.06656666666
+	,64046.15093333333
+	,61816.85673333333
+	,59641.6389
+	,57520.01186666667
+	,55450.91686666667
+	,53435.44463333334
+	,51472.74659999999
+	,49797.75893333333
+	,47931.68399999999
+	,46342.41386666667
+	,44790.6352
+	,43063.9904
+	,41593.82656666667
+	,40161.2078
+	,38766.94873333333
+	,37408.035500000005
+	,36086.36723333333
+	,34982.1009
+	,33726.184700000005
+	,32504.627066666664
+	,31485.391033333333
+	,30325.924299999995
+	,29358.870866666668
+	,28261.72886666667
+	,27346.259133333337
+	,26309.630666666668
+	,25446.109333333337
+	,24606.31466666666
+	,23789.036633333333
+	,22992.725766666666
+	,22089.98256666667
+	,21338.9507
+	,20608.175933333332
+	,19899.429266666673
+	,19208.985966666663
+	,18539.6718
+	,17891.95376666667
+	,17364.939166666667
+	,16748.392566666666
+	,16149.834566666666
+	,15568.358633333331
+	,15003.114899999995
+	,14546.711900000004
+	,14016.51476666668
+	,13501.955399999997
+	,13083.52543333332
+	,12597.311733333336
+	,12129.522600000006
+	,11748.641100000008
+	,11304.392533333332
+	,10873.59123333334
+	,10525.318466666678
+	,10120.059999999998
+	,9792.029966666674
+	,9412.818866666668
+	,9106.356566666664
+	,8749.58660000001
+	,8462.833966666678
+	,8184.323566666659
+	,7860.759633333325
+	,7597.733033333323
+	,7293.038566666665
+	,7046.774200000004
+	,6807.672966666675
+	,6529.393233333336
+	,6304.519633333344
+	,6088.30343333332
+	,5838.4787333333325
+	,5636.057066666661
+	,5439.980066666671
+	,5211.958766666658
+	,5028.673766666664
+	,4851.661899999996
+	,4680.964499999992
+	,4481.746033333319
+	,4321.623333333322
+	,4167.117500000012
+	,4015.1003333333356
+	,3868.781933333337
+	,3699.762433333332
+	,3566.0159999999937
+	,3439.021866666648
+	,3311.9977333333422
+	,3190.4257333333276
+	,3049.3186333333238
+	,2935.4291999999937
+	,2826.4814666666593
+	,2719.4717999999993
+	,2615.8282666666782
+	,2516.7184999999977
+	,2419.972266666669
+	,2311.9376666666744
+	,2224.7374999999884
+	,2139.1111999999944
+	,2058.581666666665
+	,1980.8184666666687
+	,1906.9888999999966
+	,1831.3268999999952
+	,1762.4878000000026
+	,1694.420433333328
+	,1615.7132666666682
+	,1552.0410999999924
+	,1489.507566666677
+	,1429.7868333333365
+	,1368.536533333332
+	,1309.0801333333268
+	,1255.35129999999
+	,1205.6234666666617
+	,1156.6549999999988
+	,1110.744866666675
+	,1067.4004000000034
+	,1023.3583000000023
+	,981.3761666666638
+	,941.7097333333513
+	,894.7593000000148
+	,857.0879000000035
+	,821.5070333333375
+	,786.2416999999745
+	,753.1470333333127
+	,720.3422999999797
+	,687.6947999999975
+	,657.0003666666647
+	,625.3770333333329
+	,597.6308333333387
+	,571.0700333333225
+	,546.7586333333165
+	,519.8439666666478
+	,496.0180000000012
+	,472.2794666666693
+	,448.386233333343
+	,425.8826666666816
+	,405.1468000000071
+	,388.0233999999861
+	,368.01030000002356
+	,354.54693333333125
+	,342.5697666666626
+	,325.5452000000126
+	,309.4644999999825
+	,294.0261000000173
+	,279.64360000001034
+	,262.38773333333666
+	,253.42639999999665
+	,241.36573333333945
+	,226.7359333333443
+	,214.28259999999622
+	,205.26736666667662
+	,194.43210000001514
+	,186.55436666666841
+	,177.30740000001
+	,168.35256666666828
+	,162.24216666668266
+	,157.0899666666713
+	,151.44436666666297
+	,144.1040333333464
+	,138.19696666668946
+	,131.01966666666945
+	,122.7522333333424
+	,116.90459999998954
+	,109.35836666667213
+	,97.81483333332774
+	,88.7994666666491
+	,82.04783333333519
+	,74.64070000000841
+	,69.92240000003949
+};
+
+}
+
+double UniqCombinedBiasData::getThreshold()
+{
+	return 176000;
+}
+
+const UniqCombinedBiasData::InterpolatedData & UniqCombinedBiasData::getRawEstimates()
+{
+	return raw_estimates;
+}
+
+const UniqCombinedBiasData::InterpolatedData & UniqCombinedBiasData::getBiases()
+{
+	return biases;
+}
+
+}

dbms/src/DataStreams/LimitBlockInputStream.cpp

@@ -8,8 +8,8 @@ namespace DB
 
 using Poco::SharedPtr;
 
-LimitBlockInputStream::LimitBlockInputStream(BlockInputStreamPtr input_, size_t limit_, size_t offset_)
-	: limit(limit_), offset(offset_), pos(0)
+LimitBlockInputStream::LimitBlockInputStream(BlockInputStreamPtr input_, size_t limit_, size_t offset_, bool always_read_till_end_)
+	: limit(limit_), offset(offset_), always_read_till_end(always_read_till_end_)
 {
 	children.push_back(input_);
 }
@@ -23,7 +23,16 @@ Block LimitBlockInputStream::readImpl()
 	/// pos - сколько строк было прочитано, включая последний прочитанный блок
 
 	if (pos >= offset + limit)
-		return res;
+	{
+		if (!always_read_till_end)
+			return res;
+		else
+		{
+			while (children.back()->read())
+				;
+			return res;
+		}
+	}
 
 	do
 	{

dbms/src/DataStreams/tests/filter_stream_hitlog.cpp

@@ -132,7 +132,7 @@ int main(int argc, char ** argv)
 		Poco::SharedPtr<IBlockInputStream> in = table->read(column_names, 0, context, Settings(), stage)[0];
 		in = new ExpressionBlockInputStream(in, expression);
 		in = new FilterBlockInputStream(in, 4);
-		//in = new LimitBlockInputStream(in, 10);
+		//in = new LimitBlockInputStream(in, 10, 0);
 
 		WriteBufferFromOStream ob(std::cout);
 		RowOutputStreamPtr out_ = new TabSeparatedRowOutputStream(ob, expression->getSampleBlock());

dbms/src/DataStreams/tests/sorting_stream.cpp

@@ -152,7 +152,7 @@ int main(int argc, char ** argv)
 		Poco::SharedPtr<IBlockInputStream> in = table->read(column_names, 0, Context{}, Settings(), stage, argc == 2 ? atoi(argv[1]) : 1048576)[0];
 		in = new PartialSortingBlockInputStream(in, sort_columns);
 		in = new MergeSortingBlockInputStream(in, sort_columns, DEFAULT_BLOCK_SIZE, 0, 0, "");
-		//in = new LimitBlockInputStream(in, 10);
+		//in = new LimitBlockInputStream(in, 10, 0);
 
 		WriteBufferFromOStream ob(std::cout);
 		RowOutputStreamPtr out_ = new TabSeparatedRowOutputStream(ob, sample);

dbms/src/DataStreams/tests/union_stream2.cpp

@@ -45,7 +45,7 @@ int main(int argc, char ** argv)
 			streams[i] = new DB::AsynchronousBlockInputStream(streams[i]);
 		
 		DB::BlockInputStreamPtr stream = new DB::UnionBlockInputStream(streams, nullptr, settings.max_threads);
-		stream = new DB::LimitBlockInputStream(stream, 10);
+		stream = new DB::LimitBlockInputStream(stream, 10, 0);
 
 		DB::FormatFactory format_factory;
 		DB::WriteBufferFromFileDescriptor wb(STDERR_FILENO);

dbms/src/Functions/FunctionsCoding.cpp

@@ -8,14 +8,18 @@ void registerFunctionsCoding(FunctionFactory & factory)
 {
 	factory.registerFunction<FunctionToStringCutToZero>();
 	factory.registerFunction<FunctionIPv6NumToString>();
+	factory.registerFunction<FunctionCutIPv6>();
 	factory.registerFunction<FunctionIPv6StringToNum>();
 	factory.registerFunction<FunctionIPv4NumToString>();
 	factory.registerFunction<FunctionIPv4StringToNum>();
 	factory.registerFunction<FunctionIPv4NumToStringClassC>();
+	factory.registerFunction<FunctionIPv4ToIPv6>();
 	factory.registerFunction<FunctionHex>();
 	factory.registerFunction<FunctionUnhex>();
 	factory.registerFunction<FunctionBitmaskToArray>();
 	factory.registerFunction<FunctionBitTest>();
+	factory.registerFunction<FunctionBitTestAny>();
+	factory.registerFunction<FunctionBitTestAll>();
 }
 
 }

dbms/src/IO/ReadBufferAIO.cpp

@@ -11,11 +11,10 @@ namespace DB
 
 /// Примечание: выделяется дополнительная страница, которая содежрит те данные, которые
 /// не влезают в основной буфер.
-ReadBufferAIO::ReadBufferAIO(const std::string & filename_, size_t buffer_size_, int flags_,
-	char * existing_memory_)
+ReadBufferAIO::ReadBufferAIO(const std::string & filename_, size_t buffer_size_, int flags_, char * existing_memory_)
 	: ReadBufferFromFileBase(buffer_size_ + DEFAULT_AIO_FILE_BLOCK_SIZE, existing_memory_, DEFAULT_AIO_FILE_BLOCK_SIZE),
-	fill_buffer(BufferWithOwnMemory<ReadBuffer>(this->memory.size(), nullptr, DEFAULT_AIO_FILE_BLOCK_SIZE)),
-	filename(filename_)
+	  fill_buffer(BufferWithOwnMemory<ReadBuffer>(internalBuffer().size(), nullptr, DEFAULT_AIO_FILE_BLOCK_SIZE)),
+	  filename(filename_)
 {
 	ProfileEvents::increment(ProfileEvents::FileOpen);
 

dbms/src/Interpreters/ExpressionAnalyzer.cpp

@@ -886,8 +886,45 @@ static SharedPtr<InterpreterSelectQuery> interpretSubquery(
 		select_query->children.emplace_back(select_query->table);
 	}
 	else
+	{
 		query = subquery->children.at(0);
 
+		/** В подзапросе могут быть указаны столбцы с одинаковыми именами. Например, SELECT x, x FROM t
+		  * Это плохо, потому что результат такого запроса нельзя сохранить в таблицу, потому что в таблице не может быть одноимённых столбцов.
+		  * Сохранение в таблицу требуется для GLOBAL-подзапросов.
+		  *
+		  * Чтобы избежать такой ситуации, будем переименовывать одинаковые столбцы.
+		  */
+
+		std::set<std::string> all_column_names;
+		std::set<std::string> assigned_column_names;
+
+		if (ASTSelectQuery * select = typeid_cast<ASTSelectQuery *>(query.get()))
+		{
+			for (auto & expr : select->select_expression_list->children)
+				all_column_names.insert(expr->getAliasOrColumnName());
+
+			for (auto & expr : select->select_expression_list->children)
+			{
+				auto name = expr->getAliasOrColumnName();
+
+				if (!assigned_column_names.insert(name).second)
+				{
+					size_t i = 1;
+					while (all_column_names.end() != all_column_names.find(name + "_" + toString(i)))
+						++i;
+
+					name = name + "_" + toString(i);
+					expr = expr->clone();	/// Отменяет склейку одинаковых выражений в дереве.
+					expr->setAlias(name);
+
+					all_column_names.insert(name);
+					assigned_column_names.insert(name);
+				}
+			}
+		}
+	}
+
 	if (required_columns.empty())
 		return new InterpreterSelectQuery(query, subquery_context, QueryProcessingStage::Complete, subquery_depth + 1);
 	else
@@ -1743,8 +1780,6 @@ bool ExpressionAnalyzer::appendJoin(ExpressionActionsChain & chain, bool only_ty
 
 	if (!subquery_for_set.join)
 	{
-		Names join_key_names_left(join_key_names_left_set.begin(), join_key_names_left_set.end());
-		Names join_key_names_right(join_key_names_right_set.begin(), join_key_names_right_set.end());
 		JoinPtr join = new Join(join_key_names_left, join_key_names_right, settings.limits, ast_join.kind, ast_join.strictness);
 
 		Names required_joined_columns(join_key_names_right.begin(), join_key_names_right.end());
@@ -2137,27 +2172,31 @@ void ExpressionAnalyzer::collectJoinedColumns(NameSet & joined_columns, NamesAnd
 		auto & keys = typeid_cast<ASTExpressionList &>(*node.using_expr_list);
 		for (const auto & key : keys.children)
 		{
-			if (!join_key_names_left_set.insert(key->getColumnName()).second)
-				throw Exception("Duplicate column in USING list", ErrorCodes::DUPLICATE_COLUMN);
+			if (join_key_names_left.end() == std::find(join_key_names_left.begin(), join_key_names_left.end(), key->getColumnName()))
+				join_key_names_left.push_back(key->getColumnName());
+			else
+				throw Exception("Duplicate column " + key->getColumnName() + " in USING list", ErrorCodes::DUPLICATE_COLUMN);
 
-			if (!join_key_names_right_set.insert(key->getAliasOrColumnName()).second)
-				throw Exception("Duplicate column in USING list", ErrorCodes::DUPLICATE_COLUMN);
+			if (join_key_names_right.end() == std::find(join_key_names_right.begin(), join_key_names_right.end(), key->getAliasOrColumnName()))
+				join_key_names_right.push_back(key->getAliasOrColumnName());
+			else
+				throw Exception("Duplicate column " + key->getAliasOrColumnName() + " in USING list", ErrorCodes::DUPLICATE_COLUMN);
 		}
 	}
 
 	for (const auto i : ext::range(0, nested_result_sample.columns()))
 	{
 		const auto & col = nested_result_sample.getByPosition(i);
-		if (!join_key_names_right_set.count(col.name))
+		if (join_key_names_right.end() == std::find(join_key_names_right.begin(), join_key_names_right.end(), col.name))
 		{
 			joined_columns.insert(col.name);
 			joined_columns_name_type.emplace_back(col.name, col.type);
 		}
 	}
 
-/*	for (const auto & name : join_key_names_left_set)
+/*	for (const auto & name : join_key_names_left)
 		std::cerr << "JOIN key (left): " << name << std::endl;
-	for (const auto & name : join_key_names_right_set)
+	for (const auto & name : join_key_names_right)
 		std::cerr << "JOIN key (right): " << name << std::endl;
 	std::cerr << std::endl;
 	for (const auto & name : joined_columns)

dbms/src/Interpreters/InterpreterSelectQuery.cpp

@@ -98,7 +98,7 @@ void InterpreterSelectQuery::init(BlockInputStreamPtr input, const Names & requi
 
 void InterpreterSelectQuery::basicInit(BlockInputStreamPtr input_)
 {
-	if (query.table && typeid_cast<ASTSelectQuery *>(&*query.table))
+	if (query.table && typeid_cast<ASTSelectQuery *>(query.table.get()))
 	{
 		if (table_column_names.empty())
 		{
@@ -107,10 +107,10 @@ void InterpreterSelectQuery::basicInit(BlockInputStreamPtr input_)
 	}
 	else
 	{
-		if (query.table && typeid_cast<const ASTFunction *>(&*query.table))
+		if (query.table && typeid_cast<const ASTFunction *>(query.table.get()))
 		{
 			/// Получить табличную функцию
-			TableFunctionPtr table_function_ptr = context.getTableFunctionFactory().get(typeid_cast<const ASTFunction *>(&*query.table)->name, context);
+			TableFunctionPtr table_function_ptr = context.getTableFunctionFactory().get(typeid_cast<const ASTFunction *>(query.table.get())->name, context);
 			/// Выполнить ее и запомнить результат
 			storage = table_function_ptr->execute(query.table, context);
 		}
@@ -329,7 +329,7 @@ BlockIO InterpreterSelectQuery::execute()
 	executeUnion();
 
 	/// Ограничения на результат, квота на результат, а также колбек для прогресса.
-	if (IProfilingBlockInputStream * stream = dynamic_cast<IProfilingBlockInputStream *>(&*streams[0]))
+	if (IProfilingBlockInputStream * stream = dynamic_cast<IProfilingBlockInputStream *>(streams[0].get()))
 	{
 		/// Ограничения действуют только на конечный результат.
 		if (to_stage == QueryProcessingStage::Complete)
@@ -590,7 +590,7 @@ void InterpreterSelectQuery::executeSingleQuery()
 			{
 				transformStreams([&](auto & stream)
 				{
-					if (IProfilingBlockInputStream * p_stream = dynamic_cast<IProfilingBlockInputStream *>(&*stream))
+					if (IProfilingBlockInputStream * p_stream = dynamic_cast<IProfilingBlockInputStream *>(stream.get()))
 						p_stream->enableExtremes();
 				});
 			}
@@ -651,7 +651,7 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns()
 	/// Список столбцов, которых нужно прочитать, чтобы выполнить запрос.
 	Names required_columns = query_analyzer->getRequiredColumns();
 
-	if (query.table && typeid_cast<ASTSelectQuery *>(&*query.table))
+	if (query.table && typeid_cast<ASTSelectQuery *>(query.table.get()))
 	{
 		/** Для подзапроса не действуют ограничения на максимальный размер результата.
 		 *  Так как результат поздапроса - ещё не результат всего запроса.
@@ -792,7 +792,7 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns()
 
 		transformStreams([&](auto & stream)
 		{
-			if (IProfilingBlockInputStream * p_stream = dynamic_cast<IProfilingBlockInputStream *>(&*stream))
+			if (IProfilingBlockInputStream * p_stream = dynamic_cast<IProfilingBlockInputStream *>(stream.get()))
 			{
 				p_stream->setLimits(limits);
 				p_stream->setQuota(quota);
@@ -1091,9 +1091,38 @@ void InterpreterSelectQuery::executeLimit()
 	/// Если есть LIMIT
 	if (query.limit_length)
 	{
+		/** Редкий случай:
+		  *  если нет WITH TOTALS и есть подзапрос в FROM, и там на одном из уровней есть WITH TOTALS,
+		  *  то при использовании LIMIT-а следует читать данные до конца, а не отменять выполнение запроса раньше,
+		  *  потому что при отмене выполнения запроса, мы не получим данные для totals с удалённого сервера.
+		  */
+		bool always_read_till_end = false;
+		if (!query.group_by_with_totals && query.table && typeid_cast<const ASTSelectQuery *>(query.table.get()))
+		{
+			const ASTSelectQuery * subquery = static_cast<const ASTSelectQuery *>(query.table.get());
+
+			while (subquery->table)
+			{
+				if (subquery->group_by_with_totals)
+				{
+					/** NOTE Можно ещё проверять, что таблица в подзапросе - распределённая, и что она смотрит только на один шард.
+					  * В остальных случаях totals будет вычислен на сервере-инициаторе запроса, и читать данные до конца не обязательно.
+					  */
+
+					always_read_till_end = true;
+					break;
+				}
+
+				if (typeid_cast<const ASTSelectQuery *>(subquery->table.get()))
+					subquery = static_cast<const ASTSelectQuery *>(subquery->table.get());
+				else
+					break;
+			}
+		}
+
 		transformStreams([&](auto & stream)
 		{
-			stream = new LimitBlockInputStream(stream, limit_length, limit_offset);
+			stream = new LimitBlockInputStream(stream, limit_length, limit_offset, always_read_till_end);
 		});
 	}
 }

dbms/src/Interpreters/Join.cpp

@@ -376,12 +376,18 @@ void Join::setSampleBlock(const Block & block)
 
 	sample_block_with_columns_to_add = block;
 
-	/// Удаляем из sample_block_with_columns_to_add ключевые столбцы.
-	for (const auto & name : key_names_right)
+	/// Переносим из sample_block_with_columns_to_add ключевые столбцы в sample_block_with_keys, сохраняя порядок.
+	size_t pos = 0;
+	while (pos < sample_block_with_columns_to_add.columns())
 	{
-		size_t pos = sample_block_with_columns_to_add.getPositionByName(name);
-		sample_block_with_keys.insert(sample_block_with_columns_to_add.unsafeGetByPosition(pos));
-		sample_block_with_columns_to_add.erase(pos);
+		const auto & name = sample_block_with_columns_to_add.unsafeGetByPosition(pos).name;
+		if (key_names_right.end() != std::find(key_names_right.begin(), key_names_right.end(), name))
+		{
+			sample_block_with_keys.insert(sample_block_with_columns_to_add.unsafeGetByPosition(pos));
+			sample_block_with_columns_to_add.erase(pos);
+		}
+		else
+			++pos;
 	}
 
 	for (size_t i = 0, size = sample_block_with_columns_to_add.columns(); i < size; ++i)
@@ -426,7 +432,9 @@ bool Join::insertFromBlock(const Block & block)
 
 	if (getFullness(kind))
 	{
-		/// Переносим ключевые столбцы в начало блока.
+		/** Переносим ключевые столбцы в начало блока.
+		  * Именно там их будет ожидать NonJoinedBlockInputStream.
+		  */
 		size_t key_num = 0;
 		for (const auto & name : key_names_right)
 		{
@@ -810,6 +818,8 @@ void Join::checkTypesOfKeys(const Block & block_left, const Block & block_right)
 
 void Join::joinBlock(Block & block) const
 {
+//	std::cerr << "joinBlock: " << block.dumpStructure() << "\n";
+
 	Poco::ScopedReadRWLock lock(rwlock);
 
 	checkTypesOfKeys(block, sample_block_with_keys);
@@ -917,6 +927,8 @@ public:
 
 		result_sample_block = left_sample_block;
 
+//		std::cerr << result_sample_block.dumpStructure() << "\n";
+
 		/// Добавляем в блок новые столбцы.
 		for (size_t i = 0; i < num_columns_right; ++i)
 		{
@@ -932,10 +944,11 @@ public:
 		{
 			const String & name = left_sample_block.getByPosition(i).name;
 
-			if (parent.key_names_left.end() == std::find(parent.key_names_left.begin(), parent.key_names_left.end(), name))
+			auto found_key_column = std::find(parent.key_names_left.begin(), parent.key_names_left.end(), name);
+			if (parent.key_names_left.end() == found_key_column)
 				column_numbers_left.push_back(i);
 			else
-				column_numbers_keys_and_right.push_back(i);
+				column_numbers_keys_and_right.push_back(found_key_column - parent.key_names_left.begin());
 		}
 
 		for (size_t i = 0; i < num_columns_right; ++i)
@@ -1046,8 +1059,6 @@ private:
 
 		for (; it != end; ++it)
 		{
-//			std::cerr << it->second.getUsed() << "\n";
-
 			if (it->second.getUsed())
 				continue;
 

dbms/src/Storages/ITableDeclaration.cpp

@@ -259,7 +259,7 @@ void ITableDeclaration::check(const Block & block, bool need_all) const
 		const ColumnWithTypeAndName & column = block.getByPosition(i);
 
 		if (names_in_block.count(column.name))
-			throw Exception("Duplicate column  " + column.name + " in block",
+			throw Exception("Duplicate column " + column.name + " in block",
 							ErrorCodes::DUPLICATE_COLUMN);
 
 		names_in_block.insert(column.name);

dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp

@@ -1,6 +1,7 @@
 #include <DB/Storages/MergeTree/MergeTreeDataSelectExecutor.h>
 #include <DB/Storages/MergeTree/MergeTreeBlockInputStream.h>
-#include <DB/Interpreters/ExpressionAnalyzer.h>
+#include <DB/Storages/MergeTree/MergeTreeReadPool.h>
+#include <DB/Storages/MergeTree/MergeTreeThreadBlockInputStream.h>
 #include <DB/Parsers/ASTIdentifier.h>
 #include <DB/DataStreams/ExpressionBlockInputStream.h>
 #include <DB/DataStreams/FilterBlockInputStream.h>
@@ -346,9 +347,9 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongThreads(
 	const Names & virt_columns,
 	const Settings & settings)
 {
-	const size_t min_marks_for_concurrent_read =
+	const std::size_t min_marks_for_concurrent_read =
 		(settings.merge_tree_min_rows_for_concurrent_read + data.index_granularity - 1) / data.index_granularity;
-	const size_t max_marks_to_use_cache =
+	const std::size_t max_marks_to_use_cache =
 		(settings.merge_tree_max_rows_to_use_cache + data.index_granularity - 1) / data.index_granularity;
 
 	/// Посчитаем засечки для каждого куска.
@@ -370,7 +371,27 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongThreads(
 
 	BlockInputStreams res;
 
-	if (sum_marks > 0)
+	if (sum_marks > 0 && settings.merge_tree_uniform_read_distribution == 1)
+	{
+		MergeTreeReadPoolPtr pool = std::make_shared<MergeTreeReadPool>(
+			threads, sum_marks, min_marks_for_concurrent_read, parts, data, prewhere_actions, prewhere_column, true,
+			column_names);
+
+		for (std::size_t i = 0; i < threads; ++i)
+			res.emplace_back(new MergeTreeThreadBlockInputStream{
+				i, pool, min_marks_for_concurrent_read, max_block_size, data, use_uncompressed_cache, prewhere_actions,
+				prewhere_column, settings, virt_columns
+			});
+
+		/// Оценим общее количество строк - для прогресс-бара.
+		const std::size_t total_rows = data.index_granularity * sum_marks;
+
+		/// Выставим приблизительное количество строк только для первого источника
+		static_cast<IProfilingBlockInputStream &>(*res.front()).setTotalRowsApprox(total_rows);
+
+		LOG_TRACE(log, "Reading approx. " << total_rows);
+	}
+	else if (sum_marks > 0)
 	{
 		const size_t min_marks_per_thread = (sum_marks - 1) / threads + 1;
 
@@ -468,7 +489,10 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongThreadsFinal
 	const Settings & settings,
 	const Context & context)
 {
-	size_t max_marks_to_use_cache = (settings.merge_tree_max_rows_to_use_cache + data.index_granularity - 1) / data.index_granularity;
+	const size_t max_marks_to_use_cache =
+		(settings.merge_tree_max_rows_to_use_cache + data.index_granularity - 1) / data.index_granularity;
+	const size_t min_marks_for_read_task =
+		(settings.merge_tree_min_rows_for_concurrent_read + data.index_granularity - 1) / data.index_granularity;
 
 	size_t sum_marks = 0;
 	for (size_t i = 0; i < parts.size(); ++i)
@@ -480,26 +504,55 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongThreadsFinal
 
 	BlockInputStreams to_merge;
 
-	for (size_t part_index = 0; part_index < parts.size(); ++part_index)
+	if (settings.merge_tree_uniform_read_distribution == 1)
 	{
-		RangesInDataPart & part = parts[part_index];
+		MergeTreeReadPoolPtr pool = std::make_shared<MergeTreeReadPool>(
+			parts.size(), sum_marks, min_marks_for_read_task, parts, data, prewhere_actions, prewhere_column, true,
+			column_names, true);
 
-		BlockInputStreamPtr source_stream = new MergeTreeBlockInputStream(
-			data.getFullPath() + part.data_part->name + '/', max_block_size, column_names, data,
-			part.data_part, part.ranges, use_uncompressed_cache,
-			prewhere_actions, prewhere_column, true, settings.min_bytes_to_use_direct_io, settings.max_read_buffer_size);
-
-		for (const String & virt_column : virt_columns)
+		for (const auto i : ext::range(0, parts.size()))
 		{
-			if (virt_column == "_part")
-				source_stream = new AddingConstColumnBlockInputStream<String>(
-					source_stream, new DataTypeString, part.data_part->name, "_part");
-			else if (virt_column == "_part_index")
-				source_stream = new AddingConstColumnBlockInputStream<UInt64>(
-					source_stream, new DataTypeUInt64, part.part_index_in_query, "_part_index");
+			BlockInputStreamPtr source_stream{
+				new MergeTreeThreadBlockInputStream{
+					i, pool, min_marks_for_read_task, max_block_size, data, use_uncompressed_cache, prewhere_actions,
+					prewhere_column, settings, virt_columns
+				}
+			};
+
+			to_merge.push_back(new ExpressionBlockInputStream(source_stream, data.getPrimaryExpression()));
 		}
 
-		to_merge.push_back(new ExpressionBlockInputStream(source_stream, data.getPrimaryExpression()));
+		/// Оценим общее количество строк - для прогресс-бара.
+		const std::size_t total_rows = data.index_granularity * sum_marks;
+
+		/// Выставим приблизительное количество строк только для первого источника
+		static_cast<IProfilingBlockInputStream &>(*to_merge.front()).setTotalRowsApprox(total_rows);
+
+		LOG_TRACE(log, "Reading approx. " << total_rows);
+	}
+	else
+	{
+		for (size_t part_index = 0; part_index < parts.size(); ++part_index)
+		{
+			RangesInDataPart & part = parts[part_index];
+
+			BlockInputStreamPtr source_stream = new MergeTreeBlockInputStream(
+				data.getFullPath() + part.data_part->name + '/', max_block_size, column_names, data,
+				part.data_part, part.ranges, use_uncompressed_cache,
+				prewhere_actions, prewhere_column, true, settings.min_bytes_to_use_direct_io, settings.max_read_buffer_size);
+
+			for (const String & virt_column : virt_columns)
+			{
+				if (virt_column == "_part")
+					source_stream = new AddingConstColumnBlockInputStream<String>(
+						source_stream, new DataTypeString, part.data_part->name, "_part");
+				else if (virt_column == "_part_index")
+					source_stream = new AddingConstColumnBlockInputStream<UInt64>(
+						source_stream, new DataTypeUInt64, part.part_index_in_query, "_part_index");
+			}
+
+			to_merge.push_back(new ExpressionBlockInputStream(source_stream, data.getPrimaryExpression()));
+		}
 	}
 
 	BlockInputStreams res;

dbms/src/Storages/StorageBuffer.cpp

@@ -149,8 +149,11 @@ static void appendBlock(const Block & from, Block & to)
 			throw Exception("Cannot append block to another: different type of columns at index " + toString(column_no)
 				+ ". Block 1: " + from.dumpStructure() + ". Block 2: " + to.dumpStructure(), ErrorCodes::BLOCKS_HAS_DIFFERENT_STRUCTURE);
 
-		for (size_t row_no = 0; row_no < rows; ++row_no)
-			col_to.insertFrom(col_from, row_no);
+		if (col_to.empty())
+			to.getByPosition(column_no).column = col_from.clone();
+		else
+			for (size_t row_no = 0; row_no < rows; ++row_no)
+				col_to.insertFrom(col_from, row_no);
 	}
 }
 
@@ -243,24 +246,18 @@ private:
 			buffer.data = sorted_block.cloneEmpty();
 		}
 
-		/// Если после вставки в буфер, ограничения будут превышены, то будем сбрасывать буфер.
+		/** Если после вставки в буфер, ограничения будут превышены, то будем сбрасывать буфер.
+		  * Это также защищает от неограниченного потребления оперативки, так как в случае невозможности записать в таблицу,
+		  *  будет выкинуто исключение, а новые данные не будут добавлены в буфер.
+		  */
 		if (storage.checkThresholds(buffer, time(0), sorted_block.rowsInFirstColumn(), sorted_block.bytes()))
 		{
-			/// Вытащим из буфера блок, заменим буфер на пустой. После этого можно разблокировать mutex.
-			Block block_to_write;
-			buffer.data.swap(block_to_write);
-			buffer.first_write_time = 0;
 			lock.unlock();
-
-			if (!storage.no_destination)
-			{
-				auto destination = storage.context.tryGetTable(storage.destination_database, storage.destination_table);
-				appendBlock(sorted_block, block_to_write);
-				storage.writeBlockToDestination(block_to_write, destination);
-			}
+			storage.flushBuffer(buffer, false);
+			lock.lock();
 		}
-		else
-			appendBlock(sorted_block, buffer.data);
+
+		appendBlock(sorted_block, buffer.data);
 	}
 };
 
@@ -278,7 +275,14 @@ void StorageBuffer::shutdown()
 	if (flush_thread.joinable())
 		flush_thread.join();
 
-	optimize(context.getSettings());
+	try
+	{
+		optimize(context.getSettings());
+	}
+	catch (...)
+	{
+		tryLogCurrentException(__PRETTY_FUNCTION__);
+	}
 }
 
 
@@ -331,8 +335,16 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds)
 	{
 		std::lock_guard<std::mutex> lock(buffer.mutex);
 
-		if (check_thresholds && !checkThresholds(buffer, current_time))
-			return;
+		if (check_thresholds)
+		{
+			if (!checkThresholds(buffer, current_time))
+				return;
+		}
+		else
+		{
+			if (buffer.data.rowsInFirstColumn() == 0)
+				return;
+		}
 
 		buffer.data.swap(block_to_write);
 		buffer.first_write_time = 0;
@@ -357,10 +369,11 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds)
 			  * Замечание: остаётся проблема - из-за того, что в разных попытках вставляются разные блоки,
 			  *  теряется идемпотентность вставки в ReplicatedMergeTree.
 			  */
-			appendBlock(block_to_write, buffer.data);
-			buffer.data.swap(block_to_write);
+			appendBlock(buffer.data, block_to_write);
 		}
 
+		buffer.data.swap(block_to_write);
+
 		if (!buffer.first_write_time)
 			buffer.first_write_time = current_time;
 

dbms/src/Storages/tests/storage_log.cpp

@@ -85,7 +85,7 @@ int main(int argc, char ** argv)
 
 			DB::WriteBufferFromOStream out_buf(std::cout);
 
-			DB::LimitBlockInputStream in_limit(in, 10);
+			DB::LimitBlockInputStream in_limit(in, 10, 0);
 			DB::RowOutputStreamPtr output_ = new DB::TabSeparatedRowOutputStream(out_buf, sample);
 			DB::BlockOutputStreamFromRowOutputStream output(output_);
 

dbms/tests/queries/0_stateless/00076_ip_coding_functions.reference

@@ -16,3 +16,79 @@
 1
 1
 1
+00000000000000000000FFFF4D583737
+00000000000000000000FFFF4D583737
+00000000000000000000FFFF7F000001
+00000000000000000000FFFFC1FC110A
+2001:db8:ac10:fe01:feed:babe:cafe:f00d
+2001:db8:ac10:fe01:feed:babe:cafe:f000
+2001:db8:ac10:fe01:feed:babe:cafe:0
+2001:db8:ac10:fe01:feed:babe:ca00:0
+2001:db8:ac10:fe01:feed:babe::
+2001:db8:ac10:fe01:feed:ba00::
+2001:db8:ac10:fe01:feed::
+2001:db8:ac10:fe01:fe00::
+2001:db8:ac10:fe01::
+2001:db8:ac10:fe00::
+2001:db8:ac10::
+2001:db8:ac00::
+2001:db8::
+2001:d00::
+2001::
+2000::
+::
+2001:db8:ac10:fe01:feed:babe:cafe:f00d
+2001:db8:ac10:fe01:feed:babe:cafe:f00d
+2001:db8:ac10:fe01:feed:babe:cafe:f00d
+2001:db8:ac10:fe01:feed:babe:cafe:f00d
+2001:db8:ac10:fe01:feed:babe:cafe:f00d
+2001:db8:ac10:fe01:feed:babe:cafe:f00d
+2001:db8:ac10:fe01:feed:babe:cafe:f00d
+2001:db8:ac10:fe01:feed:babe:cafe:f00d
+2001:db8:ac10:fe01:feed:babe:cafe:f00d
+2001:db8:ac10:fe01:feed:babe:cafe:f00d
+2001:db8:ac10:fe01:feed:babe:cafe:f00d
+2001:db8:ac10:fe01:feed:babe:cafe:f00d
+2001:db8:ac10:fe01:feed:babe:cafe:f00d
+2001:db8:ac10:fe01:feed:babe:cafe:f00d
+2001:db8:ac10:fe01:feed:babe:cafe:f00d
+2001:db8:ac10:fe01:feed:babe:cafe:f00d
+::ffff:193.252.17.10
+::ffff:193.252.17.10
+::ffff:193.252.17.10
+::ffff:193.252.17.10
+::ffff:193.252.17.10
+::ffff:193.252.17.10
+::ffff:193.252.17.10
+::ffff:193.252.17.10
+::ffff:193.252.17.10
+::ffff:193.252.17.10
+::ffff:193.252.17.10
+::ffff:193.252.17.10
+::ffff:193.252.17.10
+::ffff:193.252.17.10
+::ffff:193.252.17.10
+::ffff:193.252.17.10
+::ffff:193.252.17.10
+::ffff:193.252.17.0
+::ffff:193.252.0.0
+::ffff:193.0.0.0
+::ffff:0.0.0.0
+::ff00:0:0
+::
+::
+::
+::
+::
+::
+::
+::
+::
+::
+::
+2001:db8:ac10:fe01:abad:babe:fa00:0
+2001:db8:ac10:fe01:dead:c0de:ca00:0
+2001:db8:ac10:fe01:feed:babe:ca00:0
+::ffff:77.0.0.0
+::ffff:127.0.0.0
+::ffff:193.0.0.0

dbms/tests/queries/0_stateless/00076_ip_coding_functions.sql

@@ -17,3 +17,109 @@ select IPv6StringToNum('') == toFixedString(materialize(''), 16);
 select IPv6StringToNum(materialize('')) == toFixedString(materialize(''), 16);
 select IPv6StringToNum('not an ip string') == toFixedString(materialize(''), 16);
 select IPv6StringToNum(materialize('not an ip string')) == toFixedString(materialize(''), 16);
+
+/* IPv4ToIPv6 */
+
+SELECT hex(IPv4ToIPv6(1297626935));
+
+/* Тест с таблицей */
+
+DROP TABLE IF EXISTS test.addresses;
+CREATE TABLE test.addresses(addr UInt32) ENGINE = Memory;
+INSERT INTO test.addresses(addr) VALUES (1297626935), (2130706433), (3254522122);
+SELECT hex(IPv4ToIPv6(addr)) FROM test.addresses ORDER BY addr ASC;
+
+/* cutIPv6 */
+
+/*  Реальный IPv6-адрес */
+
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 0);
+
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 1, 0);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 2, 0);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 3, 0);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 4, 0);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 5, 0);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 6, 0);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 7, 0);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 8, 0);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 9, 0);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 10, 0);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 11, 0);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 12, 0);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 13, 0);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 14, 0);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 15, 0);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 16, 0);
+
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 1);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 2);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 3);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 4);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 5);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 6);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 7);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 8);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 9);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 10);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 11);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 12);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 13);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 14);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 15);
+SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 16);
+
+/*  IPv4-mapped IPv6-адрес */
+
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 0);
+
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 1, 0);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 2, 0);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 3, 0);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 4, 0);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 5, 0);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 6, 0);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 7, 0);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 8, 0);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 9, 0);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 10, 0);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 11, 0);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 12, 0);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 13, 0);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 14, 0);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 15, 0);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 16, 0);
+
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 1);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 2);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 3);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 4);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 5);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 6);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 7);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 8);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 9);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 10);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 11);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 12);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 13);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 14);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 15);
+SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 16);
+
+/* Тест с таблицами */
+
+/*  Реальные IPv6-адреса */
+
+DROP TABLE IF EXISTS test.addresses;
+CREATE TABLE test.addresses(addr String) ENGINE = Memory;
+INSERT INTO test.addresses(addr) VALUES ('20010DB8AC10FE01FEEDBABECAFEF00D'), ('20010DB8AC10FE01DEADC0DECAFED00D'), ('20010DB8AC10FE01ABADBABEFACEB00C');
+SELECT cutIPv6(toFixedString(unhex(addr), 16), 3, 0) FROM test.addresses ORDER BY addr ASC;
+
+/*  IPv4-mapped IPv6-адреса */
+
+DROP TABLE IF EXISTS test.addresses;
+CREATE TABLE test.addresses(addr String) ENGINE = Memory;
+INSERT INTO test.addresses(addr) VALUES ('00000000000000000000FFFFC1FC110A'), ('00000000000000000000FFFF4D583737'), ('00000000000000000000FFFF7F000001');
+SELECT cutIPv6(toFixedString(unhex(addr), 16), 0, 3) FROM test.addresses ORDER BY addr ASC;
+

dbms/tests/queries/0_stateless/00212_aggregate_function_uniq.reference

@@ -239,25 +239,25 @@
 31	162
 35	162
 36	162
-0	54226
-1	54034
-3	54016
-6	53982
-7	54076
-9	54218
-10	27075
-11	54093
-13	54108
-14	54096
-17	54294
-19	54070
-20	54028
-21	54170
-22	54106
-26	54103
-31	54050
-35	54130
-36	53868
+0	53901
+1	54056
+3	53999
+6	54129
+7	54213
+9	53853
+10	26975
+11	54064
+13	53963
+14	53997
+17	54129
+19	53923
+20	53958
+21	54117
+22	54150
+26	54047
+31	53948
+35	53931
+36	53982
 0.125	1
 0.5	1
 0.05	1
@@ -291,25 +291,25 @@
 0.043	162
 0.037	162
 0.071	162
-0.045	54170
-0.125	54076
-0.5	54034
-0.05	54070
-0.143	53982
-0.091	27075
-0.056	54294
-0.048	54028
-0.083	54093
-0.25	54016
-1	54226
-0.1	54218
-0.028	54130
-0.027	53868
-0.031	54050
-0.067	54096
-0.043	54106
-0.037	54103
-0.071	54108
+0.045	54117
+0.125	54213
+0.5	54056
+0.05	53923
+0.143	54129
+0.091	26975
+0.056	54129
+0.048	53958
+0.083	54064
+0.25	53999
+1	53901
+0.1	53853
+0.028	53931
+0.027	53982
+0.031	53948
+0.067	53997
+0.043	54150
+0.037	54047
+0.071	53963
 0.5	1
 0.05	1
 0.25	1
@@ -343,25 +343,25 @@
 0.037	162
 0.1	163
 1	162
-0.5	54034
-0.05	54070
-0.25	54016
-0.048	54028
-0.091	27075
-0.043	54106
-0.071	54108
-0.083	54093
-0.125	54076
-0.031	54050
-0.143	53982
-0.028	54130
-0.067	54096
-0.045	54170
-0.027	53868
-0.056	54294
-0.037	54103
-0.1	54218
-1	54226
+0.5	54056
+0.05	53923
+0.25	53999
+0.048	53958
+0.091	26975
+0.043	54150
+0.071	53963
+0.083	54064
+0.125	54213
+0.031	53948
+0.143	54129
+0.028	53931
+0.067	53997
+0.045	54117
+0.027	53982
+0.056	54129
+0.037	54047
+0.1	53853
+1	53901
 1	1
 3	1
 6	1

dbms/tests/queries/0_stateless/00216_bit_test_function_family.reference

@@ -0,0 +1,771 @@
+1	1	1	1
+1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1
+1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1	1
+1	0	0	0	0	0	0	0	0	0	0
+1	1	1	0	0	0	0	0	0	0	1
+1	2	2	0	0	0	0	0	0	1	0
+1	3	3	0	0	0	0	0	0	1	1
+1	4	4	0	0	0	0	0	1	0	0
+1	5	5	0	0	0	0	0	1	0	1
+1	6	6	0	0	0	0	0	1	1	0
+1	7	7	0	0	0	0	0	1	1	1
+1	8	8	0	0	0	0	1	0	0	0
+1	9	9	0	0	0	0	1	0	0	1
+1	10	10	0	0	0	0	1	0	1	0
+1	11	11	0	0	0	0	1	0	1	1
+1	12	12	0	0	0	0	1	1	0	0
+1	13	13	0	0	0	0	1	1	0	1
+1	14	14	0	0	0	0	1	1	1	0
+1	15	15	0	0	0	0	1	1	1	1
+1	16	16	0	0	0	1	0	0	0	0
+1	17	17	0	0	0	1	0	0	0	1
+1	18	18	0	0	0	1	0	0	1	0
+1	19	19	0	0	0	1	0	0	1	1
+1	20	20	0	0	0	1	0	1	0	0
+1	21	21	0	0	0	1	0	1	0	1
+1	22	22	0	0	0	1	0	1	1	0
+1	23	23	0	0	0	1	0	1	1	1
+1	24	24	0	0	0	1	1	0	0	0
+1	25	25	0	0	0	1	1	0	0	1
+1	26	26	0	0	0	1	1	0	1	0
+1	27	27	0	0	0	1	1	0	1	1
+1	28	28	0	0	0	1	1	1	0	0
+1	29	29	0	0	0	1	1	1	0	1
+1	30	30	0	0	0	1	1	1	1	0
+1	31	31	0	0	0	1	1	1	1	1
+1	32	32	0	0	1	0	0	0	0	0
+1	33	33	0	0	1	0	0	0	0	1
+1	34	34	0	0	1	0	0	0	1	0
+1	35	35	0	0	1	0	0	0	1	1
+1	36	36	0	0	1	0	0	1	0	0
+1	37	37	0	0	1	0	0	1	0	1
+1	38	38	0	0	1	0	0	1	1	0
+1	39	39	0	0	1	0	0	1	1	1
+1	40	40	0	0	1	0	1	0	0	0
+1	41	41	0	0	1	0	1	0	0	1
+1	42	42	0	0	1	0	1	0	1	0
+1	43	43	0	0	1	0	1	0	1	1
+1	44	44	0	0	1	0	1	1	0	0
+1	45	45	0	0	1	0	1	1	0	1
+1	46	46	0	0	1	0	1	1	1	0
+1	47	47	0	0	1	0	1	1	1	1
+1	48	48	0	0	1	1	0	0	0	0
+1	49	49	0	0	1	1	0	0	0	1
+1	50	50	0	0	1	1	0	0	1	0
+1	51	51	0	0	1	1	0	0	1	1
+1	52	52	0	0	1	1	0	1	0	0
+1	53	53	0	0	1	1	0	1	0	1
+1	54	54	0	0	1	1	0	1	1	0
+1	55	55	0	0	1	1	0	1	1	1
+1	56	56	0	0	1	1	1	0	0	0
+1	57	57	0	0	1	1	1	0	0	1
+1	58	58	0	0	1	1	1	0	1	0
+1	59	59	0	0	1	1	1	0	1	1
+1	60	60	0	0	1	1	1	1	0	0
+1	61	61	0	0	1	1	1	1	0	1
+1	62	62	0	0	1	1	1	1	1	0
+1	63	63	0	0	1	1	1	1	1	1
+1	64	64	0	1	0	0	0	0	0	0
+1	65	65	0	1	0	0	0	0	0	1
+1	66	66	0	1	0	0	0	0	1	0
+1	67	67	0	1	0	0	0	0	1	1
+1	68	68	0	1	0	0	0	1	0	0
+1	69	69	0	1	0	0	0	1	0	1
+1	70	70	0	1	0	0	0	1	1	0
+1	71	71	0	1	0	0	0	1	1	1
+1	72	72	0	1	0	0	1	0	0	0
+1	73	73	0	1	0	0	1	0	0	1
+1	74	74	0	1	0	0	1	0	1	0
+1	75	75	0	1	0	0	1	0	1	1
+1	76	76	0	1	0	0	1	1	0	0
+1	77	77	0	1	0	0	1	1	0	1
+1	78	78	0	1	0	0	1	1	1	0
+1	79	79	0	1	0	0	1	1	1	1
+1	80	80	0	1	0	1	0	0	0	0
+1	81	81	0	1	0	1	0	0	0	1
+1	82	82	0	1	0	1	0	0	1	0
+1	83	83	0	1	0	1	0	0	1	1
+1	84	84	0	1	0	1	0	1	0	0
+1	85	85	0	1	0	1	0	1	0	1
+1	86	86	0	1	0	1	0	1	1	0
+1	87	87	0	1	0	1	0	1	1	1
+1	88	88	0	1	0	1	1	0	0	0
+1	89	89	0	1	0	1	1	0	0	1
+1	90	90	0	1	0	1	1	0	1	0
+1	91	91	0	1	0	1	1	0	1	1
+1	92	92	0	1	0	1	1	1	0	0
+1	93	93	0	1	0	1	1	1	0	1
+1	94	94	0	1	0	1	1	1	1	0
+1	95	95	0	1	0	1	1	1	1	1
+1	96	96	0	1	1	0	0	0	0	0
+1	97	97	0	1	1	0	0	0	0	1
+1	98	98	0	1	1	0	0	0	1	0
+1	99	99	0	1	1	0	0	0	1	1
+1	100	100	0	1	1	0	0	1	0	0
+1	101	101	0	1	1	0	0	1	0	1
+1	102	102	0	1	1	0	0	1	1	0
+1	103	103	0	1	1	0	0	1	1	1
+1	104	104	0	1	1	0	1	0	0	0
+1	105	105	0	1	1	0	1	0	0	1
+1	106	106	0	1	1	0	1	0	1	0
+1	107	107	0	1	1	0	1	0	1	1
+1	108	108	0	1	1	0	1	1	0	0
+1	109	109	0	1	1	0	1	1	0	1
+1	110	110	0	1	1	0	1	1	1	0
+1	111	111	0	1	1	0	1	1	1	1
+1	112	112	0	1	1	1	0	0	0	0
+1	113	113	0	1	1	1	0	0	0	1
+1	114	114	0	1	1	1	0	0	1	0
+1	115	115	0	1	1	1	0	0	1	1
+1	116	116	0	1	1	1	0	1	0	0
+1	117	117	0	1	1	1	0	1	0	1
+1	118	118	0	1	1	1	0	1	1	0
+1	119	119	0	1	1	1	0	1	1	1
+1	120	120	0	1	1	1	1	0	0	0
+1	121	121	0	1	1	1	1	0	0	1
+1	122	122	0	1	1	1	1	0	1	0
+1	123	123	0	1	1	1	1	0	1	1
+1	124	124	0	1	1	1	1	1	0	0
+1	125	125	0	1	1	1	1	1	0	1
+1	126	126	0	1	1	1	1	1	1	0
+1	127	127	0	1	1	1	1	1	1	1
+1	128	128	1	0	0	0	0	0	0	0
+1	129	129	1	0	0	0	0	0	0	1
+1	130	130	1	0	0	0	0	0	1	0
+1	131	131	1	0	0	0	0	0	1	1
+1	132	132	1	0	0	0	0	1	0	0
+1	133	133	1	0	0	0	0	1	0	1
+1	134	134	1	0	0	0	0	1	1	0
+1	135	135	1	0	0	0	0	1	1	1
+1	136	136	1	0	0	0	1	0	0	0
+1	137	137	1	0	0	0	1	0	0	1
+1	138	138	1	0	0	0	1	0	1	0
+1	139	139	1	0	0	0	1	0	1	1
+1	140	140	1	0	0	0	1	1	0	0
+1	141	141	1	0	0	0	1	1	0	1
+1	142	142	1	0	0	0	1	1	1	0
+1	143	143	1	0	0	0	1	1	1	1
+1	144	144	1	0	0	1	0	0	0	0
+1	145	145	1	0	0	1	0	0	0	1
+1	146	146	1	0	0	1	0	0	1	0
+1	147	147	1	0	0	1	0	0	1	1
+1	148	148	1	0	0	1	0	1	0	0
+1	149	149	1	0	0	1	0	1	0	1
+1	150	150	1	0	0	1	0	1	1	0
+1	151	151	1	0	0	1	0	1	1	1
+1	152	152	1	0	0	1	1	0	0	0
+1	153	153	1	0	0	1	1	0	0	1
+1	154	154	1	0	0	1	1	0	1	0
+1	155	155	1	0	0	1	1	0	1	1
+1	156	156	1	0	0	1	1	1	0	0
+1	157	157	1	0	0	1	1	1	0	1
+1	158	158	1	0	0	1	1	1	1	0
+1	159	159	1	0	0	1	1	1	1	1
+1	160	160	1	0	1	0	0	0	0	0
+1	161	161	1	0	1	0	0	0	0	1
+1	162	162	1	0	1	0	0	0	1	0
+1	163	163	1	0	1	0	0	0	1	1
+1	164	164	1	0	1	0	0	1	0	0
+1	165	165	1	0	1	0	0	1	0	1
+1	166	166	1	0	1	0	0	1	1	0
+1	167	167	1	0	1	0	0	1	1	1
+1	168	168	1	0	1	0	1	0	0	0
+1	169	169	1	0	1	0	1	0	0	1
+1	170	170	1	0	1	0	1	0	1	0
+1	171	171	1	0	1	0	1	0	1	1
+1	172	172	1	0	1	0	1	1	0	0
+1	173	173	1	0	1	0	1	1	0	1
+1	174	174	1	0	1	0	1	1	1	0
+1	175	175	1	0	1	0	1	1	1	1
+1	176	176	1	0	1	1	0	0	0	0
+1	177	177	1	0	1	1	0	0	0	1
+1	178	178	1	0	1	1	0	0	1	0
+1	179	179	1	0	1	1	0	0	1	1
+1	180	180	1	0	1	1	0	1	0	0
+1	181	181	1	0	1	1	0	1	0	1
+1	182	182	1	0	1	1	0	1	1	0
+1	183	183	1	0	1	1	0	1	1	1
+1	184	184	1	0	1	1	1	0	0	0
+1	185	185	1	0	1	1	1	0	0	1
+1	186	186	1	0	1	1	1	0	1	0
+1	187	187	1	0	1	1	1	0	1	1
+1	188	188	1	0	1	1	1	1	0	0
+1	189	189	1	0	1	1	1	1	0	1
+1	190	190	1	0	1	1	1	1	1	0
+1	191	191	1	0	1	1	1	1	1	1
+1	192	192	1	1	0	0	0	0	0	0
+1	193	193	1	1	0	0	0	0	0	1
+1	194	194	1	1	0	0	0	0	1	0
+1	195	195	1	1	0	0	0	0	1	1
+1	196	196	1	1	0	0	0	1	0	0
+1	197	197	1	1	0	0	0	1	0	1
+1	198	198	1	1	0	0	0	1	1	0
+1	199	199	1	1	0	0	0	1	1	1
+1	200	200	1	1	0	0	1	0	0	0
+1	201	201	1	1	0	0	1	0	0	1
+1	202	202	1	1	0	0	1	0	1	0
+1	203	203	1	1	0	0	1	0	1	1
+1	204	204	1	1	0	0	1	1	0	0
+1	205	205	1	1	0	0	1	1	0	1
+1	206	206	1	1	0	0	1	1	1	0
+1	207	207	1	1	0	0	1	1	1	1
+1	208	208	1	1	0	1	0	0	0	0
+1	209	209	1	1	0	1	0	0	0	1
+1	210	210	1	1	0	1	0	0	1	0
+1	211	211	1	1	0	1	0	0	1	1
+1	212	212	1	1	0	1	0	1	0	0
+1	213	213	1	1	0	1	0	1	0	1
+1	214	214	1	1	0	1	0	1	1	0
+1	215	215	1	1	0	1	0	1	1	1
+1	216	216	1	1	0	1	1	0	0	0
+1	217	217	1	1	0	1	1	0	0	1
+1	218	218	1	1	0	1	1	0	1	0
+1	219	219	1	1	0	1	1	0	1	1
+1	220	220	1	1	0	1	1	1	0	0
+1	221	221	1	1	0	1	1	1	0	1
+1	222	222	1	1	0	1	1	1	1	0
+1	223	223	1	1	0	1	1	1	1	1
+1	224	224	1	1	1	0	0	0	0	0
+1	225	225	1	1	1	0	0	0	0	1
+1	226	226	1	1	1	0	0	0	1	0
+1	227	227	1	1	1	0	0	0	1	1
+1	228	228	1	1	1	0	0	1	0	0
+1	229	229	1	1	1	0	0	1	0	1
+1	230	230	1	1	1	0	0	1	1	0
+1	231	231	1	1	1	0	0	1	1	1
+1	232	232	1	1	1	0	1	0	0	0
+1	233	233	1	1	1	0	1	0	0	1
+1	234	234	1	1	1	0	1	0	1	0
+1	235	235	1	1	1	0	1	0	1	1
+1	236	236	1	1	1	0	1	1	0	0
+1	237	237	1	1	1	0	1	1	0	1
+1	238	238	1	1	1	0	1	1	1	0
+1	239	239	1	1	1	0	1	1	1	1
+1	240	240	1	1	1	1	0	0	0	0
+1	241	241	1	1	1	1	0	0	0	1
+1	242	242	1	1	1	1	0	0	1	0
+1	243	243	1	1	1	1	0	0	1	1
+1	244	244	1	1	1	1	0	1	0	0
+1	245	245	1	1	1	1	0	1	0	1
+1	246	246	1	1	1	1	0	1	1	0
+1	247	247	1	1	1	1	0	1	1	1
+1	248	248	1	1	1	1	1	0	0	0
+1	249	249	1	1	1	1	1	0	0	1
+1	250	250	1	1	1	1	1	0	1	0
+1	251	251	1	1	1	1	1	0	1	1
+1	252	252	1	1	1	1	1	1	0	0
+1	253	253	1	1	1	1	1	1	0	1
+1	254	254	1	1	1	1	1	1	1	0
+1	255	255	1	1	1	1	1	1	1	1
+1	0	0	0	0	0	0	0	0	0	0
+1	1	1	0	0	0	0	0	0	0	1
+1	2	2	0	0	0	0	0	0	1	0
+1	3	3	0	0	0	0	0	0	1	1
+1	4	4	0	0	0	0	0	1	0	0
+1	5	5	0	0	0	0	0	1	0	1
+1	6	6	0	0	0	0	0	1	1	0
+1	7	7	0	0	0	0	0	1	1	1
+1	8	8	0	0	0	0	1	0	0	0
+1	9	9	0	0	0	0	1	0	0	1
+1	10	10	0	0	0	0	1	0	1	0
+1	11	11	0	0	0	0	1	0	1	1
+1	12	12	0	0	0	0	1	1	0	0
+1	13	13	0	0	0	0	1	1	0	1
+1	14	14	0	0	0	0	1	1	1	0
+1	15	15	0	0	0	0	1	1	1	1
+1	16	16	0	0	0	1	0	0	0	0
+1	17	17	0	0	0	1	0	0	0	1
+1	18	18	0	0	0	1	0	0	1	0
+1	19	19	0	0	0	1	0	0	1	1
+1	20	20	0	0	0	1	0	1	0	0
+1	21	21	0	0	0	1	0	1	0	1
+1	22	22	0	0	0	1	0	1	1	0
+1	23	23	0	0	0	1	0	1	1	1
+1	24	24	0	0	0	1	1	0	0	0
+1	25	25	0	0	0	1	1	0	0	1
+1	26	26	0	0	0	1	1	0	1	0
+1	27	27	0	0	0	1	1	0	1	1
+1	28	28	0	0	0	1	1	1	0	0
+1	29	29	0	0	0	1	1	1	0	1
+1	30	30	0	0	0	1	1	1	1	0
+1	31	31	0	0	0	1	1	1	1	1
+1	32	32	0	0	1	0	0	0	0	0
+1	33	33	0	0	1	0	0	0	0	1
+1	34	34	0	0	1	0	0	0	1	0
+1	35	35	0	0	1	0	0	0	1	1
+1	36	36	0	0	1	0	0	1	0	0
+1	37	37	0	0	1	0	0	1	0	1
+1	38	38	0	0	1	0	0	1	1	0
+1	39	39	0	0	1	0	0	1	1	1
+1	40	40	0	0	1	0	1	0	0	0
+1	41	41	0	0	1	0	1	0	0	1
+1	42	42	0	0	1	0	1	0	1	0
+1	43	43	0	0	1	0	1	0	1	1
+1	44	44	0	0	1	0	1	1	0	0
+1	45	45	0	0	1	0	1	1	0	1
+1	46	46	0	0	1	0	1	1	1	0
+1	47	47	0	0	1	0	1	1	1	1
+1	48	48	0	0	1	1	0	0	0	0
+1	49	49	0	0	1	1	0	0	0	1
+1	50	50	0	0	1	1	0	0	1	0
+1	51	51	0	0	1	1	0	0	1	1
+1	52	52	0	0	1	1	0	1	0	0
+1	53	53	0	0	1	1	0	1	0	1
+1	54	54	0	0	1	1	0	1	1	0
+1	55	55	0	0	1	1	0	1	1	1
+1	56	56	0	0	1	1	1	0	0	0
+1	57	57	0	0	1	1	1	0	0	1
+1	58	58	0	0	1	1	1	0	1	0
+1	59	59	0	0	1	1	1	0	1	1
+1	60	60	0	0	1	1	1	1	0	0
+1	61	61	0	0	1	1	1	1	0	1
+1	62	62	0	0	1	1	1	1	1	0
+1	63	63	0	0	1	1	1	1	1	1
+1	64	64	0	1	0	0	0	0	0	0
+1	65	65	0	1	0	0	0	0	0	1
+1	66	66	0	1	0	0	0	0	1	0
+1	67	67	0	1	0	0	0	0	1	1
+1	68	68	0	1	0	0	0	1	0	0
+1	69	69	0	1	0	0	0	1	0	1
+1	70	70	0	1	0	0	0	1	1	0
+1	71	71	0	1	0	0	0	1	1	1
+1	72	72	0	1	0	0	1	0	0	0
+1	73	73	0	1	0	0	1	0	0	1
+1	74	74	0	1	0	0	1	0	1	0
+1	75	75	0	1	0	0	1	0	1	1
+1	76	76	0	1	0	0	1	1	0	0
+1	77	77	0	1	0	0	1	1	0	1
+1	78	78	0	1	0	0	1	1	1	0
+1	79	79	0	1	0	0	1	1	1	1
+1	80	80	0	1	0	1	0	0	0	0
+1	81	81	0	1	0	1	0	0	0	1
+1	82	82	0	1	0	1	0	0	1	0
+1	83	83	0	1	0	1	0	0	1	1
+1	84	84	0	1	0	1	0	1	0	0
+1	85	85	0	1	0	1	0	1	0	1
+1	86	86	0	1	0	1	0	1	1	0
+1	87	87	0	1	0	1	0	1	1	1
+1	88	88	0	1	0	1	1	0	0	0
+1	89	89	0	1	0	1	1	0	0	1
+1	90	90	0	1	0	1	1	0	1	0
+1	91	91	0	1	0	1	1	0	1	1
+1	92	92	0	1	0	1	1	1	0	0
+1	93	93	0	1	0	1	1	1	0	1
+1	94	94	0	1	0	1	1	1	1	0
+1	95	95	0	1	0	1	1	1	1	1
+1	96	96	0	1	1	0	0	0	0	0
+1	97	97	0	1	1	0	0	0	0	1
+1	98	98	0	1	1	0	0	0	1	0
+1	99	99	0	1	1	0	0	0	1	1
+1	100	100	0	1	1	0	0	1	0	0
+1	101	101	0	1	1	0	0	1	0	1
+1	102	102	0	1	1	0	0	1	1	0
+1	103	103	0	1	1	0	0	1	1	1
+1	104	104	0	1	1	0	1	0	0	0
+1	105	105	0	1	1	0	1	0	0	1
+1	106	106	0	1	1	0	1	0	1	0
+1	107	107	0	1	1	0	1	0	1	1
+1	108	108	0	1	1	0	1	1	0	0
+1	109	109	0	1	1	0	1	1	0	1
+1	110	110	0	1	1	0	1	1	1	0
+1	111	111	0	1	1	0	1	1	1	1
+1	112	112	0	1	1	1	0	0	0	0
+1	113	113	0	1	1	1	0	0	0	1
+1	114	114	0	1	1	1	0	0	1	0
+1	115	115	0	1	1	1	0	0	1	1
+1	116	116	0	1	1	1	0	1	0	0
+1	117	117	0	1	1	1	0	1	0	1
+1	118	118	0	1	1	1	0	1	1	0
+1	119	119	0	1	1	1	0	1	1	1
+1	120	120	0	1	1	1	1	0	0	0
+1	121	121	0	1	1	1	1	0	0	1
+1	122	122	0	1	1	1	1	0	1	0
+1	123	123	0	1	1	1	1	0	1	1
+1	124	124	0	1	1	1	1	1	0	0
+1	125	125	0	1	1	1	1	1	0	1
+1	126	126	0	1	1	1	1	1	1	0
+1	127	127	0	1	1	1	1	1	1	1
+1	128	128	1	0	0	0	0	0	0	0
+1	129	129	1	0	0	0	0	0	0	1
+1	130	130	1	0	0	0	0	0	1	0
+1	131	131	1	0	0	0	0	0	1	1
+1	132	132	1	0	0	0	0	1	0	0
+1	133	133	1	0	0	0	0	1	0	1
+1	134	134	1	0	0	0	0	1	1	0
+1	135	135	1	0	0	0	0	1	1	1
+1	136	136	1	0	0	0	1	0	0	0
+1	137	137	1	0	0	0	1	0	0	1
+1	138	138	1	0	0	0	1	0	1	0
+1	139	139	1	0	0	0	1	0	1	1
+1	140	140	1	0	0	0	1	1	0	0
+1	141	141	1	0	0	0	1	1	0	1
+1	142	142	1	0	0	0	1	1	1	0
+1	143	143	1	0	0	0	1	1	1	1
+1	144	144	1	0	0	1	0	0	0	0
+1	145	145	1	0	0	1	0	0	0	1
+1	146	146	1	0	0	1	0	0	1	0
+1	147	147	1	0	0	1	0	0	1	1
+1	148	148	1	0	0	1	0	1	0	0
+1	149	149	1	0	0	1	0	1	0	1
+1	150	150	1	0	0	1	0	1	1	0
+1	151	151	1	0	0	1	0	1	1	1
+1	152	152	1	0	0	1	1	0	0	0
+1	153	153	1	0	0	1	1	0	0	1
+1	154	154	1	0	0	1	1	0	1	0
+1	155	155	1	0	0	1	1	0	1	1
+1	156	156	1	0	0	1	1	1	0	0
+1	157	157	1	0	0	1	1	1	0	1
+1	158	158	1	0	0	1	1	1	1	0
+1	159	159	1	0	0	1	1	1	1	1
+1	160	160	1	0	1	0	0	0	0	0
+1	161	161	1	0	1	0	0	0	0	1
+1	162	162	1	0	1	0	0	0	1	0
+1	163	163	1	0	1	0	0	0	1	1
+1	164	164	1	0	1	0	0	1	0	0
+1	165	165	1	0	1	0	0	1	0	1
+1	166	166	1	0	1	0	0	1	1	0
+1	167	167	1	0	1	0	0	1	1	1
+1	168	168	1	0	1	0	1	0	0	0
+1	169	169	1	0	1	0	1	0	0	1
+1	170	170	1	0	1	0	1	0	1	0
+1	171	171	1	0	1	0	1	0	1	1
+1	172	172	1	0	1	0	1	1	0	0
+1	173	173	1	0	1	0	1	1	0	1
+1	174	174	1	0	1	0	1	1	1	0
+1	175	175	1	0	1	0	1	1	1	1
+1	176	176	1	0	1	1	0	0	0	0
+1	177	177	1	0	1	1	0	0	0	1
+1	178	178	1	0	1	1	0	0	1	0
+1	179	179	1	0	1	1	0	0	1	1
+1	180	180	1	0	1	1	0	1	0	0
+1	181	181	1	0	1	1	0	1	0	1
+1	182	182	1	0	1	1	0	1	1	0
+1	183	183	1	0	1	1	0	1	1	1
+1	184	184	1	0	1	1	1	0	0	0
+1	185	185	1	0	1	1	1	0	0	1
+1	186	186	1	0	1	1	1	0	1	0
+1	187	187	1	0	1	1	1	0	1	1
+1	188	188	1	0	1	1	1	1	0	0
+1	189	189	1	0	1	1	1	1	0	1
+1	190	190	1	0	1	1	1	1	1	0
+1	191	191	1	0	1	1	1	1	1	1
+1	192	192	1	1	0	0	0	0	0	0
+1	193	193	1	1	0	0	0	0	0	1
+1	194	194	1	1	0	0	0	0	1	0
+1	195	195	1	1	0	0	0	0	1	1
+1	196	196	1	1	0	0	0	1	0	0
+1	197	197	1	1	0	0	0	1	0	1
+1	198	198	1	1	0	0	0	1	1	0
+1	199	199	1	1	0	0	0	1	1	1
+1	200	200	1	1	0	0	1	0	0	0
+1	201	201	1	1	0	0	1	0	0	1
+1	202	202	1	1	0	0	1	0	1	0
+1	203	203	1	1	0	0	1	0	1	1
+1	204	204	1	1	0	0	1	1	0	0
+1	205	205	1	1	0	0	1	1	0	1
+1	206	206	1	1	0	0	1	1	1	0
+1	207	207	1	1	0	0	1	1	1	1
+1	208	208	1	1	0	1	0	0	0	0
+1	209	209	1	1	0	1	0	0	0	1
+1	210	210	1	1	0	1	0	0	1	0
+1	211	211	1	1	0	1	0	0	1	1
+1	212	212	1	1	0	1	0	1	0	0
+1	213	213	1	1	0	1	0	1	0	1
+1	214	214	1	1	0	1	0	1	1	0
+1	215	215	1	1	0	1	0	1	1	1
+1	216	216	1	1	0	1	1	0	0	0
+1	217	217	1	1	0	1	1	0	0	1
+1	218	218	1	1	0	1	1	0	1	0
+1	219	219	1	1	0	1	1	0	1	1
+1	220	220	1	1	0	1	1	1	0	0
+1	221	221	1	1	0	1	1	1	0	1
+1	222	222	1	1	0	1	1	1	1	0
+1	223	223	1	1	0	1	1	1	1	1
+1	224	224	1	1	1	0	0	0	0	0
+1	225	225	1	1	1	0	0	0	0	1
+1	226	226	1	1	1	0	0	0	1	0
+1	227	227	1	1	1	0	0	0	1	1
+1	228	228	1	1	1	0	0	1	0	0
+1	229	229	1	1	1	0	0	1	0	1
+1	230	230	1	1	1	0	0	1	1	0
+1	231	231	1	1	1	0	0	1	1	1
+1	232	232	1	1	1	0	1	0	0	0
+1	233	233	1	1	1	0	1	0	0	1
+1	234	234	1	1	1	0	1	0	1	0
+1	235	235	1	1	1	0	1	0	1	1
+1	236	236	1	1	1	0	1	1	0	0
+1	237	237	1	1	1	0	1	1	0	1
+1	238	238	1	1	1	0	1	1	1	0
+1	239	239	1	1	1	0	1	1	1	1
+1	240	240	1	1	1	1	0	0	0	0
+1	241	241	1	1	1	1	0	0	0	1
+1	242	242	1	1	1	1	0	0	1	0
+1	243	243	1	1	1	1	0	0	1	1
+1	244	244	1	1	1	1	0	1	0	0
+1	245	245	1	1	1	1	0	1	0	1
+1	246	246	1	1	1	1	0	1	1	0
+1	247	247	1	1	1	1	0	1	1	1
+1	248	248	1	1	1	1	1	0	0	0
+1	249	249	1	1	1	1	1	0	0	1
+1	250	250	1	1	1	1	1	0	1	0
+1	251	251	1	1	1	1	1	0	1	1
+1	252	252	1	1	1	1	1	1	0	0
+1	253	253	1	1	1	1	1	1	0	1
+1	254	254	1	1	1	1	1	1	1	0
+1	255	255	1	1	1	1	1	1	1	1
+1	0	0	0	0	0	0	0	0	0	0
+1	1	1	0	0	0	0	0	0	0	1
+1	2	2	0	0	0	0	0	0	1	0
+1	3	3	0	0	0	0	0	0	1	1
+1	4	4	0	0	0	0	0	1	0	0
+1	5	5	0	0	0	0	0	1	0	1
+1	6	6	0	0	0	0	0	1	1	0
+1	7	7	0	0	0	0	0	1	1	1
+1	8	8	0	0	0	0	1	0	0	0
+1	9	9	0	0	0	0	1	0	0	1
+1	10	10	0	0	0	0	1	0	1	0
+1	11	11	0	0	0	0	1	0	1	1
+1	12	12	0	0	0	0	1	1	0	0
+1	13	13	0	0	0	0	1	1	0	1
+1	14	14	0	0	0	0	1	1	1	0
+1	15	15	0	0	0	0	1	1	1	1
+1	16	16	0	0	0	1	0	0	0	0
+1	17	17	0	0	0	1	0	0	0	1
+1	18	18	0	0	0	1	0	0	1	0
+1	19	19	0	0	0	1	0	0	1	1
+1	20	20	0	0	0	1	0	1	0	0
+1	21	21	0	0	0	1	0	1	0	1
+1	22	22	0	0	0	1	0	1	1	0
+1	23	23	0	0	0	1	0	1	1	1
+1	24	24	0	0	0	1	1	0	0	0
+1	25	25	0	0	0	1	1	0	0	1
+1	26	26	0	0	0	1	1	0	1	0
+1	27	27	0	0	0	1	1	0	1	1
+1	28	28	0	0	0	1	1	1	0	0
+1	29	29	0	0	0	1	1	1	0	1
+1	30	30	0	0	0	1	1	1	1	0
+1	31	31	0	0	0	1	1	1	1	1
+1	32	32	0	0	1	0	0	0	0	0
+1	33	33	0	0	1	0	0	0	0	1
+1	34	34	0	0	1	0	0	0	1	0
+1	35	35	0	0	1	0	0	0	1	1
+1	36	36	0	0	1	0	0	1	0	0
+1	37	37	0	0	1	0	0	1	0	1
+1	38	38	0	0	1	0	0	1	1	0
+1	39	39	0	0	1	0	0	1	1	1
+1	40	40	0	0	1	0	1	0	0	0
+1	41	41	0	0	1	0	1	0	0	1
+1	42	42	0	0	1	0	1	0	1	0
+1	43	43	0	0	1	0	1	0	1	1
+1	44	44	0	0	1	0	1	1	0	0
+1	45	45	0	0	1	0	1	1	0	1
+1	46	46	0	0	1	0	1	1	1	0
+1	47	47	0	0	1	0	1	1	1	1
+1	48	48	0	0	1	1	0	0	0	0
+1	49	49	0	0	1	1	0	0	0	1
+1	50	50	0	0	1	1	0	0	1	0
+1	51	51	0	0	1	1	0	0	1	1
+1	52	52	0	0	1	1	0	1	0	0
+1	53	53	0	0	1	1	0	1	0	1
+1	54	54	0	0	1	1	0	1	1	0
+1	55	55	0	0	1	1	0	1	1	1
+1	56	56	0	0	1	1	1	0	0	0
+1	57	57	0	0	1	1	1	0	0	1
+1	58	58	0	0	1	1	1	0	1	0
+1	59	59	0	0	1	1	1	0	1	1
+1	60	60	0	0	1	1	1	1	0	0
+1	61	61	0	0	1	1	1	1	0	1
+1	62	62	0	0	1	1	1	1	1	0
+1	63	63	0	0	1	1	1	1	1	1
+1	64	64	0	1	0	0	0	0	0	0
+1	65	65	0	1	0	0	0	0	0	1
+1	66	66	0	1	0	0	0	0	1	0
+1	67	67	0	1	0	0	0	0	1	1
+1	68	68	0	1	0	0	0	1	0	0
+1	69	69	0	1	0	0	0	1	0	1
+1	70	70	0	1	0	0	0	1	1	0
+1	71	71	0	1	0	0	0	1	1	1
+1	72	72	0	1	0	0	1	0	0	0
+1	73	73	0	1	0	0	1	0	0	1
+1	74	74	0	1	0	0	1	0	1	0
+1	75	75	0	1	0	0	1	0	1	1
+1	76	76	0	1	0	0	1	1	0	0
+1	77	77	0	1	0	0	1	1	0	1
+1	78	78	0	1	0	0	1	1	1	0
+1	79	79	0	1	0	0	1	1	1	1
+1	80	80	0	1	0	1	0	0	0	0
+1	81	81	0	1	0	1	0	0	0	1
+1	82	82	0	1	0	1	0	0	1	0
+1	83	83	0	1	0	1	0	0	1	1
+1	84	84	0	1	0	1	0	1	0	0
+1	85	85	0	1	0	1	0	1	0	1
+1	86	86	0	1	0	1	0	1	1	0
+1	87	87	0	1	0	1	0	1	1	1
+1	88	88	0	1	0	1	1	0	0	0
+1	89	89	0	1	0	1	1	0	0	1
+1	90	90	0	1	0	1	1	0	1	0
+1	91	91	0	1	0	1	1	0	1	1
+1	92	92	0	1	0	1	1	1	0	0
+1	93	93	0	1	0	1	1	1	0	1
+1	94	94	0	1	0	1	1	1	1	0
+1	95	95	0	1	0	1	1	1	1	1
+1	96	96	0	1	1	0	0	0	0	0
+1	97	97	0	1	1	0	0	0	0	1
+1	98	98	0	1	1	0	0	0	1	0
+1	99	99	0	1	1	0	0	0	1	1
+1	100	100	0	1	1	0	0	1	0	0
+1	101	101	0	1	1	0	0	1	0	1
+1	102	102	0	1	1	0	0	1	1	0
+1	103	103	0	1	1	0	0	1	1	1
+1	104	104	0	1	1	0	1	0	0	0
+1	105	105	0	1	1	0	1	0	0	1
+1	106	106	0	1	1	0	1	0	1	0
+1	107	107	0	1	1	0	1	0	1	1
+1	108	108	0	1	1	0	1	1	0	0
+1	109	109	0	1	1	0	1	1	0	1
+1	110	110	0	1	1	0	1	1	1	0
+1	111	111	0	1	1	0	1	1	1	1
+1	112	112	0	1	1	1	0	0	0	0
+1	113	113	0	1	1	1	0	0	0	1
+1	114	114	0	1	1	1	0	0	1	0
+1	115	115	0	1	1	1	0	0	1	1
+1	116	116	0	1	1	1	0	1	0	0
+1	117	117	0	1	1	1	0	1	0	1
+1	118	118	0	1	1	1	0	1	1	0
+1	119	119	0	1	1	1	0	1	1	1
+1	120	120	0	1	1	1	1	0	0	0
+1	121	121	0	1	1	1	1	0	0	1
+1	122	122	0	1	1	1	1	0	1	0
+1	123	123	0	1	1	1	1	0	1	1
+1	124	124	0	1	1	1	1	1	0	0
+1	125	125	0	1	1	1	1	1	0	1
+1	126	126	0	1	1	1	1	1	1	0
+1	127	127	0	1	1	1	1	1	1	1
+1	128	128	1	0	0	0	0	0	0	0
+1	129	129	1	0	0	0	0	0	0	1
+1	130	130	1	0	0	0	0	0	1	0
+1	131	131	1	0	0	0	0	0	1	1
+1	132	132	1	0	0	0	0	1	0	0
+1	133	133	1	0	0	0	0	1	0	1
+1	134	134	1	0	0	0	0	1	1	0
+1	135	135	1	0	0	0	0	1	1	1
+1	136	136	1	0	0	0	1	0	0	0
+1	137	137	1	0	0	0	1	0	0	1
+1	138	138	1	0	0	0	1	0	1	0
+1	139	139	1	0	0	0	1	0	1	1
+1	140	140	1	0	0	0	1	1	0	0
+1	141	141	1	0	0	0	1	1	0	1
+1	142	142	1	0	0	0	1	1	1	0
+1	143	143	1	0	0	0	1	1	1	1
+1	144	144	1	0	0	1	0	0	0	0
+1	145	145	1	0	0	1	0	0	0	1
+1	146	146	1	0	0	1	0	0	1	0
+1	147	147	1	0	0	1	0	0	1	1
+1	148	148	1	0	0	1	0	1	0	0
+1	149	149	1	0	0	1	0	1	0	1
+1	150	150	1	0	0	1	0	1	1	0
+1	151	151	1	0	0	1	0	1	1	1
+1	152	152	1	0	0	1	1	0	0	0
+1	153	153	1	0	0	1	1	0	0	1
+1	154	154	1	0	0	1	1	0	1	0
+1	155	155	1	0	0	1	1	0	1	1
+1	156	156	1	0	0	1	1	1	0	0
+1	157	157	1	0	0	1	1	1	0	1
+1	158	158	1	0	0	1	1	1	1	0
+1	159	159	1	0	0	1	1	1	1	1
+1	160	160	1	0	1	0	0	0	0	0
+1	161	161	1	0	1	0	0	0	0	1
+1	162	162	1	0	1	0	0	0	1	0
+1	163	163	1	0	1	0	0	0	1	1
+1	164	164	1	0	1	0	0	1	0	0
+1	165	165	1	0	1	0	0	1	0	1
+1	166	166	1	0	1	0	0	1	1	0
+1	167	167	1	0	1	0	0	1	1	1
+1	168	168	1	0	1	0	1	0	0	0
+1	169	169	1	0	1	0	1	0	0	1
+1	170	170	1	0	1	0	1	0	1	0
+1	171	171	1	0	1	0	1	0	1	1
+1	172	172	1	0	1	0	1	1	0	0
+1	173	173	1	0	1	0	1	1	0	1
+1	174	174	1	0	1	0	1	1	1	0
+1	175	175	1	0	1	0	1	1	1	1
+1	176	176	1	0	1	1	0	0	0	0
+1	177	177	1	0	1	1	0	0	0	1
+1	178	178	1	0	1	1	0	0	1	0
+1	179	179	1	0	1	1	0	0	1	1
+1	180	180	1	0	1	1	0	1	0	0
+1	181	181	1	0	1	1	0	1	0	1
+1	182	182	1	0	1	1	0	1	1	0
+1	183	183	1	0	1	1	0	1	1	1
+1	184	184	1	0	1	1	1	0	0	0
+1	185	185	1	0	1	1	1	0	0	1
+1	186	186	1	0	1	1	1	0	1	0
+1	187	187	1	0	1	1	1	0	1	1
+1	188	188	1	0	1	1	1	1	0	0
+1	189	189	1	0	1	1	1	1	0	1
+1	190	190	1	0	1	1	1	1	1	0
+1	191	191	1	0	1	1	1	1	1	1
+1	192	192	1	1	0	0	0	0	0	0
+1	193	193	1	1	0	0	0	0	0	1
+1	194	194	1	1	0	0	0	0	1	0
+1	195	195	1	1	0	0	0	0	1	1
+1	196	196	1	1	0	0	0	1	0	0
+1	197	197	1	1	0	0	0	1	0	1
+1	198	198	1	1	0	0	0	1	1	0
+1	199	199	1	1	0	0	0	1	1	1
+1	200	200	1	1	0	0	1	0	0	0
+1	201	201	1	1	0	0	1	0	0	1
+1	202	202	1	1	0	0	1	0	1	0
+1	203	203	1	1	0	0	1	0	1	1
+1	204	204	1	1	0	0	1	1	0	0
+1	205	205	1	1	0	0	1	1	0	1
+1	206	206	1	1	0	0	1	1	1	0
+1	207	207	1	1	0	0	1	1	1	1
+1	208	208	1	1	0	1	0	0	0	0
+1	209	209	1	1	0	1	0	0	0	1
+1	210	210	1	1	0	1	0	0	1	0
+1	211	211	1	1	0	1	0	0	1	1
+1	212	212	1	1	0	1	0	1	0	0
+1	213	213	1	1	0	1	0	1	0	1
+1	214	214	1	1	0	1	0	1	1	0
+1	215	215	1	1	0	1	0	1	1	1
+1	216	216	1	1	0	1	1	0	0	0
+1	217	217	1	1	0	1	1	0	0	1
+1	218	218	1	1	0	1	1	0	1	0
+1	219	219	1	1	0	1	1	0	1	1
+1	220	220	1	1	0	1	1	1	0	0
+1	221	221	1	1	0	1	1	1	0	1
+1	222	222	1	1	0	1	1	1	1	0
+1	223	223	1	1	0	1	1	1	1	1
+1	224	224	1	1	1	0	0	0	0	0
+1	225	225	1	1	1	0	0	0	0	1
+1	226	226	1	1	1	0	0	0	1	0
+1	227	227	1	1	1	0	0	0	1	1
+1	228	228	1	1	1	0	0	1	0	0
+1	229	229	1	1	1	0	0	1	0	1
+1	230	230	1	1	1	0	0	1	1	0
+1	231	231	1	1	1	0	0	1	1	1
+1	232	232	1	1	1	0	1	0	0	0
+1	233	233	1	1	1	0	1	0	0	1
+1	234	234	1	1	1	0	1	0	1	0
+1	235	235	1	1	1	0	1	0	1	1
+1	236	236	1	1	1	0	1	1	0	0
+1	237	237	1	1	1	0	1	1	0	1
+1	238	238	1	1	1	0	1	1	1	0
+1	239	239	1	1	1	0	1	1	1	1
+1	240	240	1	1	1	1	0	0	0	0
+1	241	241	1	1	1	1	0	0	0	1
+1	242	242	1	1	1	1	0	0	1	0
+1	243	243	1	1	1	1	0	0	1	1
+1	244	244	1	1	1	1	0	1	0	0
+1	245	245	1	1	1	1	0	1	0	1
+1	246	246	1	1	1	1	0	1	1	0
+1	247	247	1	1	1	1	0	1	1	1
+1	248	248	1	1	1	1	1	0	0	0
+1	249	249	1	1	1	1	1	0	0	1
+1	250	250	1	1	1	1	1	0	1	0
+1	251	251	1	1	1	1	1	0	1	1
+1	252	252	1	1	1	1	1	1	0	0
+1	253	253	1	1	1	1	1	1	0	1
+1	254	254	1	1	1	1	1	1	1	0
+1	255	255	1	1	1	1	1	1	1	1

dbms/tests/queries/0_stateless/00216_bit_test_function_family.sql

@@ -0,0 +1,115 @@
+select
+	bitTest(0, 0) = 0,
+	bitTest(1, 0) = 1,
+	bitTest(1, 1) = 0,
+	bitTest(0xff, 7) = 1;
+
+select
+	bitTestAll(0, 0) = 0,
+	bitTestAll(1, 0) = 1,
+	bitTestAll(1, 1) = 0,
+	bitTestAll(0xff, 0) = 1,
+	bitTestAll(0xff, 1) = 1,
+	bitTestAll(0xff, 2) = 1,
+	bitTestAll(0xff, 3) = 1,
+	bitTestAll(0xff, 4) = 1,
+	bitTestAll(0xff, 5) = 1,
+	bitTestAll(0xff, 6) = 1,
+	bitTestAll(0xff, 7) = 1,
+	bitTestAll(0xff, 0, 1) = 1,
+	bitTestAll(0xff, 2, 3) = 1,
+	bitTestAll(0xff, 4, 5) = 1,
+	bitTestAll(0xff, 6, 7) = 1,
+	bitTestAll(0xff, 0, 1, 2, 3) = 1,
+	bitTestAll(0xff, 4, 5, 6, 7) = 1,
+	bitTestAll(0xff, 0, 1, 2, 3, 4, 5, 6, 7) = 1,
+	bitTestAll(0x81, 0) = 1,
+	bitTestAll(0x81, 1) = 0,
+	bitTestAll(0x81, 2) = 0,
+	bitTestAll(0x81, 3) = 0,
+	bitTestAll(0x81, 4) = 0,
+	bitTestAll(0x81, 5) = 0,
+	bitTestAll(0x81, 6) = 0,
+	bitTestAll(0x81, 7) = 1,
+	bitTestAll(0x81, 0, 1) = 0,
+	bitTestAll(0x81, 2, 3) = 0,
+	bitTestAll(0x81, 4, 5) = 0,
+	bitTestAll(0x81, 6, 7) = 0,
+	bitTestAll(0x81, 0, 1, 2, 3) = 0,
+	bitTestAll(0x81, 4, 5, 6, 7) = 0,
+	bitTestAll(0x81, 0, 1, 2, 3, 4, 5, 6, 7) = 0,
+	bitTestAll(0x81, 0, 7) = 1;
+
+select
+	bitTestAny(0, 0) = 0,
+	bitTestAny(1, 0) = 1,
+	bitTestAny(1, 1) = 0,
+	bitTestAny(0xff, 0) = 1,
+	bitTestAny(0xff, 1) = 1,
+	bitTestAny(0xff, 2) = 1,
+	bitTestAny(0xff, 3) = 1,
+	bitTestAny(0xff, 4) = 1,
+	bitTestAny(0xff, 5) = 1,
+	bitTestAny(0xff, 6) = 1,
+	bitTestAny(0xff, 7) = 1,
+	bitTestAny(0xff, 0, 1) = 1,
+	bitTestAny(0xff, 2, 3) = 1,
+	bitTestAny(0xff, 4, 5) = 1,
+	bitTestAny(0xff, 6, 7) = 1,
+	bitTestAny(0xff, 0, 1, 2, 3) = 1,
+	bitTestAny(0xff, 4, 5, 6, 7) = 1,
+	bitTestAny(0xff, 0, 1, 2, 3, 4, 5, 6, 7) = 1,
+	bitTestAny(0x81, 0) = 1,
+	bitTestAny(0x81, 1) = 0,
+	bitTestAny(0x81, 2) = 0,
+	bitTestAny(0x81, 3) = 0,
+	bitTestAny(0x81, 4) = 0,
+	bitTestAny(0x81, 5) = 0,
+	bitTestAny(0x81, 6) = 0,
+	bitTestAny(0x81, 7) = 1,
+	bitTestAny(0x81, 0, 1) = 1,
+	bitTestAny(0x81, 2, 3) = 0,
+	bitTestAny(0x81, 4, 5) = 0,
+	bitTestAny(0x81, 6, 7) = 1,
+	bitTestAny(0x81, 0, 1, 2, 3) = 1,
+	bitTestAny(0x81, 4, 5, 6, 7) = 1,
+	bitTestAny(0x81, 0, 1, 2, 3, 4, 5, 6, 7) = 1;
+
+select n = n_,
+	number as n,
+	bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(b7, 1), b6), 1), b5), 1), b4), 1), b3), 1), b2), 1), b1), 1), b0) as n_,
+	bitTest(n, 7) as b7,
+	bitTest(n, 6) as b6,
+	bitTest(n, 5) as b5,
+	bitTest(n, 4) as b4,
+	bitTest(n, 3) as b3,
+	bitTest(n, 2) as b2,
+	bitTest(n, 1) as b1,
+	bitTest(n, 0) as b0
+from system.numbers limit 256;
+
+select n = n_,
+	number as n,
+	bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(b7, 1), b6), 1), b5), 1), b4), 1), b3), 1), b2), 1), b1), 1), b0) as n_,
+	bitTestAll(n, 7) as b7,
+	bitTestAll(n, 6) as b6,
+	bitTestAll(n, 5) as b5,
+	bitTestAll(n, 4) as b4,
+	bitTestAll(n, 3) as b3,
+	bitTestAll(n, 2) as b2,
+	bitTestAll(n, 1) as b1,
+	bitTestAll(n, 0) as b0
+from system.numbers limit 256;
+
+select n = n_,
+	number as n,
+	bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(b7, 1), b6), 1), b5), 1), b4), 1), b3), 1), b2), 1), b1), 1), b0) as n_,
+	bitTestAny(n, 7) as b7,
+	bitTestAny(n, 6) as b6,
+	bitTestAny(n, 5) as b5,
+	bitTestAny(n, 4) as b4,
+	bitTestAny(n, 3) as b3,
+	bitTestAny(n, 2) as b2,
+	bitTestAny(n, 1) as b1,
+	bitTestAny(n, 0) as b0
+from system.numbers limit 256;

dbms/tests/queries/0_stateless/00217_global_subquery_columns_with_same_name.reference

@@ -0,0 +1,2 @@
+42	1
+1

dbms/tests/queries/0_stateless/00217_global_subquery_columns_with_same_name.sql

@@ -0,0 +1,2 @@
+SELECT k, a FROM (SELECT 42 AS k FROM remote('127.0.0.1', system.one)) GLOBAL ALL FULL OUTER JOIN (SELECT 42 AS k, 1 AS a, a) USING k;
+SELECT 1 FROM remote('127.0.0.1', system.one) WHERE (1, 1) GLOBAL IN (SELECT 1 AS a, a);

dbms/tests/queries/0_stateless/00218_like_regexp_newline.reference

@@ -0,0 +1,11 @@
+1
+1
+1
+1
+1
+1
+1
+1
+1
+0
+0

dbms/tests/queries/0_stateless/00218_like_regexp_newline.sql

@@ -0,0 +1,11 @@
+SELECT 'abcdef' LIKE '%abc%def%';
+SELECT 'abctdef' LIKE '%abc%def%';
+SELECT 'abc\ndef' LIKE '%abc%def%';
+SELECT 'abc\ntdef' LIKE '%abc%def%';
+SELECT 'abct\ndef' LIKE '%abc%def%';
+SELECT 'abc\n\ndef' LIKE '%abc%def%';
+SELECT 'abc\n\ntdef' LIKE '%abc%def%';
+SELECT 'abc\nt\ndef' LIKE '%abc%def%';
+SELECT 'abct\n\ndef' LIKE '%abc%def%';
+SELECT 'ab\ndef' LIKE '%abc%def%';
+SELECT 'abc\nef' LIKE '%abc%def%';

dbms/tests/queries/0_stateless/00219_full_right_join_column_order.reference

@@ -0,0 +1,6 @@
+2	3000
+2	3000
+1	2000
+2	3000
+1	2000
+2	3000

dbms/tests/queries/0_stateless/00219_full_right_join_column_order.sql

@@ -0,0 +1,6 @@
+SELECT a, b FROM (SELECT 1 AS a, 2000 AS b) ANY RIGHT JOIN (SELECT 2 AS a, 3000 AS b) USING a, b;
+SELECT a, b FROM (SELECT 1 AS a, 2000 AS b) ANY RIGHT JOIN (SELECT 2 AS a, 3000 AS b) USING b, a;
+
+SELECT a, b FROM (SELECT 1 AS a, 2000 AS b) ANY RIGHT JOIN (SELECT 2 AS a, 3000 AS b UNION ALL SELECT 1 AS a, 2000 AS b) USING a, b;
+SELECT a, b FROM (SELECT 1 AS a, 2000 AS b) ANY RIGHT JOIN (SELECT 2 AS a, 3000 AS b UNION ALL SELECT 1 AS a, 2000 AS b) USING b, a;
+

dbms/tests/queries/0_stateless/00220_with_totals_in_subquery_remote_and_limit.reference

@@ -0,0 +1,3 @@
+1
+
+1

dbms/tests/queries/0_stateless/00220_with_totals_in_subquery_remote_and_limit.sql

@@ -0,0 +1 @@
+SELECT x FROM (SELECT count() AS x FROM remote('127.0.0.1', system.one) WITH TOTALS) LIMIT 1;