mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-21 17:20:50 +00:00
Merge
This commit is contained in:
commit
193113dc63
@ -15,6 +15,7 @@
|
||||
#include <DB/Columns/ColumnArray.h>
|
||||
|
||||
#include <stats/IntHash.h>
|
||||
#include <statdaemons/ext/range.hpp>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -234,9 +235,16 @@ namespace detail
|
||||
template <typename ResultType>
|
||||
void getMany(const double * levels, size_t size, ResultType * result) const
|
||||
{
|
||||
const double * levels_end = levels + size;
|
||||
const double * level = levels;
|
||||
UInt64 pos = count * *level;
|
||||
std::size_t indices[size];
|
||||
std::copy(ext::range_iterator<size_t>{}, ext::make_range_iterator(size), indices);
|
||||
std::sort(indices, indices + size, [levels] (auto i1, auto i2) {
|
||||
return levels[i1] < levels[i2];
|
||||
});
|
||||
|
||||
const auto indices_end = indices + size;
|
||||
auto index = indices;
|
||||
|
||||
UInt64 pos = count * levels[*index];
|
||||
|
||||
UInt64 accumulated = 0;
|
||||
|
||||
@ -251,15 +259,14 @@ namespace detail
|
||||
|
||||
if (i < SMALL_THRESHOLD)
|
||||
{
|
||||
*result = i;
|
||||
result[*index] = i;
|
||||
|
||||
++level;
|
||||
++result;
|
||||
++index;
|
||||
|
||||
if (level == levels_end)
|
||||
if (index == indices_end)
|
||||
return;
|
||||
|
||||
pos = count * *level;
|
||||
pos = count * levels[*index];
|
||||
}
|
||||
}
|
||||
|
||||
@ -274,24 +281,22 @@ namespace detail
|
||||
|
||||
if (i < BIG_SIZE)
|
||||
{
|
||||
*result = indexInBigToValue(i);
|
||||
result[*index] = indexInBigToValue(i);
|
||||
|
||||
++level;
|
||||
++result;
|
||||
++index;
|
||||
|
||||
if (level == levels_end)
|
||||
if (index == indices_end)
|
||||
return;
|
||||
|
||||
pos = count * *level;
|
||||
pos = count * levels[*index];
|
||||
}
|
||||
}
|
||||
|
||||
while (level < levels_end)
|
||||
while (index < indices_end)
|
||||
{
|
||||
*result = BIG_THRESHOLD;
|
||||
result[*index] = BIG_THRESHOLD;
|
||||
|
||||
++level;
|
||||
++result;
|
||||
++index;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -124,7 +124,7 @@ struct AggregateFunctionSequenceMatchData final
|
||||
/// Max number of iterations to match the pattern against a sequence, exception thrown when exceeded
|
||||
constexpr auto sequence_match_max_iterations = 1000000;
|
||||
|
||||
class AggregateFunctionSequenceMatch final : public IAggregateFunctionHelper<AggregateFunctionSequenceMatchData>
|
||||
class AggregateFunctionSequenceMatch : public IAggregateFunctionHelper<AggregateFunctionSequenceMatchData>
|
||||
{
|
||||
public:
|
||||
static bool sufficientArgs(const std::size_t arg_count) { return arg_count >= 3; }
|
||||
@ -218,7 +218,14 @@ public:
|
||||
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
|
||||
{
|
||||
const_cast<Data &>(data(place)).sort();
|
||||
static_cast<ColumnUInt8 &>(to).getData().push_back(match(place));
|
||||
|
||||
const auto & data_ref = data(place);
|
||||
|
||||
const auto events_begin = std::begin(data_ref.eventsList);
|
||||
const auto events_end = std::end(data_ref.eventsList);
|
||||
auto events_it = events_begin;
|
||||
|
||||
static_cast<ColumnUInt8 &>(to).getData().push_back(match(events_it, events_end));
|
||||
}
|
||||
|
||||
private:
|
||||
@ -233,21 +240,6 @@ private:
|
||||
TimeGreater
|
||||
};
|
||||
|
||||
static std::string to_string(const PatternActionType type)
|
||||
{
|
||||
static const std::map<PatternActionType, std::string> map{
|
||||
{ PatternActionType::SpecificEvent, "SpecificEvent" },
|
||||
{ PatternActionType::AnyEvent, "AnyEvent" },
|
||||
{ PatternActionType::KleeneStar, "KleeneStar" },
|
||||
{ PatternActionType::TimeLessOrEqual, "TimeLessOrEqual" },
|
||||
{ PatternActionType::TimeLess, "TimeLess", },
|
||||
{ PatternActionType::TimeGreaterOrEqual, "TimeGreaterOrEqual" },
|
||||
{ PatternActionType::TimeGreater, "TimeGreater" }
|
||||
};
|
||||
|
||||
return map.find(type)->second;
|
||||
}
|
||||
|
||||
struct PatternAction final
|
||||
{
|
||||
PatternActionType type;
|
||||
@ -353,18 +345,15 @@ private:
|
||||
this->actions = std::move(actions);
|
||||
}
|
||||
|
||||
bool match(const ConstAggregateDataPtr & place) const
|
||||
protected:
|
||||
template <typename T1, typename T2>
|
||||
bool match(T1 & events_it, const T2 events_end) const
|
||||
{
|
||||
const auto action_begin = std::begin(actions);
|
||||
const auto action_end = std::end(actions);
|
||||
auto action_it = action_begin;
|
||||
|
||||
const auto & data_ref = data(place);
|
||||
const auto events_begin = std::begin(data_ref.eventsList);
|
||||
const auto events_end = std::end(data_ref.eventsList);
|
||||
auto events_it = events_begin;
|
||||
|
||||
auto base_it = events_begin;
|
||||
auto base_it = events_it;
|
||||
|
||||
/// an iterator to action plus an iterator to row in events list plus timestamp at the start of sequence
|
||||
using backtrack_info = std::tuple<decltype(action_it), decltype(events_it), decltype(base_it)>;
|
||||
@ -392,11 +381,6 @@ private:
|
||||
std::size_t i = 0;
|
||||
while (action_it != action_end && events_it != events_end)
|
||||
{
|
||||
// std::cout << "start_timestamp " << base_it->first << "; ";
|
||||
// std::cout << "elapsed " << (events_it->first - base_it->first) << "; ";
|
||||
// std::cout << "action " << (action_it - action_begin) << " { " << to_string(action_it->type) << ' ' << action_it->extra << " }; ";
|
||||
// std::cout << "symbol " << (events_it - events_begin) << " { " << events_it->first << ' ' << events_it->second.to_ulong() << " }" << std::endl;
|
||||
|
||||
if (action_it->type == PatternActionType::SpecificEvent)
|
||||
{
|
||||
if (events_it->second.test(action_it->extra))
|
||||
@ -492,9 +476,40 @@ private:
|
||||
return action_it == action_end;
|
||||
}
|
||||
|
||||
private:
|
||||
std::string pattern;
|
||||
std::size_t arg_count;
|
||||
PatternActions actions;
|
||||
};
|
||||
|
||||
class AggregateFunctionSequenceCount final : public AggregateFunctionSequenceMatch
|
||||
{
|
||||
public:
|
||||
String getName() const override { return "sequenceCount"; }
|
||||
|
||||
DataTypePtr getReturnType() const override { return new DataTypeUInt64; }
|
||||
|
||||
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
|
||||
{
|
||||
const_cast<Data &>(data(place)).sort();
|
||||
static_cast<ColumnUInt64 &>(to).getData().push_back(count(place));
|
||||
}
|
||||
|
||||
private:
|
||||
UInt64 count(const ConstAggregateDataPtr & place) const
|
||||
{
|
||||
const auto & data_ref = data(place);
|
||||
|
||||
const auto events_begin = std::begin(data_ref.eventsList);
|
||||
const auto events_end = std::end(data_ref.eventsList);
|
||||
auto events_it = events_begin;
|
||||
|
||||
std::size_t count = 0;
|
||||
while (events_it != events_end && match(events_it, events_end))
|
||||
++count;
|
||||
|
||||
return count;
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -20,6 +20,7 @@
|
||||
#include <DB/Columns/ColumnString.h>
|
||||
|
||||
#include <DB/AggregateFunctions/IUnaryAggregateFunction.h>
|
||||
#include <DB/AggregateFunctions/UniqCombinedBiasData.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -94,83 +95,79 @@ struct AggregateFunctionUniqExactData<String>
|
||||
static String getName() { return "uniqExact"; }
|
||||
};
|
||||
|
||||
template <typename T, HyperLogLogMode mode>
|
||||
struct BaseUniqCombinedData
|
||||
{
|
||||
using Key = UInt64;
|
||||
using Set = CombinedCardinalityEstimator<
|
||||
Key,
|
||||
HashSet<Key, DefaultHash<Key>, HashTableGrower<> >,
|
||||
16,
|
||||
14,
|
||||
17,
|
||||
DefaultHash<Key>,
|
||||
UInt64,
|
||||
HyperLogLogBiasEstimator<UniqCombinedBiasData>,
|
||||
mode
|
||||
>;
|
||||
|
||||
Set set;
|
||||
};
|
||||
|
||||
template <HyperLogLogMode mode>
|
||||
struct BaseUniqCombinedData<String, mode>
|
||||
{
|
||||
using Key = UInt64;
|
||||
using Set = CombinedCardinalityEstimator<
|
||||
Key,
|
||||
HashSet<Key, TrivialHash, HashTableGrower<> >,
|
||||
16,
|
||||
14,
|
||||
17,
|
||||
TrivialHash,
|
||||
UInt64,
|
||||
HyperLogLogBiasEstimator<UniqCombinedBiasData>,
|
||||
mode
|
||||
>;
|
||||
|
||||
Set set;
|
||||
};
|
||||
|
||||
/// Агрегатные функции uniqCombinedRaw, uniqCombinedLinearCounting, и uniqCombinedBiasCorrected
|
||||
/// предназначены для разработки новых версий функции uniqCombined.
|
||||
/// Пользователи должны использовать только uniqCombined.
|
||||
|
||||
template <typename T>
|
||||
struct AggregateFunctionUniqCombinedRawData
|
||||
: public BaseUniqCombinedData<T, HyperLogLogMode::Raw>
|
||||
{
|
||||
static String getName() { return "uniqCombinedRaw"; }
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct AggregateFunctionUniqCombinedLinearCountingData
|
||||
: public BaseUniqCombinedData<T, HyperLogLogMode::LinearCounting>
|
||||
{
|
||||
static String getName() { return "uniqCombinedLinearCounting"; }
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct AggregateFunctionUniqCombinedBiasCorrectedData
|
||||
: public BaseUniqCombinedData<T, HyperLogLogMode::BiasCorrected>
|
||||
{
|
||||
static String getName() { return "uniqCombinedBiasCorrected"; }
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct AggregateFunctionUniqCombinedData
|
||||
: public BaseUniqCombinedData<T, HyperLogLogMode::FullFeatured>
|
||||
{
|
||||
using Key = UInt32;
|
||||
using Set = CombinedCardinalityEstimator<Key, HashSet<Key, TrivialHash, HashTableGrower<> >, 16, 14, 17, TrivialHash>;
|
||||
Set set;
|
||||
|
||||
static String getName() { return "uniqCombined"; }
|
||||
};
|
||||
|
||||
template <>
|
||||
struct AggregateFunctionUniqCombinedData<String>
|
||||
{
|
||||
using Key = UInt64;
|
||||
using Set = CombinedCardinalityEstimator<Key, HashSet<Key, TrivialHash, HashTableGrower<> >, 16, 14, 17, TrivialHash>;
|
||||
Set set;
|
||||
|
||||
static String getName() { return "uniqCombined"; }
|
||||
};
|
||||
|
||||
namespace detail
|
||||
{
|
||||
|
||||
/** Хэш-функция для uniqCombined.
|
||||
*/
|
||||
template<typename T, typename Enable = void>
|
||||
struct CombinedCardinalityTraits
|
||||
{
|
||||
static UInt32 hash(T key)
|
||||
{
|
||||
return key;
|
||||
}
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
struct CombinedCardinalityTraits<T, typename std::enable_if<std::is_same<T, Int64>::value>::type>
|
||||
{
|
||||
using U = typename std::make_unsigned<T>::type;
|
||||
|
||||
static UInt32 hash(T key)
|
||||
{
|
||||
return intHash32<0>(static_cast<U>(key));
|
||||
};
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
struct CombinedCardinalityTraits<T, typename std::enable_if<std::is_same<T, UInt64>::value>::type>
|
||||
{
|
||||
static UInt32 hash(T key)
|
||||
{
|
||||
return intHash32<0>(key);
|
||||
};
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
struct CombinedCardinalityTraits<T, typename std::enable_if<std::is_same<T, Float64>::value>::type>
|
||||
{
|
||||
static UInt32 hash(T key)
|
||||
{
|
||||
UInt64 res = 0;
|
||||
memcpy(reinterpret_cast<char *>(&res), reinterpret_cast<char *>(&key), sizeof(key));
|
||||
return intHash32<0>(res);
|
||||
}
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
struct CombinedCardinalityTraits<T, typename std::enable_if<std::is_same<T, Float32>::value>::type>
|
||||
{
|
||||
static UInt32 hash(T key)
|
||||
{
|
||||
UInt32 res = 0;
|
||||
memcpy(reinterpret_cast<char *>(&res), reinterpret_cast<char *>(&key), sizeof(key));
|
||||
return res;
|
||||
}
|
||||
};
|
||||
|
||||
/** Хэш-функция для uniq.
|
||||
*/
|
||||
template <typename T> struct AggregateFunctionUniqTraits
|
||||
@ -201,39 +198,49 @@ template <> struct AggregateFunctionUniqTraits<Float64>
|
||||
/** Структура для делегации работы по добавлению одного элемента в агрегатные функции uniq.
|
||||
* Используется для частичной специализации для добавления строк.
|
||||
*/
|
||||
template<typename T, typename Data>
|
||||
struct OneAdder
|
||||
{
|
||||
static void addOne(Data & data, const IColumn & column, size_t row_num)
|
||||
{
|
||||
data.set.insert(AggregateFunctionUniqTraits<T>::hash(static_cast<const ColumnVector<T> &>(column).getData()[row_num]));
|
||||
}
|
||||
};
|
||||
template <typename T, typename Data, typename Enable = void>
|
||||
struct OneAdder;
|
||||
|
||||
template<typename Data>
|
||||
struct OneAdder<String, Data>
|
||||
template <typename T, typename Data>
|
||||
struct OneAdder<T, Data, typename std::enable_if<
|
||||
std::is_same<Data, AggregateFunctionUniqUniquesHashSetData>::value ||
|
||||
std::is_same<Data, AggregateFunctionUniqHLL12Data<T> >::value ||
|
||||
std::is_same<Data, AggregateFunctionUniqCombinedRawData<T> >::value ||
|
||||
std::is_same<Data, AggregateFunctionUniqCombinedLinearCountingData<T> >::value ||
|
||||
std::is_same<Data, AggregateFunctionUniqCombinedBiasCorrectedData<T> >::value ||
|
||||
std::is_same<Data, AggregateFunctionUniqCombinedData<T> >::value>::type>
|
||||
{
|
||||
static void addOne(Data & data, const IColumn & column, size_t row_num)
|
||||
template <typename T2 = T>
|
||||
static void addOne(Data & data, const IColumn & column, size_t row_num,
|
||||
typename std::enable_if<!std::is_same<T2, String>::value>::type * = nullptr)
|
||||
{
|
||||
const auto & value = static_cast<const ColumnVector<T2> &>(column).getData()[row_num];
|
||||
data.set.insert(AggregateFunctionUniqTraits<T2>::hash(value));
|
||||
}
|
||||
|
||||
template <typename T2 = T>
|
||||
static void addOne(Data & data, const IColumn & column, size_t row_num,
|
||||
typename std::enable_if<std::is_same<T2, String>::value>::type * = nullptr)
|
||||
{
|
||||
/// Имейте ввиду, что вычисление приближённое.
|
||||
StringRef value = column.getDataAt(row_num);
|
||||
data.set.insert(CityHash64(value.data, value.size));
|
||||
}
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
struct OneAdder<T, AggregateFunctionUniqExactData<T> >
|
||||
template <typename T, typename Data>
|
||||
struct OneAdder<T, Data, typename std::enable_if<
|
||||
std::is_same<Data, AggregateFunctionUniqExactData<T> >::value>::type>
|
||||
{
|
||||
static void addOne(AggregateFunctionUniqExactData<T> & data, const IColumn & column, size_t row_num)
|
||||
template <typename T2 = T>
|
||||
static void addOne(Data & data, const IColumn & column, size_t row_num,
|
||||
typename std::enable_if<!std::is_same<T2, String>::value>::type * = nullptr)
|
||||
{
|
||||
data.set.insert(static_cast<const ColumnVector<T> &>(column).getData()[row_num]);
|
||||
data.set.insert(static_cast<const ColumnVector<T2> &>(column).getData()[row_num]);
|
||||
}
|
||||
};
|
||||
|
||||
template<>
|
||||
struct OneAdder<String, AggregateFunctionUniqExactData<String> >
|
||||
{
|
||||
static void addOne(AggregateFunctionUniqExactData<String> & data, const IColumn & column, size_t row_num)
|
||||
template <typename T2 = T>
|
||||
static void addOne(Data & data, const IColumn & column, size_t row_num,
|
||||
typename std::enable_if<std::is_same<T2, String>::value>::type * = nullptr)
|
||||
{
|
||||
StringRef value = column.getDataAt(row_num);
|
||||
|
||||
@ -246,26 +253,6 @@ struct OneAdder<String, AggregateFunctionUniqExactData<String> >
|
||||
}
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
struct OneAdder<T, AggregateFunctionUniqCombinedData<T> >
|
||||
{
|
||||
static void addOne(AggregateFunctionUniqCombinedData<T> & data, const IColumn & column, size_t row_num)
|
||||
{
|
||||
const auto & value = static_cast<const ColumnVector<T> &>(column).getData()[row_num];
|
||||
data.set.insert(CombinedCardinalityTraits<T>::hash(value));
|
||||
}
|
||||
};
|
||||
|
||||
template<>
|
||||
struct OneAdder<String, AggregateFunctionUniqCombinedData<String> >
|
||||
{
|
||||
static void addOne(AggregateFunctionUniqCombinedData<String> & data, const IColumn & column, size_t row_num)
|
||||
{
|
||||
StringRef value = column.getDataAt(row_num);
|
||||
data.set.insert(CityHash64(value.data, value.size));
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
39
dbms/include/DB/AggregateFunctions/UniqCombinedBiasData.h
Normal file
39
dbms/include/DB/AggregateFunctions/UniqCombinedBiasData.h
Normal file
@ -0,0 +1,39 @@
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/** Данные для HyperLogLogBiasEstimator в функции uniqCombined.
|
||||
* Схема разработки следующая:
|
||||
* 1. Собрать ClickHouse.
|
||||
* 2. Запустить скрипт src/dbms/scripts/gen-bias-data.py, который возвращает один массив для getRawEstimates()
|
||||
* и другой массив для getBiases().
|
||||
* 3. Обновить массивы raw_estimates и biases. Также обновить размер массивов в InterpolatedData.
|
||||
* 4. Собрать ClickHouse.
|
||||
* 5. Запустить скрипт src/dbms/scripts/linear-counting-threshold.py, который создаёт 3 файла:
|
||||
* - raw_graph.txt (1-й столбец: настоящее количество уникальных значений;
|
||||
* 2-й столбец: относительная погрешность в случае HyperLogLog без применения каких-либо поправок)
|
||||
* - linear_counting_graph.txt (1-й столбец: настоящее количество уникальных значений;
|
||||
* 2-й столбец: относительная погрешность в случае HyperLogLog с применением LinearCounting)
|
||||
* - bias_corrected_graph.txt (1-й столбец: настоящее количество уникальных значений;
|
||||
* 2-й столбец: относительная погрешность в случае HyperLogLog с применением поправок из алгортима HyperLogLog++)
|
||||
* 6. Сгенерить график с gnuplot на основе этих данных.
|
||||
* 7. Определить минимальное количество уникальных значений, при котором лучше исправить погрешность
|
||||
* с помощью её оценки (т.е. по алгоритму HyperLogLog++), чем применить алгоритм LinearCounting.
|
||||
* 7. Соответственно обновить константу в функции getThreshold()
|
||||
* 8. Собрать ClickHouse.
|
||||
*/
|
||||
struct UniqCombinedBiasData
|
||||
{
|
||||
using InterpolatedData = std::array<double, 178>;
|
||||
|
||||
static double getThreshold();
|
||||
/// Оценки количества уникальных значений по алгоритму HyperLogLog без применения каких-либо поправок.
|
||||
static const InterpolatedData & getRawEstimates();
|
||||
/// Соответствующие оценки погрешности.
|
||||
static const InterpolatedData & getBiases();
|
||||
};
|
||||
|
||||
}
|
@ -34,17 +34,32 @@ template
|
||||
UInt8 medium_set_power2_max,
|
||||
UInt8 K,
|
||||
typename Hash = IntHash32<Key>,
|
||||
typename HashValueType = UInt32,
|
||||
typename BiasEstimator = TrivialBiasEstimator,
|
||||
HyperLogLogMode mode = HyperLogLogMode::FullFeatured,
|
||||
typename DenominatorType = double
|
||||
>
|
||||
class CombinedCardinalityEstimator
|
||||
{
|
||||
public:
|
||||
using Self = CombinedCardinalityEstimator<Key, HashContainer, small_set_size_max, medium_set_power2_max, K, Hash, DenominatorType>;
|
||||
using Self = CombinedCardinalityEstimator
|
||||
<
|
||||
Key,
|
||||
HashContainer,
|
||||
small_set_size_max,
|
||||
medium_set_power2_max,
|
||||
K,
|
||||
Hash,
|
||||
HashValueType,
|
||||
BiasEstimator,
|
||||
mode,
|
||||
DenominatorType
|
||||
>;
|
||||
|
||||
private:
|
||||
using Small = SmallSet<Key, small_set_size_max>;
|
||||
using Medium = HashContainer;
|
||||
using Large = HyperLogLogCounter<K, Hash, DenominatorType>;
|
||||
using Large = HyperLogLogCounter<K, Hash, HashValueType, DenominatorType, BiasEstimator, mode>;
|
||||
|
||||
public:
|
||||
CombinedCardinalityEstimator()
|
||||
|
@ -21,7 +21,7 @@ class HyperLogLogWithSmallSetOptimization
|
||||
{
|
||||
private:
|
||||
using Small = SmallSet<Key, small_set_size>;
|
||||
using Large = HyperLogLogCounter<K, Hash, DenominatorType>;
|
||||
using Large = HyperLogLogCounter<K, Hash, UInt32, DenominatorType>;
|
||||
|
||||
Small small;
|
||||
Large * large = nullptr;
|
||||
|
@ -45,7 +45,7 @@
|
||||
*/
|
||||
#define DEFAULT_MERGE_BLOCK_SIZE 8192
|
||||
|
||||
#define DEFAULT_MAX_QUERY_SIZE 65536
|
||||
#define DEFAULT_MAX_QUERY_SIZE 262144
|
||||
#define SHOW_CHARS_ON_SYNTAX_ERROR 160L
|
||||
#define DEFAULT_MAX_DISTRIBUTED_CONNECTIONS 1024
|
||||
#define DEFAULT_INTERACTIVE_DELAY 100000
|
||||
|
@ -16,7 +16,12 @@ using Poco::SharedPtr;
|
||||
class LimitBlockInputStream : public IProfilingBlockInputStream
|
||||
{
|
||||
public:
|
||||
LimitBlockInputStream(BlockInputStreamPtr input_, size_t limit_, size_t offset_ = 0);
|
||||
/** Если always_read_till_end = false (по-умолчанию), то после чтения достаточного количества данных,
|
||||
* возвращает пустой блок, и это приводит к отмене выполнения запроса.
|
||||
* Если always_read_till_end = true - читает все данные до конца, но игнорирует их. Это нужно в редких случаях:
|
||||
* когда иначе, из-за отмены запроса, мы бы не получили данные для GROUP BY WITH TOTALS с удалённого сервера.
|
||||
*/
|
||||
LimitBlockInputStream(BlockInputStreamPtr input_, size_t limit_, size_t offset_, bool always_read_till_end_ = false);
|
||||
|
||||
String getName() const override { return "Limit"; }
|
||||
|
||||
@ -33,7 +38,8 @@ protected:
|
||||
private:
|
||||
size_t limit;
|
||||
size_t offset;
|
||||
size_t pos;
|
||||
size_t pos = 0;
|
||||
bool always_read_till_end;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -42,31 +42,9 @@ namespace DB
|
||||
const auto ipv4_bytes_length = 4;
|
||||
const auto ipv6_bytes_length = 16;
|
||||
|
||||
class FunctionIPv6NumToString : public IFunction
|
||||
class IPv6Format
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "IPv6NumToString";
|
||||
static IFunction * create(const Context & context) { return new FunctionIPv6NumToString; }
|
||||
|
||||
String getName() const { return name; }
|
||||
|
||||
DataTypePtr getReturnType(const DataTypes & arguments) const
|
||||
{
|
||||
if (arguments.size() != 1)
|
||||
throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
|
||||
+ toString(arguments.size()) + ", should be 1.",
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
const auto ptr = typeid_cast<const DataTypeFixedString *>(arguments[0].get());
|
||||
if (!ptr || ptr->getN() != ipv6_bytes_length)
|
||||
throw Exception("Illegal type " + arguments[0]->getName() +
|
||||
" of argument of function " + getName() +
|
||||
", expected FixedString(" + toString(ipv6_bytes_length) + ")",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return new DataTypeString;
|
||||
}
|
||||
|
||||
private:
|
||||
/// integer logarithm, return ceil(log(value, base)) (the smallest integer greater or equal than log(value, base)
|
||||
static constexpr uint32_t int_log(const uint32_t value, const uint32_t base, const bool carry = false)
|
||||
{
|
||||
@ -99,23 +77,25 @@ public:
|
||||
}
|
||||
|
||||
/// print IPv4 address as %u.%u.%u.%u
|
||||
static void ipv4_format(const unsigned char * src, char *& dst)
|
||||
static void ipv4_format(const unsigned char * src, char *& dst, UInt8 zeroed_tail_bytes_count)
|
||||
{
|
||||
constexpr auto size = sizeof(UInt32);
|
||||
const auto limit = ipv4_bytes_length - zeroed_tail_bytes_count;
|
||||
|
||||
for (const auto i : ext::range(0, size))
|
||||
for (const auto i : ext::range(0, ipv4_bytes_length))
|
||||
{
|
||||
print_integer<10, UInt8>(dst, src[i]);
|
||||
UInt8 byte = (i < limit) ? src[i] : 0;
|
||||
print_integer<10, UInt8>(dst, byte);
|
||||
|
||||
if (i != size - 1)
|
||||
if (i != ipv4_bytes_length - 1)
|
||||
*dst++ = '.';
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
/** rewritten inet_ntop6 from http://svn.apache.org/repos/asf/apr/apr/trunk/network_io/unix/inet_pton.c
|
||||
* performs significantly faster than the reference implementation due to the absence of sprintf calls,
|
||||
* bounds checking, unnecessary string copying and length calculation */
|
||||
static const void ipv6_format(const unsigned char * src, char *& dst)
|
||||
static const void apply(const unsigned char * src, char *& dst, UInt8 zeroed_tail_bytes_count = 0)
|
||||
{
|
||||
struct { int base, len; } best{-1}, cur{-1};
|
||||
std::array<uint16_t, ipv6_bytes_length / sizeof(uint16_t)> words{};
|
||||
@ -123,7 +103,7 @@ public:
|
||||
/** Preprocess:
|
||||
* Copy the input (bytewise) array into a wordwise array.
|
||||
* Find the longest run of 0x00's in src[] for :: shorthanding. */
|
||||
for (const auto i : ext::range(0, ipv6_bytes_length))
|
||||
for (const auto i : ext::range(0, ipv6_bytes_length - zeroed_tail_bytes_count))
|
||||
words[i / 2] |= src[i] << ((1 - (i % 2)) << 3);
|
||||
|
||||
for (const auto i : ext::range(0, words.size()))
|
||||
@ -172,7 +152,7 @@ public:
|
||||
/// Is this address an encapsulated IPv4?
|
||||
if (i == 6 && best.base == 0 && (best.len == 6 || (best.len == 5 && words[5] == 0xffffu)))
|
||||
{
|
||||
ipv4_format(src + 12, dst);
|
||||
ipv4_format(src + 12, dst, std::min(zeroed_tail_bytes_count, static_cast<UInt8>(ipv4_bytes_length)));
|
||||
break;
|
||||
}
|
||||
|
||||
@ -185,6 +165,32 @@ public:
|
||||
|
||||
*dst++ = '\0';
|
||||
}
|
||||
};
|
||||
|
||||
class FunctionIPv6NumToString : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "IPv6NumToString";
|
||||
static IFunction * create(const Context & context) { return new FunctionIPv6NumToString; }
|
||||
|
||||
String getName() const { return name; }
|
||||
|
||||
DataTypePtr getReturnType(const DataTypes & arguments) const
|
||||
{
|
||||
if (arguments.size() != 1)
|
||||
throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
|
||||
+ toString(arguments.size()) + ", should be 1.",
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
const auto ptr = typeid_cast<const DataTypeFixedString *>(arguments[0].get());
|
||||
if (!ptr || ptr->getN() != ipv6_bytes_length)
|
||||
throw Exception("Illegal type " + arguments[0]->getName() +
|
||||
" of argument of function " + getName() +
|
||||
", expected FixedString(" + toString(ipv6_bytes_length) + ")",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return new DataTypeString;
|
||||
}
|
||||
|
||||
void execute(Block & block, const ColumnNumbers & arguments, const size_t result)
|
||||
{
|
||||
@ -216,7 +222,7 @@ public:
|
||||
|
||||
for (size_t offset = 0, i = 0; offset < vec_in.size(); offset += ipv6_bytes_length, ++i)
|
||||
{
|
||||
ipv6_format(&vec_in[offset], pos);
|
||||
IPv6Format::apply(&vec_in[offset], pos);
|
||||
offsets_res[i] = pos - begin;
|
||||
}
|
||||
|
||||
@ -236,7 +242,7 @@ public:
|
||||
|
||||
char buf[INET6_ADDRSTRLEN];
|
||||
char * dst = buf;
|
||||
ipv6_format(reinterpret_cast<const unsigned char *>(data_in.data()), dst);
|
||||
IPv6Format::apply(reinterpret_cast<const unsigned char *>(data_in.data()), dst);
|
||||
|
||||
block.getByPosition(result).column = new ColumnConstString{col_in->size(), buf};
|
||||
}
|
||||
@ -247,6 +253,172 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
class FunctionCutIPv6 : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "cutIPv6";
|
||||
static IFunction * create(const Context & context) { return new FunctionCutIPv6; }
|
||||
|
||||
String getName() const { return name; }
|
||||
|
||||
DataTypePtr getReturnType(const DataTypes & arguments) const
|
||||
{
|
||||
if (arguments.size() != 3)
|
||||
throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
|
||||
+ toString(arguments.size()) + ", should be 3.",
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
const auto ptr = typeid_cast<const DataTypeFixedString *>(arguments[0].get());
|
||||
if (!ptr || ptr->getN() != ipv6_bytes_length)
|
||||
throw Exception("Illegal type " + arguments[0]->getName() +
|
||||
" of argument 1 of function " + getName() +
|
||||
", expected FixedString(" + toString(ipv6_bytes_length) + ")",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
if (!typeid_cast<const DataTypeUInt8 *>(arguments[1].get()))
|
||||
throw Exception("Illegal type " + arguments[1]->getName() +
|
||||
" of argument 2 of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
if (!typeid_cast<const DataTypeUInt8 *>(arguments[2].get()))
|
||||
throw Exception("Illegal type " + arguments[2]->getName() +
|
||||
" of argument 3 of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return new DataTypeString;
|
||||
}
|
||||
|
||||
void execute(Block & block, const ColumnNumbers & arguments, const size_t result)
|
||||
{
|
||||
const auto & col_name_type = block.getByPosition(arguments[0]);
|
||||
const ColumnPtr & column = col_name_type.column;
|
||||
|
||||
const auto & col_ipv6_zeroed_tail_bytes_type = block.getByPosition(arguments[1]);
|
||||
const auto & col_ipv6_zeroed_tail_bytes = col_ipv6_zeroed_tail_bytes_type.column;
|
||||
const auto & col_ipv4_zeroed_tail_bytes_type = block.getByPosition(arguments[2]);
|
||||
const auto & col_ipv4_zeroed_tail_bytes = col_ipv4_zeroed_tail_bytes_type.column;
|
||||
|
||||
if (const auto col_in = typeid_cast<const ColumnFixedString *>(column.get()))
|
||||
{
|
||||
if (col_in->getN() != ipv6_bytes_length)
|
||||
throw Exception("Illegal type " + col_name_type.type->getName() +
|
||||
" of column " + col_in->getName() +
|
||||
" argument of function " + getName() +
|
||||
", expected FixedString(" + toString(ipv6_bytes_length) + ")",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
const auto ipv6_zeroed_tail_bytes = typeid_cast<const ColumnConst<UInt8> *>(col_ipv6_zeroed_tail_bytes.get());
|
||||
if (!ipv6_zeroed_tail_bytes)
|
||||
throw Exception("Illegal type " + col_ipv6_zeroed_tail_bytes_type.type->getName() +
|
||||
" of argument 2 of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
UInt8 ipv6_zeroed_tail_bytes_count = ipv6_zeroed_tail_bytes->getData();
|
||||
if (ipv6_zeroed_tail_bytes_count > ipv6_bytes_length)
|
||||
throw Exception("Illegal value for argument 2 " + col_ipv6_zeroed_tail_bytes_type.type->getName() +
|
||||
" of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
const auto ipv4_zeroed_tail_bytes = typeid_cast<const ColumnConst<UInt8> *>(col_ipv4_zeroed_tail_bytes.get());
|
||||
if (!ipv4_zeroed_tail_bytes)
|
||||
throw Exception("Illegal type " + col_ipv4_zeroed_tail_bytes_type.type->getName() +
|
||||
" of argument 3 of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
UInt8 ipv4_zeroed_tail_bytes_count = ipv4_zeroed_tail_bytes->getData();
|
||||
if (ipv4_zeroed_tail_bytes_count > ipv6_bytes_length)
|
||||
throw Exception("Illegal value for argument 3 " + col_ipv4_zeroed_tail_bytes_type.type->getName() +
|
||||
" of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
const auto size = col_in->size();
|
||||
const auto & vec_in = col_in->getChars();
|
||||
|
||||
auto col_res = new ColumnString;
|
||||
block.getByPosition(result).column = col_res;
|
||||
|
||||
ColumnString::Chars_t & vec_res = col_res->getChars();
|
||||
ColumnString::Offsets_t & offsets_res = col_res->getOffsets();
|
||||
vec_res.resize(size * INET6_ADDRSTRLEN);
|
||||
offsets_res.resize(size);
|
||||
|
||||
auto begin = reinterpret_cast<char *>(&vec_res[0]);
|
||||
auto pos = begin;
|
||||
|
||||
for (size_t offset = 0, i = 0; offset < vec_in.size(); offset += ipv6_bytes_length, ++i)
|
||||
{
|
||||
const auto address = &vec_in[offset];
|
||||
UInt8 zeroed_tail_bytes_count = isIPv4Mapped(address) ? ipv4_zeroed_tail_bytes_count : ipv6_zeroed_tail_bytes_count;
|
||||
cutAddress(address, pos, zeroed_tail_bytes_count);
|
||||
offsets_res[i] = pos - begin;
|
||||
}
|
||||
|
||||
vec_res.resize(pos - begin);
|
||||
}
|
||||
else if (const auto col_in = typeid_cast<const ColumnConst<String> *>(column.get()))
|
||||
{
|
||||
const auto data_type_fixed_string = typeid_cast<const DataTypeFixedString *>(col_in->getDataType().get());
|
||||
if (!data_type_fixed_string || data_type_fixed_string->getN() != ipv6_bytes_length)
|
||||
throw Exception("Illegal type " + col_name_type.type->getName() +
|
||||
" of column " + col_in->getName() +
|
||||
" argument of function " + getName() +
|
||||
", expected FixedString(" + toString(ipv6_bytes_length) + ")",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
const auto ipv6_zeroed_tail_bytes = typeid_cast<const ColumnConst<UInt8> *>(col_ipv6_zeroed_tail_bytes.get());
|
||||
if (!ipv6_zeroed_tail_bytes)
|
||||
throw Exception("Illegal type " + col_ipv6_zeroed_tail_bytes_type.type->getName() +
|
||||
" of argument 2 of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
UInt8 ipv6_zeroed_tail_bytes_count = ipv6_zeroed_tail_bytes->getData();
|
||||
if (ipv6_zeroed_tail_bytes_count > ipv6_bytes_length)
|
||||
throw Exception("Illegal value for argument 2 " + col_ipv6_zeroed_tail_bytes_type.type->getName() +
|
||||
" of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
const auto ipv4_zeroed_tail_bytes = typeid_cast<const ColumnConst<UInt8> *>(col_ipv4_zeroed_tail_bytes.get());
|
||||
if (!ipv4_zeroed_tail_bytes)
|
||||
throw Exception("Illegal type " + col_ipv4_zeroed_tail_bytes_type.type->getName() +
|
||||
" of argument 3 of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
UInt8 ipv4_zeroed_tail_bytes_count = ipv4_zeroed_tail_bytes->getData();
|
||||
if (ipv4_zeroed_tail_bytes_count > ipv6_bytes_length)
|
||||
throw Exception("Illegal value for argument 3 " + col_ipv6_zeroed_tail_bytes_type.type->getName() +
|
||||
" of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
const auto & data_in = col_in->getData();
|
||||
|
||||
char buf[INET6_ADDRSTRLEN];
|
||||
char * dst = buf;
|
||||
|
||||
const auto address = reinterpret_cast<const unsigned char *>(data_in.data());
|
||||
UInt8 zeroed_tail_bytes_count = isIPv4Mapped(address) ? ipv4_zeroed_tail_bytes_count : ipv6_zeroed_tail_bytes_count;
|
||||
cutAddress(address, dst, zeroed_tail_bytes_count);
|
||||
|
||||
block.getByPosition(result).column = new ColumnConstString{col_in->size(), buf};
|
||||
}
|
||||
else
|
||||
throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName()
|
||||
+ " of argument of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
|
||||
private:
|
||||
bool isIPv4Mapped(const unsigned char * address) const
|
||||
{
|
||||
return (*reinterpret_cast<const UInt64 *>(&address[0]) == 0) &&
|
||||
((*reinterpret_cast<const UInt64 *>(&address[8]) & 0x00000000FFFFFFFFull) == 0x00000000FFFF0000ull);
|
||||
}
|
||||
|
||||
void cutAddress(const unsigned char * address, char *& dst, UInt8 zeroed_tail_bytes_count)
|
||||
{
|
||||
IPv6Format::apply(address, dst, zeroed_tail_bytes_count);
|
||||
}
|
||||
};
|
||||
|
||||
class FunctionIPv6StringToNum : public IFunction
|
||||
{
|
||||
public:
|
||||
@ -767,6 +939,69 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
class FunctionIPv4ToIPv6 : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "IPv4ToIPv6";
|
||||
static IFunction * create(const Context & context) { return new FunctionIPv4ToIPv6; }
|
||||
|
||||
String getName() const { return name; }
|
||||
|
||||
DataTypePtr getReturnType(const DataTypes & arguments) const
|
||||
{
|
||||
if (arguments.size() != 1)
|
||||
throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
|
||||
+ toString(arguments.size()) + ", should be 1.",
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
if (typeid_cast<const DataTypeUInt32 *>(arguments[0].get()) == nullptr)
|
||||
throw Exception("Illegal type " + arguments[0]->getName() +
|
||||
" of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return new DataTypeFixedString{16};
|
||||
}
|
||||
|
||||
void execute(Block & block, const ColumnNumbers & arguments, const size_t result)
|
||||
{
|
||||
const auto & col_name_type = block.getByPosition(arguments[0]);
|
||||
const ColumnPtr & column = col_name_type.column;
|
||||
|
||||
if (const auto col_in = typeid_cast<const ColumnVector<UInt32> *>(column.get()))
|
||||
{
|
||||
const auto col_res = new ColumnFixedString{ipv6_bytes_length};
|
||||
block.getByPosition(result).column = col_res;
|
||||
|
||||
auto & vec_res = col_res->getChars();
|
||||
vec_res.resize(col_in->size() * ipv6_bytes_length);
|
||||
|
||||
const auto & vec_in = col_in->getData();
|
||||
|
||||
for (size_t out_offset = 0, i = 0; out_offset < vec_res.size(); out_offset += ipv6_bytes_length, ++i)
|
||||
mapIPv4ToIPv6(vec_in[i], &vec_res[out_offset]);
|
||||
}
|
||||
else if (const auto col_in = typeid_cast<const ColumnConst<UInt32> *>(column.get()))
|
||||
{
|
||||
std::string buf;
|
||||
buf.resize(ipv6_bytes_length);
|
||||
mapIPv4ToIPv6(col_in->getData(), reinterpret_cast<unsigned char *>(&buf[0]));
|
||||
|
||||
ColumnConstString * col_res = new ColumnConstString(ipv6_bytes_length, buf,
|
||||
new DataTypeFixedString{ipv6_bytes_length});
|
||||
block.getByPosition(result).column = col_res;
|
||||
}
|
||||
else
|
||||
throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName()
|
||||
+ " of argument of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
|
||||
private:
|
||||
void mapIPv4ToIPv6(UInt32 in, unsigned char * buf) const
|
||||
{
|
||||
*reinterpret_cast<UInt64 *>(&buf[0]) = 0;
|
||||
*reinterpret_cast<UInt64 *>(&buf[8]) = 0x00000000FFFF0000ull | (static_cast<UInt64>(ntohl(in)) << 32);
|
||||
}
|
||||
};
|
||||
|
||||
class FunctionHex : public IFunction
|
||||
{
|
||||
@ -1447,6 +1682,7 @@ public:
|
||||
};
|
||||
}
|
||||
|
||||
private:
|
||||
template <typename T> bool execute(
|
||||
Block & block, const ColumnNumbers & arguments, const size_t result,
|
||||
const IColumn * const value_col_untyped)
|
||||
@ -1564,4 +1800,219 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Impl>
|
||||
struct FunctionBitTestMany : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = Impl::name;
|
||||
static IFunction * create(const Context &) { return new FunctionBitTestMany; }
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
DataTypePtr getReturnType(const DataTypes & arguments) const override
|
||||
{
|
||||
if (arguments.size() < 2)
|
||||
throw Exception{
|
||||
"Number of arguments for function " + getName() + " doesn't match: passed "
|
||||
+ toString(arguments.size()) + ", should be at least 2.",
|
||||
ErrorCodes::TOO_LESS_ARGUMENTS_FOR_FUNCTION
|
||||
};
|
||||
|
||||
const auto first_arg = arguments.front().get();
|
||||
if (!typeid_cast<const DataTypeUInt8 *>(first_arg) && !typeid_cast<const DataTypeUInt16 *>(first_arg) &&
|
||||
!typeid_cast<const DataTypeUInt32 *>(first_arg) && !typeid_cast<const DataTypeUInt64 *>(first_arg) &&
|
||||
!typeid_cast<const DataTypeInt8 *>(first_arg) && !typeid_cast<const DataTypeInt16 *>(first_arg) &&
|
||||
!typeid_cast<const DataTypeInt32 *>(first_arg) && !typeid_cast<const DataTypeInt64 *>(first_arg))
|
||||
throw Exception{
|
||||
"Illegal type " + first_arg->getName() + " of first argument of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT
|
||||
};
|
||||
|
||||
|
||||
for (const auto i : ext::range(1, arguments.size()))
|
||||
{
|
||||
const auto pos_arg = arguments[i].get();
|
||||
|
||||
if (!typeid_cast<const DataTypeUInt8 *>(pos_arg) && !typeid_cast<const DataTypeUInt16 *>(pos_arg) &&
|
||||
!typeid_cast<const DataTypeUInt32 *>(pos_arg) && !typeid_cast<const DataTypeUInt64 *>(pos_arg))
|
||||
throw Exception{
|
||||
"Illegal type " + pos_arg->getName() + " of " + toString(i) + " argument of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT
|
||||
};
|
||||
}
|
||||
|
||||
return new DataTypeUInt8;
|
||||
}
|
||||
|
||||
void execute(Block & block, const ColumnNumbers & arguments, const size_t result) override
|
||||
{
|
||||
const auto value_col = block.getByPosition(arguments.front()).column.get();
|
||||
|
||||
if (!execute<UInt8>(block, arguments, result, value_col) &&
|
||||
!execute<UInt16>(block, arguments, result, value_col) &&
|
||||
!execute<UInt32>(block, arguments, result, value_col) &&
|
||||
!execute<UInt64>(block, arguments, result, value_col) &&
|
||||
!execute<Int8>(block, arguments, result, value_col) &&
|
||||
!execute<Int16>(block, arguments, result, value_col) &&
|
||||
!execute<Int32>(block, arguments, result, value_col) &&
|
||||
!execute<Int64>(block, arguments, result, value_col))
|
||||
throw Exception{
|
||||
"Illegal column " + value_col->getName() + " of argument of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_COLUMN
|
||||
};
|
||||
}
|
||||
|
||||
private:
|
||||
template <typename T> bool execute(
|
||||
Block & block, const ColumnNumbers & arguments, const size_t result,
|
||||
const IColumn * const value_col_untyped)
|
||||
{
|
||||
if (const auto value_col = typeid_cast<const ColumnVector<T> *>(value_col_untyped))
|
||||
{
|
||||
const auto size = value_col->size();
|
||||
bool is_const;
|
||||
const auto mask = createConstMask<T>(size, block, arguments, is_const);
|
||||
const auto & val = value_col->getData();
|
||||
|
||||
const auto out_col = new ColumnVector<UInt8>(size);
|
||||
ColumnPtr out_col_ptr{out_col};
|
||||
block.getByPosition(result).column = out_col_ptr;
|
||||
|
||||
auto & out = out_col->getData();
|
||||
|
||||
if (is_const)
|
||||
{
|
||||
for (const auto i : ext::range(0, size))
|
||||
out[i] = Impl::combine(val[i], mask);
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto mask = createMask<T>(size, block, arguments);
|
||||
|
||||
for (const auto i : ext::range(0, size))
|
||||
out[i] = Impl::combine(val[i], mask[i]);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
else if (const auto value_col = typeid_cast<const ColumnConst<T> *>(value_col_untyped))
|
||||
{
|
||||
const auto size = value_col->size();
|
||||
bool is_const;
|
||||
const auto mask = createConstMask<T>(size, block, arguments, is_const);
|
||||
const auto val = value_col->getData();
|
||||
|
||||
if (is_const)
|
||||
{
|
||||
block.getByPosition(result).column = new ColumnConst<UInt8>{
|
||||
size, Impl::combine(val, mask)
|
||||
};
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto mask = createMask<T>(size, block, arguments);
|
||||
const auto out_col = new ColumnVector<UInt8>(size);
|
||||
ColumnPtr out_col_ptr{out_col};
|
||||
block.getByPosition(result).column = out_col_ptr;
|
||||
|
||||
auto & out = out_col->getData();
|
||||
|
||||
for (const auto i : ext::range(0, size))
|
||||
out[i] = Impl::combine(val, mask[i]);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
template <typename ValueType>
|
||||
ValueType createConstMask(const std::size_t size, const Block & block, const ColumnNumbers & arguments, bool & is_const)
|
||||
{
|
||||
is_const = true;
|
||||
ValueType mask{};
|
||||
|
||||
for (const auto i : ext::range(1, arguments.size()))
|
||||
{
|
||||
const auto pos_col = block.getByPosition(arguments[i]).column.get();
|
||||
|
||||
if (pos_col->isConst())
|
||||
{
|
||||
const auto pos = static_cast<const ColumnConst<ValueType> *>(pos_col)->getData();
|
||||
mask = mask | 1 << pos;
|
||||
}
|
||||
else
|
||||
{
|
||||
is_const = false;
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
return mask;
|
||||
}
|
||||
|
||||
template <typename ValueType>
|
||||
PODArray<ValueType> createMask(const std::size_t size, const Block & block, const ColumnNumbers & arguments)
|
||||
{
|
||||
PODArray<ValueType> mask(size, ValueType{});
|
||||
|
||||
for (const auto i : ext::range(1, arguments.size()))
|
||||
{
|
||||
const auto pos_col = block.getByPosition(arguments[i]).column.get();
|
||||
|
||||
if (!addToMaskImpl<UInt8>(mask, pos_col) && !addToMaskImpl<UInt16>(mask, pos_col) &&
|
||||
!addToMaskImpl<UInt32>(mask, pos_col) && !addToMaskImpl<UInt64>(mask, pos_col))
|
||||
throw Exception{
|
||||
"Illegal column " + pos_col->getName() + " of argument of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_COLUMN
|
||||
};
|
||||
}
|
||||
|
||||
return mask;
|
||||
}
|
||||
|
||||
template <typename PosType, typename ValueType>
|
||||
bool addToMaskImpl(PODArray<ValueType> & mask, const IColumn * const pos_col_untyped)
|
||||
{
|
||||
if (const auto pos_col = typeid_cast<const ColumnVector<PosType> *>(pos_col_untyped))
|
||||
{
|
||||
const auto & pos = pos_col->getData();
|
||||
|
||||
for (const auto i : ext::range(0, mask.size()))
|
||||
mask[i] = mask[i] | (1 << pos[i]);
|
||||
|
||||
return true;
|
||||
}
|
||||
else if (const auto pos_col = typeid_cast<const ColumnConst<PosType> *>(pos_col_untyped))
|
||||
{
|
||||
const auto & pos = pos_col->getData();
|
||||
const auto new_mask = 1 << pos;
|
||||
|
||||
for (const auto i : ext::range(0, mask.size()))
|
||||
mask[i] = mask[i] | new_mask;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
struct BitTestAnyImpl
|
||||
{
|
||||
static constexpr auto name = "bitTestAny";
|
||||
template <typename T> static UInt8 combine(const T val, const T mask) { return (val & mask) != 0; }
|
||||
};
|
||||
|
||||
struct BitTestAllImpl
|
||||
{
|
||||
static constexpr auto name = "bitTestAll";
|
||||
template <typename T> static UInt8 combine(const T val, const T mask) { return (val & mask) == mask; }
|
||||
};
|
||||
|
||||
using FunctionBitTestAny = FunctionBitTestMany<BitTestAnyImpl>;
|
||||
using FunctionBitTestAll = FunctionBitTestMany<BitTestAllImpl>;
|
||||
|
||||
|
||||
}
|
||||
|
@ -323,8 +323,13 @@ namespace Regexps
|
||||
if (known_regexps.end() == it)
|
||||
it = known_regexps.emplace(pattern, std::make_unique<Holder>()).first;
|
||||
|
||||
return it->second->get([&pattern] {
|
||||
return new Regexp{createRegexp<like>(pattern, no_capture ? OptimizedRegularExpression::RE_NO_CAPTURE : 0)};
|
||||
return it->second->get([&pattern]
|
||||
{
|
||||
int flags = OptimizedRegularExpression::RE_DOT_NL;
|
||||
if (no_capture)
|
||||
flags |= OptimizedRegularExpression::RE_NO_CAPTURE;
|
||||
|
||||
return new Regexp{createRegexp<like>(pattern, flags)};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
@ -59,22 +59,27 @@ struct Memory : boost::noncopyable, Allocator
|
||||
|
||||
void resize(size_t new_size)
|
||||
{
|
||||
if (new_size < m_capacity)
|
||||
if (0 == m_capacity)
|
||||
{
|
||||
m_size = m_capacity = new_size;
|
||||
alloc();
|
||||
}
|
||||
else if (new_size < m_capacity)
|
||||
{
|
||||
m_size = new_size;
|
||||
return;
|
||||
}
|
||||
else
|
||||
{
|
||||
new_size = align(new_size);
|
||||
new_size = align(new_size, alignment);
|
||||
/// @todo pointer to void can be converted to pointer to any type with static_cast by ISO C++, reinterpret_cast has no advantages
|
||||
m_data = reinterpret_cast<char *>(Allocator::realloc(m_data, m_capacity, new_size, alignment));
|
||||
m_capacity = new_size;
|
||||
m_size = m_capacity;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
size_t align(size_t value) const
|
||||
static size_t align(const size_t value, const size_t alignment)
|
||||
{
|
||||
if (!alignment)
|
||||
return value;
|
||||
@ -82,6 +87,7 @@ private:
|
||||
return (value + alignment - 1) / alignment * alignment;
|
||||
}
|
||||
|
||||
private:
|
||||
void alloc()
|
||||
{
|
||||
if (!m_capacity)
|
||||
@ -93,7 +99,8 @@ private:
|
||||
ProfileEvents::increment(ProfileEvents::IOBufferAllocs);
|
||||
ProfileEvents::increment(ProfileEvents::IOBufferAllocBytes, m_capacity);
|
||||
|
||||
size_t new_capacity = align(m_capacity);
|
||||
size_t new_capacity = align(m_capacity, alignment);
|
||||
/// @todo pointer to void can be converted to pointer to any type with static_cast by ISO C++, reinterpret_cast has no advantages
|
||||
m_data = reinterpret_cast<char *>(Allocator::alloc(new_capacity, alignment));
|
||||
m_capacity = new_capacity;
|
||||
m_size = m_capacity;
|
||||
@ -104,6 +111,7 @@ private:
|
||||
if (!m_data)
|
||||
return;
|
||||
|
||||
/// @todo pointer to any type can be implicitly converted to pointer to void, no cast required
|
||||
Allocator::free(reinterpret_cast<void *>(m_data), m_capacity);
|
||||
m_data = nullptr; /// Чтобы избежать double free, если последующий вызов alloc кинет исключение.
|
||||
}
|
||||
|
@ -82,10 +82,11 @@ private:
|
||||
}
|
||||
|
||||
public:
|
||||
CachedCompressedReadBuffer(const std::string & path_, UncompressedCache * cache_, size_t estimated_size_,
|
||||
size_t aio_threshold_, size_t buf_size_ = DBMS_DEFAULT_BUFFER_SIZE)
|
||||
: ReadBuffer(nullptr, 0), path(path_), cache(cache_), buf_size(buf_size_),
|
||||
estimated_size(estimated_size_), aio_threshold(aio_threshold_), file_pos(0)
|
||||
CachedCompressedReadBuffer(
|
||||
const std::string & path_, UncompressedCache * cache_, size_t estimated_size_, size_t aio_threshold_,
|
||||
size_t buf_size_ = DBMS_DEFAULT_BUFFER_SIZE)
|
||||
: ReadBuffer(nullptr, 0), path(path_), cache(cache_), buf_size(buf_size_), estimated_size(estimated_size_),
|
||||
aio_threshold(aio_threshold_), file_pos(0)
|
||||
{
|
||||
}
|
||||
|
||||
|
@ -42,7 +42,8 @@ private:
|
||||
}
|
||||
|
||||
public:
|
||||
CompressedReadBufferFromFile(const std::string & path, size_t estimated_size, size_t aio_threshold, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE)
|
||||
CompressedReadBufferFromFile(
|
||||
const std::string & path, size_t estimated_size, size_t aio_threshold, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE)
|
||||
: BufferWithOwnMemory<ReadBuffer>(0),
|
||||
p_file_in(createReadBufferFromFileBase(path, estimated_size, aio_threshold, buf_size)),
|
||||
file_in(*p_file_in)
|
||||
|
@ -163,8 +163,8 @@ private:
|
||||
* - в "левой" таблице, он будет доступен по имени expr(x), так как ещё не было выполнено действие Project.
|
||||
* Надо запомнить оба этих варианта.
|
||||
*/
|
||||
NameSet join_key_names_left_set;
|
||||
NameSet join_key_names_right_set;
|
||||
Names join_key_names_left;
|
||||
Names join_key_names_right;
|
||||
|
||||
NamesAndTypesList columns_added_by_join;
|
||||
|
||||
|
@ -119,6 +119,9 @@ struct Settings
|
||||
* (Чтобы большие запросы не вымывали кэш.) */ \
|
||||
M(SettingUInt64, merge_tree_max_rows_to_use_cache, (1024 * 1024)) \
|
||||
\
|
||||
/** Распределять чтение из MergeTree по потокам равномерно, обеспечивая стабильное среднее время исполнения каждого потока в пределах одного чтения. */ \
|
||||
M(SettingBool, merge_tree_uniform_read_distribution, false) \
|
||||
\
|
||||
/** Минимальная длина выражения expr = x1 OR ... expr = xN для оптимизации */ \
|
||||
M(SettingUInt64, optimize_min_equality_disjunction_chain_length, 3) \
|
||||
\
|
||||
|
24
dbms/include/DB/Storages/MergeTree/MarkRange.h
Normal file
24
dbms/include/DB/Storages/MergeTree/MarkRange.h
Normal file
@ -0,0 +1,24 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
|
||||
/** Пара засечек, определяющая диапазон строк в куске. Именно, диапазон имеет вид [begin * index_granularity, end * index_granularity).
|
||||
*/
|
||||
struct MarkRange
|
||||
{
|
||||
std::size_t begin;
|
||||
std::size_t end;
|
||||
|
||||
MarkRange() = default;
|
||||
MarkRange(const std::size_t begin, const std::size_t end) : begin{begin}, end{end} {}
|
||||
};
|
||||
|
||||
using MarkRanges = std::vector<MarkRange>;
|
||||
|
||||
|
||||
}
|
@ -2,6 +2,7 @@
|
||||
|
||||
#include <DB/Storages/MergeTree/MergeTreeData.h>
|
||||
#include <DB/Storages/MergeTree/MergeTreeReader.h>
|
||||
#include <DB/Storages/MergeTree/RangesInDataPart.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -33,22 +34,6 @@ private:
|
||||
|
||||
Logger * log;
|
||||
|
||||
struct RangesInDataPart
|
||||
{
|
||||
MergeTreeData::DataPartPtr data_part;
|
||||
size_t part_index_in_query;
|
||||
MarkRanges ranges;
|
||||
|
||||
RangesInDataPart() {}
|
||||
|
||||
RangesInDataPart(MergeTreeData::DataPartPtr data_part_, size_t part_index_in_query_)
|
||||
: data_part(data_part_), part_index_in_query(part_index_in_query_)
|
||||
{
|
||||
}
|
||||
};
|
||||
|
||||
typedef std::vector<RangesInDataPart> RangesInDataParts;
|
||||
|
||||
BlockInputStreams spreadMarkRangesAmongThreads(
|
||||
RangesInDataParts parts,
|
||||
size_t threads,
|
||||
|
435
dbms/include/DB/Storages/MergeTree/MergeTreeReadPool.h
Normal file
435
dbms/include/DB/Storages/MergeTree/MergeTreeReadPool.h
Normal file
@ -0,0 +1,435 @@
|
||||
#pragma once
|
||||
|
||||
#include <DB/Core/NamesAndTypes.h>
|
||||
#include <DB/Storages/MergeTree/RangesInDataPart.h>
|
||||
#include <statdaemons/ext/range.hpp>
|
||||
#include <mutex>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
|
||||
struct MergeTreeReadTask
|
||||
{
|
||||
MergeTreeData::DataPartPtr data_part;
|
||||
MarkRanges mark_ranges;
|
||||
std::size_t part_index_in_query;
|
||||
const Names & ordered_names;
|
||||
const NameSet & column_name_set;
|
||||
const NamesAndTypesList & columns;
|
||||
const NamesAndTypesList & pre_columns;
|
||||
const bool remove_prewhere_column;
|
||||
const bool should_reorder;
|
||||
|
||||
MergeTreeReadTask(
|
||||
const MergeTreeData::DataPartPtr & data_part, const MarkRanges & ranges, const std::size_t part_index_in_query,
|
||||
const Names & ordered_names, const NameSet & column_name_set, const NamesAndTypesList & columns,
|
||||
const NamesAndTypesList & pre_columns, const bool remove_prewhere_column, const bool should_reorder)
|
||||
: data_part{data_part}, mark_ranges{ranges}, part_index_in_query{part_index_in_query},
|
||||
ordered_names{ordered_names}, column_name_set{column_name_set}, columns{columns}, pre_columns{pre_columns},
|
||||
remove_prewhere_column{remove_prewhere_column}, should_reorder{should_reorder}
|
||||
{}
|
||||
};
|
||||
|
||||
using MergeTreeReadTaskPtr = std::unique_ptr<MergeTreeReadTask>;
|
||||
|
||||
class MergeTreeReadPool
|
||||
{
|
||||
public:
|
||||
MergeTreeReadPool(
|
||||
const std::size_t threads, const std::size_t sum_marks, const std::size_t min_marks_for_concurrent_read,
|
||||
RangesInDataParts parts, MergeTreeData & data, const ExpressionActionsPtr & prewhere_actions,
|
||||
const String & prewhere_column_name, const bool check_columns, const Names & column_names,
|
||||
const bool do_not_steal_tasks = false)
|
||||
: data{data}, column_names{column_names}, do_not_steal_tasks{do_not_steal_tasks}
|
||||
{
|
||||
const auto per_part_sum_marks = fillPerPartInfo(parts, prewhere_actions, prewhere_column_name, check_columns);
|
||||
fillPerThreadInfo(threads, sum_marks, per_part_sum_marks, parts, min_marks_for_concurrent_read);
|
||||
}
|
||||
|
||||
MergeTreeReadPool(const MergeTreeReadPool &) = delete;
|
||||
MergeTreeReadPool & operator=(const MergeTreeReadPool &) = delete;
|
||||
|
||||
MergeTreeReadTaskPtr getTask(const std::size_t min_marks_to_read, const std::size_t thread)
|
||||
{
|
||||
const std::lock_guard<std::mutex> lock{mutex};
|
||||
|
||||
if (remaining_thread_tasks.empty())
|
||||
return nullptr;
|
||||
|
||||
const auto tasks_remaining_for_this_thread = !threads_tasks[thread].sum_marks_in_parts.empty();
|
||||
if (!tasks_remaining_for_this_thread && do_not_steal_tasks)
|
||||
return nullptr;
|
||||
|
||||
const auto thread_idx = tasks_remaining_for_this_thread ? thread : *std::begin(remaining_thread_tasks);
|
||||
auto & thread_tasks = threads_tasks[thread_idx];
|
||||
|
||||
auto & thread_task = thread_tasks.parts_and_ranges.back();
|
||||
const auto part_idx = thread_task.part_idx;
|
||||
|
||||
auto & part = parts[part_idx];
|
||||
auto & marks_in_part = thread_tasks.sum_marks_in_parts.back();
|
||||
|
||||
/// Берём весь кусок, если он достаточно мал
|
||||
auto need_marks = std::min(marks_in_part, min_marks_to_read);
|
||||
|
||||
/// Не будем оставлять в куске слишком мало строк.
|
||||
if (marks_in_part > need_marks &&
|
||||
marks_in_part - need_marks < min_marks_to_read)
|
||||
need_marks = marks_in_part;
|
||||
|
||||
MarkRanges ranges_to_get_from_part;
|
||||
|
||||
/// Возьмем весь кусок, если он достаточно мал.
|
||||
if (marks_in_part <= need_marks)
|
||||
{
|
||||
const auto marks_to_get_from_range = marks_in_part;
|
||||
|
||||
/// Восстановим порядок отрезков.
|
||||
std::reverse(thread_task.ranges.begin(), thread_task.ranges.end());
|
||||
|
||||
ranges_to_get_from_part = thread_task.ranges;
|
||||
|
||||
marks_in_part -= marks_to_get_from_range;
|
||||
|
||||
thread_tasks.parts_and_ranges.pop_back();
|
||||
thread_tasks.sum_marks_in_parts.pop_back();
|
||||
|
||||
if (thread_tasks.sum_marks_in_parts.empty())
|
||||
remaining_thread_tasks.erase(thread_idx);
|
||||
}
|
||||
else
|
||||
{
|
||||
/// Цикл по отрезкам куска.
|
||||
while (need_marks > 0 && !thread_task.ranges.empty())
|
||||
{
|
||||
auto & range = thread_task.ranges.back();
|
||||
|
||||
const std::size_t marks_in_range = range.end - range.begin;
|
||||
const std::size_t marks_to_get_from_range = std::min(marks_in_range, need_marks);
|
||||
|
||||
ranges_to_get_from_part.emplace_back(range.begin, range.begin + marks_to_get_from_range);
|
||||
range.begin += marks_to_get_from_range;
|
||||
if (range.begin == range.end)
|
||||
{
|
||||
std::swap(range, thread_task.ranges.back());
|
||||
thread_task.ranges.pop_back();
|
||||
}
|
||||
|
||||
marks_in_part -= marks_to_get_from_range;
|
||||
need_marks -= marks_to_get_from_range;
|
||||
}
|
||||
}
|
||||
|
||||
return std::make_unique<MergeTreeReadTask>(
|
||||
part.data_part, ranges_to_get_from_part, part.part_index_in_query, column_names,
|
||||
per_part_column_name_set[part_idx], per_part_columns[part_idx], per_part_pre_columns[part_idx],
|
||||
per_part_remove_prewhere_column[part_idx], per_part_should_reorder[part_idx]);
|
||||
}
|
||||
|
||||
public:
|
||||
std::vector<std::size_t> fillPerPartInfo(
|
||||
RangesInDataParts & parts, const ExpressionActionsPtr & prewhere_actions, const String & prewhere_column_name,
|
||||
const bool check_columns)
|
||||
{
|
||||
std::vector<std::size_t> per_part_sum_marks;
|
||||
|
||||
for (const auto i : ext::range(0, parts.size()))
|
||||
{
|
||||
auto & part = parts[i];
|
||||
|
||||
/// Посчитаем засечки для каждого куска.
|
||||
size_t sum_marks = 0;
|
||||
/// Отрезки уже перечислены справа налево, reverse в MergeTreeDataSelectExecutor.
|
||||
for (const auto & range : part.ranges)
|
||||
sum_marks += range.end - range.begin;
|
||||
|
||||
per_part_sum_marks.push_back(sum_marks);
|
||||
|
||||
per_part_columns_lock.push_back(std::make_unique<Poco::ScopedReadRWLock>(
|
||||
part.data_part->columns_lock));
|
||||
|
||||
/// inject column names required for DEFAULT evaluation in current part
|
||||
auto required_column_names = column_names;
|
||||
|
||||
const auto injected_columns = injectRequiredColumns(part.data_part, required_column_names);
|
||||
auto should_reoder = !injected_columns.empty();
|
||||
|
||||
Names required_pre_column_names;
|
||||
|
||||
if (prewhere_actions)
|
||||
{
|
||||
/// collect columns required for PREWHERE evaluation
|
||||
required_pre_column_names = prewhere_actions->getRequiredColumns();
|
||||
|
||||
/// there must be at least one column required for PREWHERE
|
||||
if (required_pre_column_names.empty())
|
||||
required_pre_column_names.push_back(required_column_names[0]);
|
||||
|
||||
/// PREWHERE columns may require some additional columns for DEFAULT evaluation
|
||||
const auto injected_pre_columns = injectRequiredColumns(part.data_part, required_pre_column_names);
|
||||
if (!injected_pre_columns.empty())
|
||||
should_reoder = true;
|
||||
|
||||
/// will be used to distinguish between PREWHERE and WHERE columns when applying filter
|
||||
const NameSet pre_name_set{
|
||||
std::begin(required_pre_column_names), std::end(required_pre_column_names)
|
||||
};
|
||||
/** Если выражение в PREWHERE - не столбец таблицы, не нужно отдавать наружу столбец с ним
|
||||
* (от storage ожидают получить только столбцы таблицы). */
|
||||
per_part_remove_prewhere_column.push_back(0 == pre_name_set.count(prewhere_column_name));
|
||||
|
||||
Names post_column_names;
|
||||
for (const auto & name : required_column_names)
|
||||
if (!pre_name_set.count(name))
|
||||
post_column_names.push_back(name);
|
||||
|
||||
required_column_names = post_column_names;
|
||||
}
|
||||
else
|
||||
per_part_remove_prewhere_column.push_back(false);
|
||||
|
||||
per_part_column_name_set.emplace_back(std::begin(required_column_names), std::end(required_column_names));
|
||||
|
||||
if (check_columns)
|
||||
{
|
||||
/** Под part->columns_lock проверим, что все запрошенные столбцы в куске того же типа, что в таблице.
|
||||
* Это может быть не так во время ALTER MODIFY. */
|
||||
if (!required_pre_column_names.empty())
|
||||
data.check(part.data_part->columns, required_pre_column_names);
|
||||
if (!required_column_names.empty())
|
||||
data.check(part.data_part->columns, required_column_names);
|
||||
|
||||
per_part_pre_columns.push_back(data.getColumnsList().addTypes(required_pre_column_names));
|
||||
per_part_columns.push_back(data.getColumnsList().addTypes(required_column_names));
|
||||
}
|
||||
else
|
||||
{
|
||||
per_part_pre_columns.push_back(part.data_part->columns.addTypes(required_pre_column_names));
|
||||
per_part_columns.push_back(part.data_part->columns.addTypes(required_column_names));
|
||||
}
|
||||
|
||||
per_part_should_reorder.push_back(should_reoder);
|
||||
|
||||
this->parts.push_back({ part.data_part, part.part_index_in_query });
|
||||
}
|
||||
|
||||
return per_part_sum_marks;
|
||||
}
|
||||
|
||||
void fillPerThreadInfo(
|
||||
const std::size_t threads, const std::size_t sum_marks, std::vector<std::size_t> per_part_sum_marks,
|
||||
RangesInDataParts & parts, const std::size_t min_marks_for_concurrent_read)
|
||||
{
|
||||
threads_tasks.resize(threads);
|
||||
|
||||
const size_t min_marks_per_thread = (sum_marks - 1) / threads + 1;
|
||||
|
||||
for (std::size_t i = 0; i < threads && !parts.empty(); ++i)
|
||||
{
|
||||
auto need_marks = min_marks_per_thread;
|
||||
|
||||
while (need_marks > 0 && !parts.empty())
|
||||
{
|
||||
const auto part_idx = parts.size() - 1;
|
||||
RangesInDataPart & part = parts.back();
|
||||
size_t & marks_in_part = per_part_sum_marks.back();
|
||||
|
||||
/// Не будем брать из куска слишком мало строк.
|
||||
if (marks_in_part >= min_marks_for_concurrent_read &&
|
||||
need_marks < min_marks_for_concurrent_read)
|
||||
need_marks = min_marks_for_concurrent_read;
|
||||
|
||||
/// Не будем оставлять в куске слишком мало строк.
|
||||
if (marks_in_part > need_marks &&
|
||||
marks_in_part - need_marks < min_marks_for_concurrent_read)
|
||||
need_marks = marks_in_part;
|
||||
|
||||
MarkRanges ranges_to_get_from_part;
|
||||
size_t marks_in_ranges = need_marks;
|
||||
|
||||
/// Возьмем весь кусок, если он достаточно мал.
|
||||
if (marks_in_part <= need_marks)
|
||||
{
|
||||
/// Оставим отрезки перечисленными справа налево для удобства.
|
||||
ranges_to_get_from_part = part.ranges;
|
||||
marks_in_ranges = marks_in_part;
|
||||
|
||||
need_marks -= marks_in_part;
|
||||
parts.pop_back();
|
||||
per_part_sum_marks.pop_back();
|
||||
}
|
||||
else
|
||||
{
|
||||
/// Цикл по отрезкам куска.
|
||||
while (need_marks > 0)
|
||||
{
|
||||
if (part.ranges.empty())
|
||||
throw Exception("Unexpected end of ranges while spreading marks among threads", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
MarkRange & range = part.ranges.back();
|
||||
|
||||
const size_t marks_in_range = range.end - range.begin;
|
||||
const size_t marks_to_get_from_range = std::min(marks_in_range, need_marks);
|
||||
|
||||
ranges_to_get_from_part.emplace_back(range.begin, range.begin + marks_to_get_from_range);
|
||||
range.begin += marks_to_get_from_range;
|
||||
marks_in_part -= marks_to_get_from_range;
|
||||
need_marks -= marks_to_get_from_range;
|
||||
if (range.begin == range.end)
|
||||
part.ranges.pop_back();
|
||||
}
|
||||
|
||||
/// Вновь перечислим отрезки справа налево, чтобы .getTask() мог забирать их с помощью .pop_back().
|
||||
std::reverse(std::begin(ranges_to_get_from_part), std::end(ranges_to_get_from_part));
|
||||
}
|
||||
|
||||
threads_tasks[i].parts_and_ranges.push_back({ part_idx, ranges_to_get_from_part });
|
||||
threads_tasks[i].sum_marks_in_parts.push_back(marks_in_ranges);
|
||||
if (marks_in_ranges != 0)
|
||||
remaining_thread_tasks.insert(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Если некоторых запрошенных столбцов нет в куске,
|
||||
* то выясняем, какие столбцы может быть необходимо дополнительно прочитать,
|
||||
* чтобы можно было вычислить DEFAULT выражение для этих столбцов.
|
||||
* Добавляет их в columns. */
|
||||
NameSet injectRequiredColumns(const MergeTreeData::DataPartPtr & part, Names & columns) const
|
||||
{
|
||||
NameSet required_columns{std::begin(columns), std::end(columns)};
|
||||
NameSet injected_columns;
|
||||
|
||||
auto all_column_files_missing = true;
|
||||
|
||||
for (size_t i = 0; i < columns.size(); ++i)
|
||||
{
|
||||
const auto & column_name = columns[i];
|
||||
|
||||
/// column has files and hence does not require evaluation
|
||||
if (part->hasColumnFiles(column_name))
|
||||
{
|
||||
all_column_files_missing = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
const auto default_it = data.column_defaults.find(column_name);
|
||||
/// columns has no explicit default expression
|
||||
if (default_it == std::end(data.column_defaults))
|
||||
continue;
|
||||
|
||||
/// collect identifiers required for evaluation
|
||||
IdentifierNameSet identifiers;
|
||||
default_it->second.expression->collectIdentifierNames(identifiers);
|
||||
|
||||
for (const auto & identifier : identifiers)
|
||||
{
|
||||
if (data.hasColumn(identifier))
|
||||
{
|
||||
/// ensure each column is added only once
|
||||
if (required_columns.count(identifier) == 0)
|
||||
{
|
||||
columns.emplace_back(identifier);
|
||||
required_columns.emplace(identifier);
|
||||
injected_columns.emplace(identifier);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (all_column_files_missing)
|
||||
{
|
||||
addMinimumSizeColumn(part, columns);
|
||||
/// correctly report added column
|
||||
injected_columns.insert(columns.back());
|
||||
}
|
||||
|
||||
return injected_columns;
|
||||
}
|
||||
|
||||
/** Добавить столбец минимального размера.
|
||||
* Используется в случае, когда ни один столбец не нужен, но нужно хотя бы знать количество строк.
|
||||
* Добавляет в columns.
|
||||
*/
|
||||
void addMinimumSizeColumn(const MergeTreeData::DataPartPtr & part, Names & columns) const
|
||||
{
|
||||
const auto get_column_size = [this, &part] (const String & name) {
|
||||
const auto & files = part->checksums.files;
|
||||
|
||||
const auto escaped_name = escapeForFileName(name);
|
||||
const auto bin_file_name = escaped_name + ".bin";
|
||||
const auto mrk_file_name = escaped_name + ".mrk";
|
||||
|
||||
return files.find(bin_file_name)->second.file_size + files.find(mrk_file_name)->second.file_size;
|
||||
};
|
||||
|
||||
const auto & storage_columns = data.getColumnsList();
|
||||
const NameAndTypePair * minimum_size_column = nullptr;
|
||||
auto minimum_size = std::numeric_limits<size_t>::max();
|
||||
|
||||
for (const auto & column : storage_columns)
|
||||
{
|
||||
if (!part->hasColumnFiles(column.name))
|
||||
continue;
|
||||
|
||||
const auto size = get_column_size(column.name);
|
||||
if (size < minimum_size)
|
||||
{
|
||||
minimum_size = size;
|
||||
minimum_size_column = &column;
|
||||
}
|
||||
}
|
||||
|
||||
if (!minimum_size_column)
|
||||
throw Exception{
|
||||
"Could not find a column of minimum size in MergeTree",
|
||||
ErrorCodes::LOGICAL_ERROR
|
||||
};
|
||||
|
||||
columns.push_back(minimum_size_column->name);
|
||||
}
|
||||
|
||||
std::vector<std::unique_ptr<Poco::ScopedReadRWLock>> per_part_columns_lock;
|
||||
MergeTreeData & data;
|
||||
Names column_names;
|
||||
const bool do_not_steal_tasks;
|
||||
std::vector<NameSet> per_part_column_name_set;
|
||||
std::vector<NamesAndTypesList> per_part_columns;
|
||||
std::vector<NamesAndTypesList> per_part_pre_columns;
|
||||
/// @todo actually all of these values are either true or false for the whole query, thus no vector required
|
||||
std::vector<bool> per_part_remove_prewhere_column;
|
||||
std::vector<bool> per_part_should_reorder;
|
||||
|
||||
struct part_t
|
||||
{
|
||||
MergeTreeData::DataPartPtr data_part;
|
||||
std::size_t part_index_in_query;
|
||||
};
|
||||
|
||||
std::vector<part_t> parts;
|
||||
|
||||
struct thread_task_t
|
||||
{
|
||||
struct part_index_and_range_t
|
||||
{
|
||||
std::size_t part_idx;
|
||||
MarkRanges ranges;
|
||||
};
|
||||
|
||||
std::vector<part_index_and_range_t> parts_and_ranges;
|
||||
std::vector<std::size_t> sum_marks_in_parts;
|
||||
};
|
||||
|
||||
std::vector<thread_task_t> threads_tasks;
|
||||
|
||||
std::unordered_set<std::size_t> remaining_thread_tasks;
|
||||
|
||||
mutable std::mutex mutex;
|
||||
};
|
||||
|
||||
using MergeTreeReadPoolPtr = std::shared_ptr<MergeTreeReadPool>;
|
||||
|
||||
|
||||
}
|
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <DB/Storages/MarkCache.h>
|
||||
#include <DB/Storages/MergeTree/MarkRange.h>
|
||||
#include <DB/Storages/MergeTree/MergeTreeData.h>
|
||||
#include <DB/DataTypes/IDataType.h>
|
||||
#include <DB/DataTypes/DataTypeNested.h>
|
||||
@ -17,19 +18,6 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/** Пара засечек, определяющая диапазон строк в куске. Именно, диапазон имеет вид [begin * index_granularity, end * index_granularity).
|
||||
*/
|
||||
struct MarkRange
|
||||
{
|
||||
size_t begin;
|
||||
size_t end;
|
||||
|
||||
MarkRange() {}
|
||||
MarkRange(size_t begin_, size_t end_) : begin(begin_), end(end_) {}
|
||||
};
|
||||
|
||||
typedef std::vector<MarkRange> MarkRanges;
|
||||
|
||||
|
||||
/** Умеет читать данные между парой засечек из одного куска. При чтении последовательных отрезков не делает лишних seek-ов.
|
||||
* При чтении почти последовательных отрезков делает seek-и быстро, не выбрасывая содержимое буфера.
|
||||
@ -44,11 +32,23 @@ public:
|
||||
UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_,
|
||||
MergeTreeData & storage_, const MarkRanges & all_mark_ranges,
|
||||
size_t aio_threshold_, size_t max_read_buffer_size_)
|
||||
: path(path_), data_part(data_part), part_name(data_part->name), columns(columns_),
|
||||
uncompressed_cache(uncompressed_cache_), mark_cache(mark_cache_),
|
||||
storage(storage_), all_mark_ranges(all_mark_ranges),
|
||||
: uncompressed_cache(uncompressed_cache_), mark_cache(mark_cache_), storage(storage_),
|
||||
aio_threshold(aio_threshold_), max_read_buffer_size(max_read_buffer_size_)
|
||||
{
|
||||
reconf(path_, data_part, columns_, all_mark_ranges);
|
||||
}
|
||||
|
||||
void reconf(
|
||||
const String & path, const MergeTreeData::DataPartPtr & data_part, const NamesAndTypesList & columns,
|
||||
const MarkRanges & all_mark_ranges)
|
||||
{
|
||||
this->path = path;
|
||||
this->data_part = data_part;
|
||||
this->part_name = data_part->name;
|
||||
this->columns = columns;
|
||||
this->all_mark_ranges = all_mark_ranges;
|
||||
this->streams.clear();
|
||||
|
||||
try
|
||||
{
|
||||
if (!Poco::File(path).exists())
|
||||
@ -74,20 +74,14 @@ public:
|
||||
{
|
||||
size_t max_rows_to_read = (to_mark - from_mark) * storage.index_granularity;
|
||||
|
||||
/** Для некоторых столбцов файлы с данными могут отсутствовать.
|
||||
* Это бывает для старых кусков, после добавления новых столбцов в структуру таблицы.
|
||||
*/
|
||||
auto has_missing_columns = false;
|
||||
|
||||
/// Указатели на столбцы смещений, общие для столбцов из вложенных структур данных
|
||||
/// Если append, все значения nullptr, и offset_columns используется только для проверки, что столбец смещений уже прочитан.
|
||||
OffsetColumns offset_columns;
|
||||
const auto read_column = [&] (const NameAndTypePair & it) {
|
||||
if (streams.end() == streams.find(it.name))
|
||||
|
||||
for (const NameAndTypePair & it : columns)
|
||||
{
|
||||
has_missing_columns = true;
|
||||
return;
|
||||
}
|
||||
if (streams.end() == streams.find(it.name))
|
||||
continue;
|
||||
|
||||
/// Все столбцы уже есть в блоке. Будем добавлять значения в конец.
|
||||
bool append = res.has(it.name);
|
||||
@ -120,24 +114,12 @@ public:
|
||||
|
||||
if (!append && column.column->size())
|
||||
res.insert(column);
|
||||
};
|
||||
|
||||
for (const NameAndTypePair & it : columns)
|
||||
read_column(it);
|
||||
|
||||
if (has_missing_columns && !res)
|
||||
{
|
||||
addMinimumSizeColumn();
|
||||
/// minimum size column is necessarily at list's front
|
||||
read_column(columns.front());
|
||||
}
|
||||
}
|
||||
catch (const Exception & e)
|
||||
{
|
||||
if (e.code() != ErrorCodes::MEMORY_LIMIT_EXCEEDED)
|
||||
{
|
||||
storage.reportBrokenPart(part_name);
|
||||
}
|
||||
|
||||
/// Более хорошая диагностика.
|
||||
throw Exception(e.message() + "\n(while reading from part " + path + " from mark " + toString(from_mark) + " to "
|
||||
@ -151,60 +133,13 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/** Добавить столбец минимального размера.
|
||||
* Используется в случае, когда ни один столбец не нужен, но нужно хотя бы знать количество строк.
|
||||
* Добавляет в columns.
|
||||
*/
|
||||
void addMinimumSizeColumn()
|
||||
{
|
||||
const auto get_column_size = [this] (const String & name) {
|
||||
const auto & files = data_part->checksums.files;
|
||||
|
||||
const auto escaped_name = escapeForFileName(name);
|
||||
const auto bin_file_name = escaped_name + ".bin";
|
||||
const auto mrk_file_name = escaped_name + ".mrk";
|
||||
|
||||
return files.find(bin_file_name)->second.file_size + files.find(mrk_file_name)->second.file_size;
|
||||
};
|
||||
|
||||
const auto & storage_columns = storage.getColumnsList();
|
||||
const NameAndTypePair * minimum_size_column = nullptr;
|
||||
auto minimum_size = std::numeric_limits<size_t>::max();
|
||||
|
||||
for (const auto & column : storage_columns)
|
||||
{
|
||||
if (!data_part->hasColumnFiles(column.name))
|
||||
continue;
|
||||
|
||||
const auto size = get_column_size(column.name);
|
||||
if (size < minimum_size)
|
||||
{
|
||||
minimum_size = size;
|
||||
minimum_size_column = &column;
|
||||
}
|
||||
}
|
||||
|
||||
if (!minimum_size_column)
|
||||
throw Exception{
|
||||
"could not find a column of minimum size in MergeTree",
|
||||
ErrorCodes::LOGICAL_ERROR
|
||||
};
|
||||
|
||||
addStream(minimum_size_column->name, *minimum_size_column->type, all_mark_ranges);
|
||||
columns.emplace(std::begin(columns), *minimum_size_column);
|
||||
|
||||
added_minimum_size_column = &columns.front();
|
||||
}
|
||||
|
||||
|
||||
/** Добавляет в блок недостающие столбцы из ordered_names, состоящие из значений по-умолчанию.
|
||||
* Недостающие столбцы добавляются в позиции, такие же как в ordered_names.
|
||||
* Если был добавлен хотя бы один столбец - то все столбцы в блоке переупорядочиваются как в ordered_names.
|
||||
*/
|
||||
void fillMissingColumns(Block & res, const Names & ordered_names)
|
||||
void fillMissingColumns(Block & res, const Names & ordered_names, const bool always_reorder = false)
|
||||
{
|
||||
fillMissingColumnsImpl(res, ordered_names, false);
|
||||
fillMissingColumnsImpl(res, ordered_names, always_reorder);
|
||||
}
|
||||
|
||||
/** То же самое, но всегда переупорядочивает столбцы в блоке, как в ordered_names
|
||||
@ -220,16 +155,14 @@ private:
|
||||
{
|
||||
MarkCache::MappedPtr marks;
|
||||
ReadBuffer * data_buffer;
|
||||
Poco::SharedPtr<CachedCompressedReadBuffer> cached_buffer;
|
||||
Poco::SharedPtr<CompressedReadBufferFromFile> non_cached_buffer;
|
||||
std::unique_ptr<CachedCompressedReadBuffer> cached_buffer;
|
||||
std::unique_ptr<CompressedReadBufferFromFile> non_cached_buffer;
|
||||
std::string path_prefix;
|
||||
size_t max_mark_range;
|
||||
|
||||
/// Используется в качестве подсказки, чтобы уменьшить количество реаллокаций при создании столбца переменной длины.
|
||||
double avg_value_size_hint = 0;
|
||||
|
||||
Stream(const String & path_prefix_, UncompressedCache * uncompressed_cache, MarkCache * mark_cache, const MarkRanges & all_mark_ranges,
|
||||
size_t aio_threshold, size_t max_read_buffer_size)
|
||||
Stream(
|
||||
const String & path_prefix_, UncompressedCache * uncompressed_cache, MarkCache * mark_cache,
|
||||
const MarkRanges & all_mark_ranges, size_t aio_threshold, size_t max_read_buffer_size)
|
||||
: path_prefix(path_prefix_)
|
||||
{
|
||||
loadMarks(mark_cache);
|
||||
@ -281,15 +214,15 @@ private:
|
||||
|
||||
if (uncompressed_cache)
|
||||
{
|
||||
cached_buffer = new CachedCompressedReadBuffer(path_prefix + ".bin", uncompressed_cache,
|
||||
estimated_size, aio_threshold, buffer_size);
|
||||
data_buffer = &*cached_buffer;
|
||||
cached_buffer = std::make_unique<CachedCompressedReadBuffer>(
|
||||
path_prefix + ".bin", uncompressed_cache, estimated_size, aio_threshold, buffer_size);
|
||||
data_buffer = cached_buffer.get();
|
||||
}
|
||||
else
|
||||
{
|
||||
non_cached_buffer = new CompressedReadBufferFromFile(path_prefix + ".bin", estimated_size,
|
||||
aio_threshold, buffer_size);
|
||||
data_buffer = &*non_cached_buffer;
|
||||
non_cached_buffer = std::make_unique<CompressedReadBufferFromFile>(
|
||||
path_prefix + ".bin", estimated_size, aio_threshold, buffer_size);
|
||||
data_buffer = non_cached_buffer.get();
|
||||
}
|
||||
}
|
||||
|
||||
@ -350,20 +283,21 @@ private:
|
||||
|
||||
typedef std::map<std::string, std::unique_ptr<Stream> > FileStreams;
|
||||
|
||||
/// Используется в качестве подсказки, чтобы уменьшить количество реаллокаций при создании столбца переменной длины.
|
||||
std::map<std::string, double> avg_value_size_hints;
|
||||
String path;
|
||||
const MergeTreeData::DataPartPtr & data_part;
|
||||
MergeTreeData::DataPartPtr data_part;
|
||||
String part_name;
|
||||
FileStreams streams;
|
||||
|
||||
/// Запрашиваемые столбцы. Возможно, с добавлением minimum_size_column.
|
||||
/// Запрашиваемые столбцы.
|
||||
NamesAndTypesList columns;
|
||||
const NameAndTypePair * added_minimum_size_column = nullptr;
|
||||
|
||||
UncompressedCache * uncompressed_cache;
|
||||
MarkCache * mark_cache;
|
||||
|
||||
MergeTreeData & storage;
|
||||
const MarkRanges & all_mark_ranges;
|
||||
MarkRanges all_mark_ranges;
|
||||
size_t aio_threshold;
|
||||
size_t max_read_buffer_size;
|
||||
|
||||
@ -386,14 +320,16 @@ private:
|
||||
+ ARRAY_SIZES_COLUMN_NAME_SUFFIX + toString(level);
|
||||
|
||||
if (!streams.count(size_name))
|
||||
streams.emplace(size_name, std::unique_ptr<Stream>(new Stream(
|
||||
path + escaped_size_name, uncompressed_cache, mark_cache, all_mark_ranges, aio_threshold, max_read_buffer_size)));
|
||||
streams.emplace(size_name, std::make_unique<Stream>(
|
||||
path + escaped_size_name, uncompressed_cache, mark_cache,
|
||||
all_mark_ranges, aio_threshold, max_read_buffer_size));
|
||||
|
||||
addStream(name, *type_arr->getNestedType(), all_mark_ranges, level + 1);
|
||||
}
|
||||
else
|
||||
streams[name].reset(new Stream(
|
||||
path + escaped_column_name, uncompressed_cache, mark_cache, all_mark_ranges, aio_threshold, max_read_buffer_size));
|
||||
streams.emplace(name, std::make_unique<Stream>(
|
||||
path + escaped_column_name, uncompressed_cache, mark_cache,
|
||||
all_mark_ranges, aio_threshold, max_read_buffer_size));
|
||||
}
|
||||
|
||||
|
||||
@ -453,8 +389,9 @@ private:
|
||||
else
|
||||
{
|
||||
Stream & stream = *streams[name];
|
||||
double & avg_value_size_hint = avg_value_size_hints[name];
|
||||
stream.seekToMark(from_mark);
|
||||
type.deserializeBinary(column, *stream.data_buffer, max_rows_to_read, stream.avg_value_size_hint);
|
||||
type.deserializeBinary(column, *stream.data_buffer, max_rows_to_read, avg_value_size_hint);
|
||||
|
||||
/// Вычисление подсказки о среднем размере значения.
|
||||
size_t column_size = column.size();
|
||||
@ -463,10 +400,10 @@ private:
|
||||
double current_avg_value_size = static_cast<double>(column.byteSize()) / column_size;
|
||||
|
||||
/// Эвристика, чтобы при изменениях, значение avg_value_size_hint быстро росло, но медленно уменьшалось.
|
||||
if (current_avg_value_size > stream.avg_value_size_hint)
|
||||
stream.avg_value_size_hint = current_avg_value_size;
|
||||
else if (current_avg_value_size * 2 < stream.avg_value_size_hint)
|
||||
stream.avg_value_size_hint = (current_avg_value_size + stream.avg_value_size_hint * 3) / 4;
|
||||
if (current_avg_value_size > avg_value_size_hint)
|
||||
avg_value_size_hint = current_avg_value_size;
|
||||
else if (current_avg_value_size * 2 < avg_value_size_hint)
|
||||
avg_value_size_hint = (current_avg_value_size + avg_value_size_hint * 3) / 4;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -548,15 +485,6 @@ private:
|
||||
if (should_evaluate_defaults)
|
||||
evaluateMissingDefaults(res, columns, storage.column_defaults, storage.context);
|
||||
|
||||
/// remove added column to ensure same content among all blocks
|
||||
if (added_minimum_size_column)
|
||||
{
|
||||
res.erase(0);
|
||||
streams.erase(added_minimum_size_column->name);
|
||||
columns.erase(std::begin(columns));
|
||||
added_minimum_size_column = nullptr;
|
||||
}
|
||||
|
||||
/// sort columns to ensure consistent order among all blocks
|
||||
if (should_sort)
|
||||
{
|
||||
@ -566,12 +494,6 @@ private:
|
||||
if (res.has(name))
|
||||
ordered_block.insert(res.getByName(name));
|
||||
|
||||
if (res.columns() != ordered_block.columns())
|
||||
throw Exception{
|
||||
"Ordered block has different number of columns than original one:\n" +
|
||||
ordered_block.dumpNames() + "\nvs.\n" + res.dumpNames(),
|
||||
ErrorCodes::LOGICAL_ERROR};
|
||||
|
||||
std::swap(res, ordered_block);
|
||||
}
|
||||
}
|
||||
|
@ -66,7 +66,7 @@ struct MergeTreeSettings
|
||||
size_t replicated_logs_to_keep = 100;
|
||||
|
||||
/// Настройки минимального количества битых данных, при котором отказываться автоматически их удалять.
|
||||
size_t max_suspicious_broken_parts = 5;
|
||||
size_t max_suspicious_broken_parts = 10;
|
||||
|
||||
/// Максимальное количество ошибок при загрузке кусков, при котором ReplicatedMergeTree соглашается запускаться.
|
||||
size_t replicated_max_unexpected_parts = 3;
|
||||
|
@ -0,0 +1,342 @@
|
||||
#pragma once
|
||||
|
||||
#include <DB/DataStreams/IProfilingBlockInputStream.h>
|
||||
#include <DB/Storages/MergeTree/MergeTreeData.h>
|
||||
#include <DB/Storages/MergeTree/PKCondition.h>
|
||||
#include <DB/Storages/MergeTree/MergeTreeReader.h>
|
||||
#include <DB/Storages/MergeTree/MergeTreeReadPool.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
|
||||
class MergeTreeThreadBlockInputStream : public IProfilingBlockInputStream
|
||||
{
|
||||
std::size_t thread;
|
||||
public:
|
||||
MergeTreeThreadBlockInputStream(
|
||||
const std::size_t thread,
|
||||
const MergeTreeReadPoolPtr & pool, const std::size_t min_marks_to_read, const std::size_t block_size,
|
||||
MergeTreeData & storage, const bool use_uncompressed_cache, const ExpressionActionsPtr & prewhere_actions,
|
||||
const String & prewhere_column, const Settings & settings, const Names & virt_column_names)
|
||||
: thread{thread}, pool{pool}, block_size_marks{block_size / storage.index_granularity},
|
||||
/// round min_marks_to_read up to nearest multiple of block_size expressed in marks
|
||||
min_marks_to_read{block_size
|
||||
? (min_marks_to_read * storage.index_granularity + block_size - 1)
|
||||
/ block_size * block_size / storage.index_granularity
|
||||
: min_marks_to_read
|
||||
},
|
||||
storage{storage}, use_uncompressed_cache{use_uncompressed_cache}, prewhere_actions{prewhere_actions},
|
||||
prewhere_column{prewhere_column}, min_bytes_to_use_direct_io{settings.min_bytes_to_use_direct_io},
|
||||
max_read_buffer_size{settings.max_read_buffer_size}, virt_column_names{virt_column_names},
|
||||
log{&Logger::get("MergeTreeThreadBlockInputStream")}
|
||||
{}
|
||||
|
||||
String getName() const override { return "MergeTreeThread"; }
|
||||
|
||||
String getID() const override
|
||||
{
|
||||
std::stringstream res;
|
||||
/// @todo print some meaningful information
|
||||
// res << "MergeTreeThread(columns";
|
||||
//
|
||||
// for (const auto & column : columns)
|
||||
// res << ", " << column.name;
|
||||
//
|
||||
// if (prewhere_actions)
|
||||
// res << ", prewhere, " << prewhere_actions->getID();
|
||||
//
|
||||
// res << ", marks";
|
||||
//
|
||||
// for (size_t i = 0; i < all_mark_ranges.size(); ++i)
|
||||
// res << ", " << all_mark_ranges[i].begin << ", " << all_mark_ranges[i].end;
|
||||
//
|
||||
// res << ")";
|
||||
return res.str();
|
||||
}
|
||||
|
||||
protected:
|
||||
/// Будем вызывать progressImpl самостоятельно.
|
||||
void progress(const Progress & value) override {}
|
||||
|
||||
Block readImpl() override
|
||||
{
|
||||
Block res;
|
||||
|
||||
while (!res)
|
||||
{
|
||||
if (!task && !getNewTask())
|
||||
break;
|
||||
|
||||
res = readFromPart();
|
||||
|
||||
if (res)
|
||||
injectVirtualColumns(res);
|
||||
|
||||
if (task->mark_ranges.empty())
|
||||
task = {};
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
private:
|
||||
bool getNewTask()
|
||||
{
|
||||
task = pool->getTask(min_marks_to_read, thread);
|
||||
|
||||
if (!task)
|
||||
{
|
||||
/** Закрываем файлы (ещё до уничтожения объекта).
|
||||
* Чтобы при создании многих источников, но одновременном чтении только из нескольких,
|
||||
* буферы не висели в памяти. */
|
||||
reader = {};
|
||||
pre_reader = {};
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto path = storage.getFullPath() + task->data_part->name + '/';
|
||||
|
||||
if (!reader)
|
||||
{
|
||||
if (use_uncompressed_cache)
|
||||
owned_uncompressed_cache = storage.context.getUncompressedCache();
|
||||
|
||||
owned_mark_cache = storage.context.getMarkCache();
|
||||
|
||||
reader = std::make_unique<MergeTreeReader>(
|
||||
path, task->data_part, task->columns, owned_uncompressed_cache.get(), owned_mark_cache.get(),
|
||||
storage, task->mark_ranges, min_bytes_to_use_direct_io, max_read_buffer_size);
|
||||
|
||||
if (prewhere_actions)
|
||||
pre_reader = std::make_unique<MergeTreeReader>(
|
||||
path, task->data_part, task->pre_columns, owned_uncompressed_cache.get(),
|
||||
owned_mark_cache.get(), storage, task->mark_ranges, min_bytes_to_use_direct_io,
|
||||
max_read_buffer_size);
|
||||
}
|
||||
else
|
||||
{
|
||||
reader->reconf(path, task->data_part, task->columns, task->mark_ranges);
|
||||
if (prewhere_actions)
|
||||
pre_reader->reconf(path, task->data_part, task->pre_columns, task->mark_ranges);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
Block readFromPart()
|
||||
{
|
||||
Block res;
|
||||
|
||||
if (prewhere_actions)
|
||||
{
|
||||
do
|
||||
{
|
||||
/// Прочитаем полный блок столбцов, нужных для вычисления выражения в PREWHERE.
|
||||
size_t space_left = std::max(1LU, block_size_marks);
|
||||
MarkRanges ranges_to_read;
|
||||
while (!task->mark_ranges.empty() && space_left)
|
||||
{
|
||||
auto & range = task->mark_ranges.back();
|
||||
|
||||
size_t marks_to_read = std::min(range.end - range.begin, space_left);
|
||||
pre_reader->readRange(range.begin, range.begin + marks_to_read, res);
|
||||
|
||||
ranges_to_read.emplace_back(range.begin, range.begin + marks_to_read);
|
||||
space_left -= marks_to_read;
|
||||
range.begin += marks_to_read;
|
||||
if (range.begin == range.end)
|
||||
task->mark_ranges.pop_back();
|
||||
}
|
||||
progressImpl({ res.rowsInFirstColumn(), res.bytes() });
|
||||
pre_reader->fillMissingColumns(res, task->ordered_names, task->should_reorder);
|
||||
|
||||
/// Вычислим выражение в PREWHERE.
|
||||
prewhere_actions->execute(res);
|
||||
|
||||
ColumnPtr column = res.getByName(prewhere_column).column;
|
||||
if (task->remove_prewhere_column)
|
||||
res.erase(prewhere_column);
|
||||
|
||||
const auto pre_bytes = res.bytes();
|
||||
|
||||
/** Если фильтр - константа (например, написано PREWHERE 1),
|
||||
* то либо вернём пустой блок, либо вернём блок без изменений.
|
||||
*/
|
||||
if (const auto column_const = typeid_cast<const ColumnConstUInt8 *>(column.get()))
|
||||
{
|
||||
if (!column_const->getData())
|
||||
{
|
||||
res.clear();
|
||||
return res;
|
||||
}
|
||||
|
||||
for (const auto & range : ranges_to_read)
|
||||
reader->readRange(range.begin, range.end, res);
|
||||
|
||||
progressImpl({ 0, res.bytes() - pre_bytes });
|
||||
}
|
||||
else if (const auto column_vec = typeid_cast<const ColumnUInt8 *>(column.get()))
|
||||
{
|
||||
size_t index_granularity = storage.index_granularity;
|
||||
|
||||
const auto & pre_filter = column_vec->getData();
|
||||
IColumn::Filter post_filter(pre_filter.size());
|
||||
|
||||
/// Прочитаем в нужных отрезках остальные столбцы и составим для них свой фильтр.
|
||||
size_t pre_filter_pos = 0;
|
||||
size_t post_filter_pos = 0;
|
||||
|
||||
for (const auto & range : ranges_to_read)
|
||||
{
|
||||
auto begin = range.begin;
|
||||
auto pre_filter_begin_pos = pre_filter_pos;
|
||||
|
||||
for (auto mark = range.begin; mark <= range.end; ++mark)
|
||||
{
|
||||
UInt8 nonzero = 0;
|
||||
|
||||
if (mark != range.end)
|
||||
{
|
||||
const size_t limit = std::min(pre_filter.size(), pre_filter_pos + index_granularity);
|
||||
for (size_t row = pre_filter_pos; row < limit; ++row)
|
||||
nonzero |= pre_filter[row];
|
||||
}
|
||||
|
||||
if (!nonzero)
|
||||
{
|
||||
if (mark > begin)
|
||||
{
|
||||
memcpy(
|
||||
&post_filter[post_filter_pos],
|
||||
&pre_filter[pre_filter_begin_pos],
|
||||
pre_filter_pos - pre_filter_begin_pos);
|
||||
post_filter_pos += pre_filter_pos - pre_filter_begin_pos;
|
||||
reader->readRange(begin, mark, res);
|
||||
}
|
||||
begin = mark + 1;
|
||||
pre_filter_begin_pos = std::min(pre_filter_pos + index_granularity, pre_filter.size());
|
||||
}
|
||||
|
||||
if (mark < range.end)
|
||||
pre_filter_pos = std::min(pre_filter_pos + index_granularity, pre_filter.size());
|
||||
}
|
||||
}
|
||||
|
||||
if (!post_filter_pos)
|
||||
{
|
||||
res.clear();
|
||||
continue;
|
||||
}
|
||||
|
||||
progressImpl({ 0, res.bytes() - pre_bytes });
|
||||
|
||||
post_filter.resize(post_filter_pos);
|
||||
|
||||
/// Отфильтруем столбцы, относящиеся к PREWHERE, используя pre_filter,
|
||||
/// остальные столбцы - используя post_filter.
|
||||
size_t rows = 0;
|
||||
for (const auto i : ext::range(0, res.columns()))
|
||||
{
|
||||
auto & col = res.getByPosition(i);
|
||||
if (col.name == prewhere_column && res.columns() > 1)
|
||||
continue;
|
||||
col.column =
|
||||
col.column->filter(task->column_name_set.count(col.name) ? post_filter : pre_filter);
|
||||
rows = col.column->size();
|
||||
}
|
||||
|
||||
/// Заменим столбец со значением условия из PREWHERE на константу.
|
||||
if (!task->remove_prewhere_column)
|
||||
res.getByName(prewhere_column).column = new ColumnConstUInt8{rows, 1};
|
||||
}
|
||||
else
|
||||
throw Exception{
|
||||
"Illegal type " + column->getName() + " of column for filter. Must be ColumnUInt8 or ColumnConstUInt8.",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER
|
||||
};
|
||||
|
||||
reader->fillMissingColumnsAndReorder(res, task->ordered_names);
|
||||
}
|
||||
while (!task->mark_ranges.empty() && !res && !isCancelled());
|
||||
}
|
||||
else
|
||||
{
|
||||
size_t space_left = std::max(1LU, block_size_marks);
|
||||
|
||||
while (!task->mark_ranges.empty() && space_left)
|
||||
{
|
||||
auto & range = task->mark_ranges.back();
|
||||
|
||||
const size_t marks_to_read = std::min(range.end - range.begin, space_left);
|
||||
reader->readRange(range.begin, range.begin + marks_to_read, res);
|
||||
|
||||
space_left -= marks_to_read;
|
||||
range.begin += marks_to_read;
|
||||
if (range.begin == range.end)
|
||||
task->mark_ranges.pop_back();
|
||||
}
|
||||
|
||||
progressImpl({ res.rowsInFirstColumn(), res.bytes() });
|
||||
|
||||
reader->fillMissingColumns(res, task->ordered_names, task->should_reorder);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
void injectVirtualColumns(Block & block)
|
||||
{
|
||||
const auto rows = block.rowsInFirstColumn();
|
||||
|
||||
/// add virtual columns
|
||||
if (!virt_column_names.empty())
|
||||
{
|
||||
for (const auto & virt_column_name : virt_column_names)
|
||||
{
|
||||
if (virt_column_name == "_part")
|
||||
{
|
||||
block.insert(ColumnWithTypeAndName{
|
||||
ColumnConst<String>{rows, task->data_part->name}.convertToFullColumn(),
|
||||
new DataTypeString,
|
||||
virt_column_name
|
||||
});
|
||||
}
|
||||
else if (virt_column_name == "_part_index")
|
||||
{
|
||||
block.insert(ColumnWithTypeAndName{
|
||||
ColumnConst<UInt64>{rows, task->part_index_in_query}.convertToFullColumn(),
|
||||
new DataTypeUInt64,
|
||||
virt_column_name
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
MergeTreeReadPoolPtr pool;
|
||||
const std::size_t block_size_marks;
|
||||
const std::size_t min_marks_to_read;
|
||||
MergeTreeData & storage;
|
||||
const bool use_uncompressed_cache;
|
||||
ExpressionActionsPtr prewhere_actions;
|
||||
const String prewhere_column;
|
||||
const std::size_t min_bytes_to_use_direct_io;
|
||||
const std::size_t max_read_buffer_size;
|
||||
const Names virt_column_names;
|
||||
|
||||
Logger * log;
|
||||
|
||||
using MergeTreeReaderPtr = std::unique_ptr<MergeTreeReader>;
|
||||
|
||||
UncompressedCachePtr owned_uncompressed_cache;
|
||||
MarkCachePtr owned_mark_cache;
|
||||
|
||||
MergeTreeReadTaskPtr task;
|
||||
MergeTreeReaderPtr reader;
|
||||
MergeTreeReaderPtr pre_reader;
|
||||
};
|
||||
|
||||
|
||||
}
|
29
dbms/include/DB/Storages/MergeTree/RangesInDataPart.h
Normal file
29
dbms/include/DB/Storages/MergeTree/RangesInDataPart.h
Normal file
@ -0,0 +1,29 @@
|
||||
#pragma once
|
||||
|
||||
#include <DB/Storages/MergeTree/MergeTreeData.h>
|
||||
#include <DB/Storages/MergeTree/MarkRange.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
|
||||
struct RangesInDataPart
|
||||
{
|
||||
MergeTreeData::DataPartPtr data_part;
|
||||
std::size_t part_index_in_query;
|
||||
MarkRanges ranges;
|
||||
|
||||
RangesInDataPart() = default;
|
||||
|
||||
RangesInDataPart(const MergeTreeData::DataPartPtr & data_part, const std::size_t part_index_in_query,
|
||||
const MarkRanges & ranges = MarkRanges{})
|
||||
: data_part{data_part}, part_index_in_query{part_index_in_query}, ranges{ranges}
|
||||
{
|
||||
}
|
||||
};
|
||||
|
||||
using RangesInDataParts = std::vector<RangesInDataPart>;
|
||||
|
||||
|
||||
}
|
262
dbms/scripts/gen-bias-data.py
Executable file
262
dbms/scripts/gen-bias-data.py
Executable file
@ -0,0 +1,262 @@
|
||||
#!/usr/bin/python3.4
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys
|
||||
import argparse
|
||||
import tempfile
|
||||
import random
|
||||
import subprocess
|
||||
import bisect
|
||||
from copy import deepcopy
|
||||
|
||||
# Псевдослучайный генератор уникальных чисел.
|
||||
# http://preshing.com/20121224/how-to-generate-a-sequence-of-unique-random-integers/
|
||||
class UniqueRandomGenerator:
|
||||
prime = 4294967291
|
||||
|
||||
def __init__(self, seed_base, seed_offset):
|
||||
self.index = self.permutePQR(self.permutePQR(seed_base) + 0x682f0161)
|
||||
self.intermediate_offset = self.permutePQR(self.permutePQR(seed_offset) + 0x46790905)
|
||||
|
||||
def next(self):
|
||||
val = self.permutePQR((self.permutePQR(self.index) + self.intermediate_offset) ^ 0x5bf03635)
|
||||
self.index = self.index + 1
|
||||
return val
|
||||
|
||||
def permutePQR(self, x):
|
||||
if x >=self.prime:
|
||||
return x
|
||||
else:
|
||||
residue = (x * x) % self.prime
|
||||
if x <= self.prime/2:
|
||||
return residue
|
||||
else:
|
||||
return self.prime - residue
|
||||
|
||||
# Создать таблицу содержащую уникальные значения.
|
||||
def generate_data_source(host, port, http_port, min_cardinality, max_cardinality, count):
|
||||
chunk_size = round((max_cardinality - min_cardinality) / float(count))
|
||||
used_values = 0
|
||||
|
||||
cur_count = 0
|
||||
next_size = 0
|
||||
|
||||
sup = 32768
|
||||
n1 = random.randrange(0, sup)
|
||||
n2 = random.randrange(0, sup)
|
||||
urng = UniqueRandomGenerator(n1, n2)
|
||||
|
||||
is_first = True
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
filename = tmp_dir + '/table.txt'
|
||||
with open(filename, 'w+b') as file_handle:
|
||||
while cur_count < count:
|
||||
|
||||
if is_first == True:
|
||||
is_first = False
|
||||
if min_cardinality != 0:
|
||||
next_size = min_cardinality + 1
|
||||
else:
|
||||
next_size = chunk_size
|
||||
else:
|
||||
next_size += chunk_size
|
||||
|
||||
while used_values < next_size:
|
||||
h = urng.next()
|
||||
used_values = used_values + 1
|
||||
out = str(h) + "\t" + str(cur_count) + "\n";
|
||||
file_handle.write(bytes(out, 'UTF-8'));
|
||||
cur_count = cur_count + 1
|
||||
|
||||
query = "DROP TABLE IF EXISTS data_source"
|
||||
subprocess.check_output(["clickhouse-client", "--host", host, "--port", str(port), "--query", query])
|
||||
query = "CREATE TABLE data_source(UserID UInt64, KeyID UInt64) ENGINE=TinyLog"
|
||||
subprocess.check_output(["clickhouse-client", "--host", host, "--port", str(port), "--query", query])
|
||||
|
||||
cat = subprocess.Popen(("cat", filename), stdout=subprocess.PIPE)
|
||||
subprocess.check_output(("POST", "http://{0}:{1}/?query=INSERT INTO data_source FORMAT TabSeparated".format(host, http_port)), stdin=cat.stdout)
|
||||
cat.wait()
|
||||
|
||||
def perform_query(host, port):
|
||||
query = "SELECT runningAccumulate(uniqExactState(UserID)) AS exact, "
|
||||
query += "runningAccumulate(uniqCombinedRawState(UserID)) AS approx "
|
||||
query += "FROM data_source GROUP BY KeyID"
|
||||
return subprocess.check_output(["clickhouse-client", "--host", host, "--port", port, "--query", query])
|
||||
|
||||
def parse_clickhouse_response(response):
|
||||
parsed = []
|
||||
lines = response.decode().split("\n")
|
||||
for cur_line in lines:
|
||||
rows = cur_line.split("\t")
|
||||
if len(rows) == 2:
|
||||
parsed.append([float(rows[0]), float(rows[1])])
|
||||
return parsed
|
||||
|
||||
def accumulate_data(accumulated_data, data):
|
||||
if not accumulated_data:
|
||||
accumulated_data = deepcopy(data)
|
||||
else:
|
||||
for row1, row2 in zip(accumulated_data, data):
|
||||
row1[1] += row2[1];
|
||||
return accumulated_data
|
||||
|
||||
def generate_raw_result(accumulated_data, count):
|
||||
expected_tab = []
|
||||
bias_tab = []
|
||||
for row in accumulated_data:
|
||||
exact = row[0]
|
||||
expected = row[1] / count
|
||||
bias = expected - exact
|
||||
|
||||
expected_tab.append(expected)
|
||||
bias_tab.append(bias)
|
||||
return [ expected_tab, bias_tab ]
|
||||
|
||||
def generate_sample(raw_estimates, biases, n_samples):
|
||||
result = []
|
||||
|
||||
min_card = raw_estimates[0]
|
||||
max_card = raw_estimates[len(raw_estimates) - 1]
|
||||
step = (max_card - min_card) / (n_samples - 1)
|
||||
|
||||
for i in range(0, n_samples + 1):
|
||||
x = min_card + i * step
|
||||
j = bisect.bisect_left(raw_estimates, x)
|
||||
|
||||
if j == len(raw_estimates):
|
||||
result.append((raw_estimates[j - 1], biases[j - 1]))
|
||||
elif raw_estimates[j] == x:
|
||||
result.append((raw_estimates[j], biases[j]))
|
||||
else:
|
||||
# Найти 6 ближайших соседей. Вычислить среднее арифметическое.
|
||||
|
||||
# 6 точек слева x [j-6 j-5 j-4 j-3 j-2 j-1]
|
||||
|
||||
begin = max(j - 6, 0) - 1
|
||||
end = j - 1
|
||||
|
||||
T = []
|
||||
for k in range(end, begin, -1):
|
||||
T.append(x - raw_estimates[k])
|
||||
|
||||
# 6 точек справа x [j j+1 j+2 j+3 j+4 j+5]
|
||||
|
||||
begin = j
|
||||
end = min(j + 5, len(raw_estimates) - 1) + 1
|
||||
|
||||
U = []
|
||||
for k in range(begin, end):
|
||||
U.append(raw_estimates[k] - x)
|
||||
|
||||
# Сливаем расстояния.
|
||||
|
||||
V = []
|
||||
|
||||
lim = min(len(T), len(U))
|
||||
k1 = 0
|
||||
k2 = 0
|
||||
|
||||
while k1 < lim and k2 < lim:
|
||||
if T[k1] == U[k2]:
|
||||
V.append(j - k1 - 1)
|
||||
V.append(j + k2)
|
||||
k1 = k1 + 1
|
||||
k2 = k2 + 1
|
||||
elif T[k1] < U[k2]:
|
||||
V.append(j - k1 - 1)
|
||||
k1 = k1 + 1
|
||||
else:
|
||||
V.append(j + k2)
|
||||
k2 = k2 + 1
|
||||
|
||||
if k1 < len(T):
|
||||
while k1 < len(T):
|
||||
V.append(j - k1 - 1)
|
||||
k1 = k1 + 1
|
||||
elif k2 < len(U):
|
||||
while k2 < len(U):
|
||||
V.append(j + k2)
|
||||
k2 = k2 + 1
|
||||
|
||||
# Выбираем 6 ближайших точек.
|
||||
# Вычисляем средние.
|
||||
|
||||
begin = 0
|
||||
end = min(len(V), 6)
|
||||
|
||||
sum = 0
|
||||
bias = 0
|
||||
for k in range(begin, end):
|
||||
sum += raw_estimates[V[k]]
|
||||
bias += biases[V[k]]
|
||||
sum /= float(end)
|
||||
bias /= float(end)
|
||||
|
||||
result.append((sum, bias))
|
||||
|
||||
# Пропустить последовательные результаты, чьи оценки одинаковые.
|
||||
final_result = []
|
||||
last = -1
|
||||
for entry in result:
|
||||
if entry[0] != last:
|
||||
final_result.append((entry[0], entry[1]))
|
||||
last = entry[0]
|
||||
|
||||
return final_result
|
||||
|
||||
def dump_arrays(data):
|
||||
|
||||
print("Size of each array: {0}\n".format(len(data)))
|
||||
|
||||
is_first = True
|
||||
sep = ''
|
||||
|
||||
print("raw_estimates = ")
|
||||
print("{")
|
||||
for row in data:
|
||||
print("\t{0}{1}".format(sep, row[0]))
|
||||
if is_first == True:
|
||||
is_first = False
|
||||
sep = ","
|
||||
print("};")
|
||||
|
||||
is_first = True
|
||||
sep = ""
|
||||
|
||||
print("\nbiases = ")
|
||||
print("{")
|
||||
for row in data:
|
||||
print("\t{0}{1}".format(sep, row[1]))
|
||||
if is_first == True:
|
||||
is_first = False
|
||||
sep = ","
|
||||
print("};")
|
||||
|
||||
def start():
|
||||
parser = argparse.ArgumentParser(description = "Generate bias correction tables for HyperLogLog-based functions.")
|
||||
parser.add_argument("-x", "--host", default="127.0.0.1", help="ClickHouse server host name");
|
||||
parser.add_argument("-p", "--port", type=int, default=9000, help="ClickHouse server TCP port");
|
||||
parser.add_argument("-t", "--http_port", type=int, default=8123, help="ClickHouse server HTTP port");
|
||||
parser.add_argument("-i", "--iterations", type=int, default=5000, help="number of iterations");
|
||||
parser.add_argument("-m", "--min_cardinality", type=int, default=16384, help="minimal cardinality");
|
||||
parser.add_argument("-M", "--max_cardinality", type=int, default=655360, help="maximal cardinality");
|
||||
parser.add_argument("-s", "--samples", type=int, default=200, help="number of sampled values");
|
||||
args = parser.parse_args()
|
||||
|
||||
accumulated_data = []
|
||||
|
||||
for i in range(0, args.iterations):
|
||||
print(i + 1)
|
||||
sys.stdout.flush()
|
||||
|
||||
generate_data_source(args.host, str(args.port), str(args.http_port), args.min_cardinality, args.max_cardinality, 1000)
|
||||
response = perform_query(args.host, str(args.port))
|
||||
data = parse_clickhouse_response(response)
|
||||
accumulated_data = accumulate_data(accumulated_data, data)
|
||||
|
||||
result = generate_raw_result(accumulated_data, args.iterations)
|
||||
sampled_data = generate_sample(result[0], result[1], args.samples)
|
||||
dump_arrays(sampled_data)
|
||||
|
||||
if __name__ == "__main__": start()
|
150
dbms/scripts/linear-counting-threshold.py
Executable file
150
dbms/scripts/linear-counting-threshold.py
Executable file
@ -0,0 +1,150 @@
|
||||
#!/usr/bin/python3.4
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys
|
||||
import argparse
|
||||
import tempfile
|
||||
import random
|
||||
import subprocess
|
||||
import bisect
|
||||
from copy import deepcopy
|
||||
|
||||
# Псевдослучайный генератор уникальных чисел.
|
||||
# http://preshing.com/20121224/how-to-generate-a-sequence-of-unique-random-integers/
|
||||
class UniqueRandomGenerator:
|
||||
prime = 4294967291
|
||||
|
||||
def __init__(self, seed_base, seed_offset):
|
||||
self.index = self.permutePQR(self.permutePQR(seed_base) + 0x682f0161)
|
||||
self.intermediate_offset = self.permutePQR(self.permutePQR(seed_offset) + 0x46790905)
|
||||
|
||||
def next(self):
|
||||
val = self.permutePQR((self.permutePQR(self.index) + self.intermediate_offset) ^ 0x5bf03635)
|
||||
self.index = self.index + 1
|
||||
return val
|
||||
|
||||
def permutePQR(self, x):
|
||||
if x >=self.prime:
|
||||
return x
|
||||
else:
|
||||
residue = (x * x) % self.prime
|
||||
if x <= self.prime/2:
|
||||
return residue
|
||||
else:
|
||||
return self.prime - residue
|
||||
|
||||
# Создать таблицу содержащую уникальные значения.
|
||||
def generate_data_source(host, port, http_port, min_cardinality, max_cardinality, count):
|
||||
chunk_size = round((max_cardinality - (min_cardinality + 1)) / float(count))
|
||||
used_values = 0
|
||||
|
||||
cur_count = 0
|
||||
next_size = 0
|
||||
|
||||
sup = 32768
|
||||
n1 = random.randrange(0, sup)
|
||||
n2 = random.randrange(0, sup)
|
||||
urng = UniqueRandomGenerator(n1, n2)
|
||||
|
||||
is_first = True
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
filename = tmp_dir + '/table.txt'
|
||||
with open(filename, 'w+b') as file_handle:
|
||||
while cur_count < count:
|
||||
|
||||
if is_first == True:
|
||||
is_first = False
|
||||
if min_cardinality != 0:
|
||||
next_size = min_cardinality + 1
|
||||
else:
|
||||
next_size = chunk_size
|
||||
else:
|
||||
next_size += chunk_size
|
||||
|
||||
while used_values < next_size:
|
||||
h = urng.next()
|
||||
used_values = used_values + 1
|
||||
out = str(h) + "\t" + str(cur_count) + "\n";
|
||||
file_handle.write(bytes(out, 'UTF-8'));
|
||||
cur_count = cur_count + 1
|
||||
|
||||
query = "DROP TABLE IF EXISTS data_source"
|
||||
subprocess.check_output(["clickhouse-client", "--host", host, "--port", str(port), "--query", query])
|
||||
query = "CREATE TABLE data_source(UserID UInt64, KeyID UInt64) ENGINE=TinyLog"
|
||||
subprocess.check_output(["clickhouse-client", "--host", host, "--port", str(port), "--query", query])
|
||||
|
||||
cat = subprocess.Popen(("cat", filename), stdout=subprocess.PIPE)
|
||||
subprocess.check_output(("POST", "http://{0}:{1}/?query=INSERT INTO data_source FORMAT TabSeparated".format(host, http_port)), stdin=cat.stdout)
|
||||
cat.wait()
|
||||
|
||||
def perform_query(host, port):
|
||||
query = "SELECT runningAccumulate(uniqExactState(UserID)) AS exact, "
|
||||
query += "runningAccumulate(uniqCombinedRawState(UserID)) AS raw, "
|
||||
query += "runningAccumulate(uniqCombinedLinearCountingState(UserID)) AS linear_counting, "
|
||||
query += "runningAccumulate(uniqCombinedBiasCorrectedState(UserID)) AS bias_corrected "
|
||||
query += "FROM data_source GROUP BY KeyID"
|
||||
return subprocess.check_output(["clickhouse-client", "--host", host, "--port", port, "--query", query])
|
||||
|
||||
def parse_clickhouse_response(response):
|
||||
parsed = []
|
||||
lines = response.decode().split("\n")
|
||||
for cur_line in lines:
|
||||
rows = cur_line.split("\t")
|
||||
if len(rows) == 4:
|
||||
parsed.append([float(rows[0]), float(rows[1]), float(rows[2]), float(rows[3])])
|
||||
return parsed
|
||||
|
||||
def accumulate_data(accumulated_data, data):
|
||||
if not accumulated_data:
|
||||
accumulated_data = deepcopy(data)
|
||||
else:
|
||||
for row1, row2 in zip(accumulated_data, data):
|
||||
row1[1] += row2[1];
|
||||
row1[2] += row2[2];
|
||||
row1[3] += row2[3];
|
||||
return accumulated_data
|
||||
|
||||
def dump_graphs(data, count):
|
||||
with open("raw_graph.txt", "w+b") as fh1, open("linear_counting_graph.txt", "w+b") as fh2, open("bias_corrected_graph.txt", "w+b") as fh3:
|
||||
expected_tab = []
|
||||
bias_tab = []
|
||||
for row in data:
|
||||
exact = row[0]
|
||||
raw = row[1] / count;
|
||||
linear_counting = row[2] / count;
|
||||
bias_corrected = row[3] / count;
|
||||
|
||||
outstr = "{0}\t{1}\n".format(exact, abs(raw - exact) / exact)
|
||||
fh1.write(bytes(outstr, 'UTF-8'))
|
||||
|
||||
outstr = "{0}\t{1}\n".format(exact, abs(linear_counting - exact) / exact)
|
||||
fh2.write(bytes(outstr, 'UTF-8'))
|
||||
|
||||
outstr = "{0}\t{1}\n".format(exact, abs(bias_corrected - exact) / exact)
|
||||
fh3.write(bytes(outstr, 'UTF-8'))
|
||||
|
||||
def start():
|
||||
parser = argparse.ArgumentParser(description = "Generate graphs that help to determine the linear counting threshold.")
|
||||
parser.add_argument("-x", "--host", default="127.0.0.1", help="clickhouse host name");
|
||||
parser.add_argument("-p", "--port", type=int, default=9000, help="clickhouse client TCP port");
|
||||
parser.add_argument("-t", "--http_port", type=int, default=8123, help="clickhouse HTTP port");
|
||||
parser.add_argument("-i", "--iterations", type=int, default=5000, help="number of iterations");
|
||||
parser.add_argument("-m", "--min_cardinality", type=int, default=16384, help="minimal cardinality");
|
||||
parser.add_argument("-M", "--max_cardinality", type=int, default=655360, help="maximal cardinality");
|
||||
args = parser.parse_args()
|
||||
|
||||
accumulated_data = []
|
||||
|
||||
for i in range(0, args.iterations):
|
||||
print(i + 1)
|
||||
sys.stdout.flush()
|
||||
|
||||
generate_data_source(args.host, str(args.port), str(args.http_port), args.min_cardinality, args.max_cardinality, 1000)
|
||||
response = perform_query(args.host, str(args.port))
|
||||
data = parse_clickhouse_response(response)
|
||||
accumulated_data = accumulate_data(accumulated_data, data)
|
||||
|
||||
dump_graphs(accumulated_data, args.iterations)
|
||||
|
||||
if __name__ == "__main__": start()
|
@ -351,6 +351,72 @@ AggregateFunctionPtr AggregateFunctionFactory::get(const String & name, const Da
|
||||
else
|
||||
throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
else if (name == "uniqCombinedRaw")
|
||||
{
|
||||
if (argument_types.size() != 1)
|
||||
throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
const IDataType & argument_type = *argument_types[0];
|
||||
|
||||
AggregateFunctionPtr res = createWithNumericType<AggregateFunctionUniq, AggregateFunctionUniqCombinedRawData>(*argument_types[0]);
|
||||
|
||||
if (res)
|
||||
return res;
|
||||
else if (typeid_cast<const DataTypeDate *>(&argument_type))
|
||||
return new AggregateFunctionUniq<DataTypeDate::FieldType, AggregateFunctionUniqCombinedRawData<DataTypeDate::FieldType>>;
|
||||
else if (typeid_cast<const DataTypeDateTime*>(&argument_type))
|
||||
return new AggregateFunctionUniq<DataTypeDateTime::FieldType, AggregateFunctionUniqCombinedRawData<DataTypeDateTime::FieldType>>;
|
||||
else if (typeid_cast<const DataTypeString*>(&argument_type) || typeid_cast<const DataTypeFixedString*>(&argument_type))
|
||||
return new AggregateFunctionUniq<String, AggregateFunctionUniqCombinedRawData<String>>;
|
||||
else
|
||||
throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
else if (name == "uniqCombinedLinearCounting")
|
||||
{
|
||||
if (argument_types.size() != 1)
|
||||
throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
const IDataType & argument_type = *argument_types[0];
|
||||
|
||||
AggregateFunctionPtr res = createWithNumericType<AggregateFunctionUniq,
|
||||
AggregateFunctionUniqCombinedLinearCountingData>(*argument_types[0]);
|
||||
|
||||
if (res)
|
||||
return res;
|
||||
else if (typeid_cast<const DataTypeDate *>(&argument_type))
|
||||
return new AggregateFunctionUniq<DataTypeDate::FieldType,
|
||||
AggregateFunctionUniqCombinedLinearCountingData<DataTypeDate::FieldType>>;
|
||||
else if (typeid_cast<const DataTypeDateTime*>(&argument_type))
|
||||
return new AggregateFunctionUniq<DataTypeDateTime::FieldType,
|
||||
AggregateFunctionUniqCombinedLinearCountingData<DataTypeDateTime::FieldType>>;
|
||||
else if (typeid_cast<const DataTypeString*>(&argument_type) || typeid_cast<const DataTypeFixedString*>(&argument_type))
|
||||
return new AggregateFunctionUniq<String, AggregateFunctionUniqCombinedLinearCountingData<String>>;
|
||||
else
|
||||
throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
else if (name == "uniqCombinedBiasCorrected")
|
||||
{
|
||||
if (argument_types.size() != 1)
|
||||
throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
const IDataType & argument_type = *argument_types[0];
|
||||
|
||||
AggregateFunctionPtr res = createWithNumericType<AggregateFunctionUniq,
|
||||
AggregateFunctionUniqCombinedBiasCorrectedData>(*argument_types[0]);
|
||||
|
||||
if (res)
|
||||
return res;
|
||||
else if (typeid_cast<const DataTypeDate *>(&argument_type))
|
||||
return new AggregateFunctionUniq<DataTypeDate::FieldType,
|
||||
AggregateFunctionUniqCombinedBiasCorrectedData<DataTypeDate::FieldType>>;
|
||||
else if (typeid_cast<const DataTypeDateTime*>(&argument_type))
|
||||
return new AggregateFunctionUniq<DataTypeDateTime::FieldType,
|
||||
AggregateFunctionUniqCombinedBiasCorrectedData<DataTypeDateTime::FieldType>>;
|
||||
else if (typeid_cast<const DataTypeString*>(&argument_type) || typeid_cast<const DataTypeFixedString*>(&argument_type))
|
||||
return new AggregateFunctionUniq<String, AggregateFunctionUniqCombinedBiasCorrectedData<String>>;
|
||||
else
|
||||
throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
else if (name == "uniqCombined")
|
||||
{
|
||||
if (argument_types.size() != 1)
|
||||
@ -562,6 +628,13 @@ AggregateFunctionPtr AggregateFunctionFactory::get(const String & name, const Da
|
||||
|
||||
return new AggregateFunctionSequenceMatch;
|
||||
}
|
||||
else if (name == "sequenceCount")
|
||||
{
|
||||
if (!AggregateFunctionSequenceCount::sufficientArgs(argument_types.size()))
|
||||
throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
return new AggregateFunctionSequenceCount;
|
||||
}
|
||||
else if (name == "varSamp")
|
||||
{
|
||||
if (argument_types.size() != 1)
|
||||
@ -726,6 +799,9 @@ const AggregateFunctionFactory::FunctionNames & AggregateFunctionFactory::getFun
|
||||
"uniq",
|
||||
"uniqHLL12",
|
||||
"uniqExact",
|
||||
"uniqCombinedRaw",
|
||||
"uniqCombinedLinearCounting",
|
||||
"uniqCombinedBiasCorrected",
|
||||
"uniqCombined",
|
||||
"uniqUpTo",
|
||||
"groupArray",
|
||||
@ -743,6 +819,7 @@ const AggregateFunctionFactory::FunctionNames & AggregateFunctionFactory::getFun
|
||||
"quantileDeterministic",
|
||||
"quantilesDeterministic",
|
||||
"sequenceMatch",
|
||||
"sequenceCount",
|
||||
"varSamp",
|
||||
"varPop",
|
||||
"stddevSamp",
|
||||
|
390
dbms/src/AggregateFunctions/UniqCombinedBiasData.cpp
Normal file
390
dbms/src/AggregateFunctions/UniqCombinedBiasData.cpp
Normal file
@ -0,0 +1,390 @@
|
||||
#include <DB/AggregateFunctions/UniqCombinedBiasData.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
const UniqCombinedBiasData::InterpolatedData raw_estimates =
|
||||
{
|
||||
700.0
|
||||
,3850.0
|
||||
,7350.0
|
||||
,10850.0
|
||||
,14350.0
|
||||
,89003.5714
|
||||
,103764.30343333333
|
||||
,105572.1915
|
||||
,109252.46533333334
|
||||
,112638.20573333332
|
||||
,116094.29566666669
|
||||
,119619.81926666666
|
||||
,123214.92233333334
|
||||
,126469.06656666666
|
||||
,130196.15093333334
|
||||
,133566.85673333335
|
||||
,136991.63890000002
|
||||
,140470.0118666667
|
||||
,144000.91686666667
|
||||
,147585.44463333333
|
||||
,151222.7466
|
||||
,154447.75893333333
|
||||
,158181.68399999998
|
||||
,161492.41386666667
|
||||
,164840.6352
|
||||
,168713.9904
|
||||
,172143.82656666666
|
||||
,175611.2078
|
||||
,179116.94873333335
|
||||
,182658.0355
|
||||
,186236.36723333332
|
||||
,189332.1009
|
||||
,192976.1847
|
||||
,196654.62706666664
|
||||
,199835.39103333335
|
||||
,203575.92429999998
|
||||
,206808.87086666666
|
||||
,210611.72886666664
|
||||
,213896.25913333334
|
||||
,217759.63066666664
|
||||
,221096.10933333333
|
||||
,224456.31466666667
|
||||
,227839.0366333333
|
||||
,231242.72576666667
|
||||
,235239.98256666667
|
||||
,238688.95070000002
|
||||
,242158.17593333332
|
||||
,245649.42926666664
|
||||
,249158.9859666667
|
||||
,252689.67179999998
|
||||
,256241.95376666667
|
||||
,259214.9391666667
|
||||
,262798.3925666667
|
||||
,266399.8345666667
|
||||
,270018.35863333335
|
||||
,273653.1149
|
||||
,276696.7119
|
||||
,280366.51476666663
|
||||
,284051.95540000004
|
||||
,287133.5254333333
|
||||
,290847.31173333334
|
||||
,294579.5226
|
||||
,297698.64109999995
|
||||
,301454.39253333333
|
||||
,305223.59123333334
|
||||
,308375.3184666667
|
||||
,312170.06
|
||||
,315342.02996666665
|
||||
,319162.8188666667
|
||||
,322356.3565666666
|
||||
,326199.5866
|
||||
,329412.83396666666
|
||||
,332634.3235666667
|
||||
,336510.7596333333
|
||||
,339747.7330333333
|
||||
,343643.0385666667
|
||||
,346896.77420000004
|
||||
,350157.6729666667
|
||||
,354079.3932333334
|
||||
,357354.5196333334
|
||||
,360638.3034333333
|
||||
,364588.47873333335
|
||||
,367886.05706666666
|
||||
,371189.98006666667
|
||||
,375161.95876666665
|
||||
,378478.6737666666
|
||||
,381801.6619
|
||||
,385130.9645
|
||||
,389131.7460333333
|
||||
,392471.6233333333
|
||||
,395817.1175
|
||||
,399165.1003333333
|
||||
,402518.7819333333
|
||||
,406549.7624333333
|
||||
,409916.016
|
||||
,413289.0218666666
|
||||
,416661.9977333333
|
||||
,420040.4257333334
|
||||
,424099.3186333333
|
||||
,427485.4292000001
|
||||
,430876.4814666666
|
||||
,434269.4718
|
||||
,437665.82826666674
|
||||
,441066.7185
|
||||
,444469.97226666665
|
||||
,448561.9376666667
|
||||
,451974.73750000005
|
||||
,455389.1112
|
||||
,458808.5816666667
|
||||
,462230.8184666667
|
||||
,465656.9889
|
||||
,469081.3269
|
||||
,472512.4878
|
||||
,475944.4204333333
|
||||
,480065.7132666667
|
||||
,483502.04110000003
|
||||
,486939.5075666667
|
||||
,490379.7868333334
|
||||
,493818.5365333333
|
||||
,497259.08013333334
|
||||
,500705.3513
|
||||
,504155.6234666666
|
||||
,507606.65499999997
|
||||
,511060.7448666667
|
||||
,514517.4004
|
||||
,517973.35829999996
|
||||
,521431.3761666666
|
||||
,524891.7097333333
|
||||
,529044.7593
|
||||
,532507.0878999999
|
||||
,535971.5070333333
|
||||
,539436.2416999999
|
||||
,542903.1470333333
|
||||
,546370.3423
|
||||
,549837.6947999999
|
||||
,553307.0003666667
|
||||
,556775.3770333333
|
||||
,560247.6308333334
|
||||
,563721.0700333334
|
||||
,567196.7586333333
|
||||
,570669.8439666666
|
||||
,574146.018
|
||||
,577622.2794666667
|
||||
,581098.3862333334
|
||||
,584575.8826666666
|
||||
,588055.1468000001
|
||||
,591538.0234
|
||||
,595018.0103000001
|
||||
,598504.5469333333
|
||||
,601992.5697666666
|
||||
,605475.5452
|
||||
,608959.4645
|
||||
,612444.0261
|
||||
,615929.6436
|
||||
,619412.3877333334
|
||||
,622903.4263999999
|
||||
,626391.3657333333
|
||||
,629876.7359333333
|
||||
,633364.2825999999
|
||||
,636855.2673666667
|
||||
,640344.4321000001
|
||||
,643836.5543666667
|
||||
,647327.3073999999
|
||||
,650818.3525666667
|
||||
,654312.2421666667
|
||||
,657807.0899666668
|
||||
,661301.4443666666
|
||||
,664794.1040333334
|
||||
,668288.1969666666
|
||||
,671781.0196666667
|
||||
,675272.7522333333
|
||||
,678766.9045999999
|
||||
,682259.3583666667
|
||||
,685747.8148333334
|
||||
,689238.7994666666
|
||||
,692732.0478333334
|
||||
,696224.6407
|
||||
,700069.9224
|
||||
};
|
||||
|
||||
const UniqCombinedBiasData::InterpolatedData biases =
|
||||
{
|
||||
0.0
|
||||
,0.0
|
||||
,0.0
|
||||
,0.0
|
||||
,0.0
|
||||
,71153.5714
|
||||
,85214.30343333333
|
||||
,83522.1915
|
||||
,80202.46533333334
|
||||
,77288.20573333332
|
||||
,74444.29566666667
|
||||
,71669.81926666667
|
||||
,68964.92233333334
|
||||
,66619.06656666666
|
||||
,64046.15093333333
|
||||
,61816.85673333333
|
||||
,59641.6389
|
||||
,57520.01186666667
|
||||
,55450.91686666667
|
||||
,53435.44463333334
|
||||
,51472.74659999999
|
||||
,49797.75893333333
|
||||
,47931.68399999999
|
||||
,46342.41386666667
|
||||
,44790.6352
|
||||
,43063.9904
|
||||
,41593.82656666667
|
||||
,40161.2078
|
||||
,38766.94873333333
|
||||
,37408.035500000005
|
||||
,36086.36723333333
|
||||
,34982.1009
|
||||
,33726.184700000005
|
||||
,32504.627066666664
|
||||
,31485.391033333333
|
||||
,30325.924299999995
|
||||
,29358.870866666668
|
||||
,28261.72886666667
|
||||
,27346.259133333337
|
||||
,26309.630666666668
|
||||
,25446.109333333337
|
||||
,24606.31466666666
|
||||
,23789.036633333333
|
||||
,22992.725766666666
|
||||
,22089.98256666667
|
||||
,21338.9507
|
||||
,20608.175933333332
|
||||
,19899.429266666673
|
||||
,19208.985966666663
|
||||
,18539.6718
|
||||
,17891.95376666667
|
||||
,17364.939166666667
|
||||
,16748.392566666666
|
||||
,16149.834566666666
|
||||
,15568.358633333331
|
||||
,15003.114899999995
|
||||
,14546.711900000004
|
||||
,14016.51476666668
|
||||
,13501.955399999997
|
||||
,13083.52543333332
|
||||
,12597.311733333336
|
||||
,12129.522600000006
|
||||
,11748.641100000008
|
||||
,11304.392533333332
|
||||
,10873.59123333334
|
||||
,10525.318466666678
|
||||
,10120.059999999998
|
||||
,9792.029966666674
|
||||
,9412.818866666668
|
||||
,9106.356566666664
|
||||
,8749.58660000001
|
||||
,8462.833966666678
|
||||
,8184.323566666659
|
||||
,7860.759633333325
|
||||
,7597.733033333323
|
||||
,7293.038566666665
|
||||
,7046.774200000004
|
||||
,6807.672966666675
|
||||
,6529.393233333336
|
||||
,6304.519633333344
|
||||
,6088.30343333332
|
||||
,5838.4787333333325
|
||||
,5636.057066666661
|
||||
,5439.980066666671
|
||||
,5211.958766666658
|
||||
,5028.673766666664
|
||||
,4851.661899999996
|
||||
,4680.964499999992
|
||||
,4481.746033333319
|
||||
,4321.623333333322
|
||||
,4167.117500000012
|
||||
,4015.1003333333356
|
||||
,3868.781933333337
|
||||
,3699.762433333332
|
||||
,3566.0159999999937
|
||||
,3439.021866666648
|
||||
,3311.9977333333422
|
||||
,3190.4257333333276
|
||||
,3049.3186333333238
|
||||
,2935.4291999999937
|
||||
,2826.4814666666593
|
||||
,2719.4717999999993
|
||||
,2615.8282666666782
|
||||
,2516.7184999999977
|
||||
,2419.972266666669
|
||||
,2311.9376666666744
|
||||
,2224.7374999999884
|
||||
,2139.1111999999944
|
||||
,2058.581666666665
|
||||
,1980.8184666666687
|
||||
,1906.9888999999966
|
||||
,1831.3268999999952
|
||||
,1762.4878000000026
|
||||
,1694.420433333328
|
||||
,1615.7132666666682
|
||||
,1552.0410999999924
|
||||
,1489.507566666677
|
||||
,1429.7868333333365
|
||||
,1368.536533333332
|
||||
,1309.0801333333268
|
||||
,1255.35129999999
|
||||
,1205.6234666666617
|
||||
,1156.6549999999988
|
||||
,1110.744866666675
|
||||
,1067.4004000000034
|
||||
,1023.3583000000023
|
||||
,981.3761666666638
|
||||
,941.7097333333513
|
||||
,894.7593000000148
|
||||
,857.0879000000035
|
||||
,821.5070333333375
|
||||
,786.2416999999745
|
||||
,753.1470333333127
|
||||
,720.3422999999797
|
||||
,687.6947999999975
|
||||
,657.0003666666647
|
||||
,625.3770333333329
|
||||
,597.6308333333387
|
||||
,571.0700333333225
|
||||
,546.7586333333165
|
||||
,519.8439666666478
|
||||
,496.0180000000012
|
||||
,472.2794666666693
|
||||
,448.386233333343
|
||||
,425.8826666666816
|
||||
,405.1468000000071
|
||||
,388.0233999999861
|
||||
,368.01030000002356
|
||||
,354.54693333333125
|
||||
,342.5697666666626
|
||||
,325.5452000000126
|
||||
,309.4644999999825
|
||||
,294.0261000000173
|
||||
,279.64360000001034
|
||||
,262.38773333333666
|
||||
,253.42639999999665
|
||||
,241.36573333333945
|
||||
,226.7359333333443
|
||||
,214.28259999999622
|
||||
,205.26736666667662
|
||||
,194.43210000001514
|
||||
,186.55436666666841
|
||||
,177.30740000001
|
||||
,168.35256666666828
|
||||
,162.24216666668266
|
||||
,157.0899666666713
|
||||
,151.44436666666297
|
||||
,144.1040333333464
|
||||
,138.19696666668946
|
||||
,131.01966666666945
|
||||
,122.7522333333424
|
||||
,116.90459999998954
|
||||
,109.35836666667213
|
||||
,97.81483333332774
|
||||
,88.7994666666491
|
||||
,82.04783333333519
|
||||
,74.64070000000841
|
||||
,69.92240000003949
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
double UniqCombinedBiasData::getThreshold()
|
||||
{
|
||||
return 176000;
|
||||
}
|
||||
|
||||
const UniqCombinedBiasData::InterpolatedData & UniqCombinedBiasData::getRawEstimates()
|
||||
{
|
||||
return raw_estimates;
|
||||
}
|
||||
|
||||
const UniqCombinedBiasData::InterpolatedData & UniqCombinedBiasData::getBiases()
|
||||
{
|
||||
return biases;
|
||||
}
|
||||
|
||||
}
|
@ -8,8 +8,8 @@ namespace DB
|
||||
|
||||
using Poco::SharedPtr;
|
||||
|
||||
LimitBlockInputStream::LimitBlockInputStream(BlockInputStreamPtr input_, size_t limit_, size_t offset_)
|
||||
: limit(limit_), offset(offset_), pos(0)
|
||||
LimitBlockInputStream::LimitBlockInputStream(BlockInputStreamPtr input_, size_t limit_, size_t offset_, bool always_read_till_end_)
|
||||
: limit(limit_), offset(offset_), always_read_till_end(always_read_till_end_)
|
||||
{
|
||||
children.push_back(input_);
|
||||
}
|
||||
@ -23,7 +23,16 @@ Block LimitBlockInputStream::readImpl()
|
||||
/// pos - сколько строк было прочитано, включая последний прочитанный блок
|
||||
|
||||
if (pos >= offset + limit)
|
||||
{
|
||||
if (!always_read_till_end)
|
||||
return res;
|
||||
else
|
||||
{
|
||||
while (children.back()->read())
|
||||
;
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
do
|
||||
{
|
||||
|
@ -132,7 +132,7 @@ int main(int argc, char ** argv)
|
||||
Poco::SharedPtr<IBlockInputStream> in = table->read(column_names, 0, context, Settings(), stage)[0];
|
||||
in = new ExpressionBlockInputStream(in, expression);
|
||||
in = new FilterBlockInputStream(in, 4);
|
||||
//in = new LimitBlockInputStream(in, 10);
|
||||
//in = new LimitBlockInputStream(in, 10, 0);
|
||||
|
||||
WriteBufferFromOStream ob(std::cout);
|
||||
RowOutputStreamPtr out_ = new TabSeparatedRowOutputStream(ob, expression->getSampleBlock());
|
||||
|
@ -152,7 +152,7 @@ int main(int argc, char ** argv)
|
||||
Poco::SharedPtr<IBlockInputStream> in = table->read(column_names, 0, Context{}, Settings(), stage, argc == 2 ? atoi(argv[1]) : 1048576)[0];
|
||||
in = new PartialSortingBlockInputStream(in, sort_columns);
|
||||
in = new MergeSortingBlockInputStream(in, sort_columns, DEFAULT_BLOCK_SIZE, 0, 0, "");
|
||||
//in = new LimitBlockInputStream(in, 10);
|
||||
//in = new LimitBlockInputStream(in, 10, 0);
|
||||
|
||||
WriteBufferFromOStream ob(std::cout);
|
||||
RowOutputStreamPtr out_ = new TabSeparatedRowOutputStream(ob, sample);
|
||||
|
@ -45,7 +45,7 @@ int main(int argc, char ** argv)
|
||||
streams[i] = new DB::AsynchronousBlockInputStream(streams[i]);
|
||||
|
||||
DB::BlockInputStreamPtr stream = new DB::UnionBlockInputStream(streams, nullptr, settings.max_threads);
|
||||
stream = new DB::LimitBlockInputStream(stream, 10);
|
||||
stream = new DB::LimitBlockInputStream(stream, 10, 0);
|
||||
|
||||
DB::FormatFactory format_factory;
|
||||
DB::WriteBufferFromFileDescriptor wb(STDERR_FILENO);
|
||||
|
@ -8,14 +8,18 @@ void registerFunctionsCoding(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionToStringCutToZero>();
|
||||
factory.registerFunction<FunctionIPv6NumToString>();
|
||||
factory.registerFunction<FunctionCutIPv6>();
|
||||
factory.registerFunction<FunctionIPv6StringToNum>();
|
||||
factory.registerFunction<FunctionIPv4NumToString>();
|
||||
factory.registerFunction<FunctionIPv4StringToNum>();
|
||||
factory.registerFunction<FunctionIPv4NumToStringClassC>();
|
||||
factory.registerFunction<FunctionIPv4ToIPv6>();
|
||||
factory.registerFunction<FunctionHex>();
|
||||
factory.registerFunction<FunctionUnhex>();
|
||||
factory.registerFunction<FunctionBitmaskToArray>();
|
||||
factory.registerFunction<FunctionBitTest>();
|
||||
factory.registerFunction<FunctionBitTestAny>();
|
||||
factory.registerFunction<FunctionBitTestAll>();
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -11,10 +11,9 @@ namespace DB
|
||||
|
||||
/// Примечание: выделяется дополнительная страница, которая содежрит те данные, которые
|
||||
/// не влезают в основной буфер.
|
||||
ReadBufferAIO::ReadBufferAIO(const std::string & filename_, size_t buffer_size_, int flags_,
|
||||
char * existing_memory_)
|
||||
ReadBufferAIO::ReadBufferAIO(const std::string & filename_, size_t buffer_size_, int flags_, char * existing_memory_)
|
||||
: ReadBufferFromFileBase(buffer_size_ + DEFAULT_AIO_FILE_BLOCK_SIZE, existing_memory_, DEFAULT_AIO_FILE_BLOCK_SIZE),
|
||||
fill_buffer(BufferWithOwnMemory<ReadBuffer>(this->memory.size(), nullptr, DEFAULT_AIO_FILE_BLOCK_SIZE)),
|
||||
fill_buffer(BufferWithOwnMemory<ReadBuffer>(internalBuffer().size(), nullptr, DEFAULT_AIO_FILE_BLOCK_SIZE)),
|
||||
filename(filename_)
|
||||
{
|
||||
ProfileEvents::increment(ProfileEvents::FileOpen);
|
||||
|
@ -886,8 +886,45 @@ static SharedPtr<InterpreterSelectQuery> interpretSubquery(
|
||||
select_query->children.emplace_back(select_query->table);
|
||||
}
|
||||
else
|
||||
{
|
||||
query = subquery->children.at(0);
|
||||
|
||||
/** В подзапросе могут быть указаны столбцы с одинаковыми именами. Например, SELECT x, x FROM t
|
||||
* Это плохо, потому что результат такого запроса нельзя сохранить в таблицу, потому что в таблице не может быть одноимённых столбцов.
|
||||
* Сохранение в таблицу требуется для GLOBAL-подзапросов.
|
||||
*
|
||||
* Чтобы избежать такой ситуации, будем переименовывать одинаковые столбцы.
|
||||
*/
|
||||
|
||||
std::set<std::string> all_column_names;
|
||||
std::set<std::string> assigned_column_names;
|
||||
|
||||
if (ASTSelectQuery * select = typeid_cast<ASTSelectQuery *>(query.get()))
|
||||
{
|
||||
for (auto & expr : select->select_expression_list->children)
|
||||
all_column_names.insert(expr->getAliasOrColumnName());
|
||||
|
||||
for (auto & expr : select->select_expression_list->children)
|
||||
{
|
||||
auto name = expr->getAliasOrColumnName();
|
||||
|
||||
if (!assigned_column_names.insert(name).second)
|
||||
{
|
||||
size_t i = 1;
|
||||
while (all_column_names.end() != all_column_names.find(name + "_" + toString(i)))
|
||||
++i;
|
||||
|
||||
name = name + "_" + toString(i);
|
||||
expr = expr->clone(); /// Отменяет склейку одинаковых выражений в дереве.
|
||||
expr->setAlias(name);
|
||||
|
||||
all_column_names.insert(name);
|
||||
assigned_column_names.insert(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (required_columns.empty())
|
||||
return new InterpreterSelectQuery(query, subquery_context, QueryProcessingStage::Complete, subquery_depth + 1);
|
||||
else
|
||||
@ -1743,8 +1780,6 @@ bool ExpressionAnalyzer::appendJoin(ExpressionActionsChain & chain, bool only_ty
|
||||
|
||||
if (!subquery_for_set.join)
|
||||
{
|
||||
Names join_key_names_left(join_key_names_left_set.begin(), join_key_names_left_set.end());
|
||||
Names join_key_names_right(join_key_names_right_set.begin(), join_key_names_right_set.end());
|
||||
JoinPtr join = new Join(join_key_names_left, join_key_names_right, settings.limits, ast_join.kind, ast_join.strictness);
|
||||
|
||||
Names required_joined_columns(join_key_names_right.begin(), join_key_names_right.end());
|
||||
@ -2137,27 +2172,31 @@ void ExpressionAnalyzer::collectJoinedColumns(NameSet & joined_columns, NamesAnd
|
||||
auto & keys = typeid_cast<ASTExpressionList &>(*node.using_expr_list);
|
||||
for (const auto & key : keys.children)
|
||||
{
|
||||
if (!join_key_names_left_set.insert(key->getColumnName()).second)
|
||||
throw Exception("Duplicate column in USING list", ErrorCodes::DUPLICATE_COLUMN);
|
||||
if (join_key_names_left.end() == std::find(join_key_names_left.begin(), join_key_names_left.end(), key->getColumnName()))
|
||||
join_key_names_left.push_back(key->getColumnName());
|
||||
else
|
||||
throw Exception("Duplicate column " + key->getColumnName() + " in USING list", ErrorCodes::DUPLICATE_COLUMN);
|
||||
|
||||
if (!join_key_names_right_set.insert(key->getAliasOrColumnName()).second)
|
||||
throw Exception("Duplicate column in USING list", ErrorCodes::DUPLICATE_COLUMN);
|
||||
if (join_key_names_right.end() == std::find(join_key_names_right.begin(), join_key_names_right.end(), key->getAliasOrColumnName()))
|
||||
join_key_names_right.push_back(key->getAliasOrColumnName());
|
||||
else
|
||||
throw Exception("Duplicate column " + key->getAliasOrColumnName() + " in USING list", ErrorCodes::DUPLICATE_COLUMN);
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto i : ext::range(0, nested_result_sample.columns()))
|
||||
{
|
||||
const auto & col = nested_result_sample.getByPosition(i);
|
||||
if (!join_key_names_right_set.count(col.name))
|
||||
if (join_key_names_right.end() == std::find(join_key_names_right.begin(), join_key_names_right.end(), col.name))
|
||||
{
|
||||
joined_columns.insert(col.name);
|
||||
joined_columns_name_type.emplace_back(col.name, col.type);
|
||||
}
|
||||
}
|
||||
|
||||
/* for (const auto & name : join_key_names_left_set)
|
||||
/* for (const auto & name : join_key_names_left)
|
||||
std::cerr << "JOIN key (left): " << name << std::endl;
|
||||
for (const auto & name : join_key_names_right_set)
|
||||
for (const auto & name : join_key_names_right)
|
||||
std::cerr << "JOIN key (right): " << name << std::endl;
|
||||
std::cerr << std::endl;
|
||||
for (const auto & name : joined_columns)
|
||||
|
@ -98,7 +98,7 @@ void InterpreterSelectQuery::init(BlockInputStreamPtr input, const Names & requi
|
||||
|
||||
void InterpreterSelectQuery::basicInit(BlockInputStreamPtr input_)
|
||||
{
|
||||
if (query.table && typeid_cast<ASTSelectQuery *>(&*query.table))
|
||||
if (query.table && typeid_cast<ASTSelectQuery *>(query.table.get()))
|
||||
{
|
||||
if (table_column_names.empty())
|
||||
{
|
||||
@ -107,10 +107,10 @@ void InterpreterSelectQuery::basicInit(BlockInputStreamPtr input_)
|
||||
}
|
||||
else
|
||||
{
|
||||
if (query.table && typeid_cast<const ASTFunction *>(&*query.table))
|
||||
if (query.table && typeid_cast<const ASTFunction *>(query.table.get()))
|
||||
{
|
||||
/// Получить табличную функцию
|
||||
TableFunctionPtr table_function_ptr = context.getTableFunctionFactory().get(typeid_cast<const ASTFunction *>(&*query.table)->name, context);
|
||||
TableFunctionPtr table_function_ptr = context.getTableFunctionFactory().get(typeid_cast<const ASTFunction *>(query.table.get())->name, context);
|
||||
/// Выполнить ее и запомнить результат
|
||||
storage = table_function_ptr->execute(query.table, context);
|
||||
}
|
||||
@ -329,7 +329,7 @@ BlockIO InterpreterSelectQuery::execute()
|
||||
executeUnion();
|
||||
|
||||
/// Ограничения на результат, квота на результат, а также колбек для прогресса.
|
||||
if (IProfilingBlockInputStream * stream = dynamic_cast<IProfilingBlockInputStream *>(&*streams[0]))
|
||||
if (IProfilingBlockInputStream * stream = dynamic_cast<IProfilingBlockInputStream *>(streams[0].get()))
|
||||
{
|
||||
/// Ограничения действуют только на конечный результат.
|
||||
if (to_stage == QueryProcessingStage::Complete)
|
||||
@ -590,7 +590,7 @@ void InterpreterSelectQuery::executeSingleQuery()
|
||||
{
|
||||
transformStreams([&](auto & stream)
|
||||
{
|
||||
if (IProfilingBlockInputStream * p_stream = dynamic_cast<IProfilingBlockInputStream *>(&*stream))
|
||||
if (IProfilingBlockInputStream * p_stream = dynamic_cast<IProfilingBlockInputStream *>(stream.get()))
|
||||
p_stream->enableExtremes();
|
||||
});
|
||||
}
|
||||
@ -651,7 +651,7 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns()
|
||||
/// Список столбцов, которых нужно прочитать, чтобы выполнить запрос.
|
||||
Names required_columns = query_analyzer->getRequiredColumns();
|
||||
|
||||
if (query.table && typeid_cast<ASTSelectQuery *>(&*query.table))
|
||||
if (query.table && typeid_cast<ASTSelectQuery *>(query.table.get()))
|
||||
{
|
||||
/** Для подзапроса не действуют ограничения на максимальный размер результата.
|
||||
* Так как результат поздапроса - ещё не результат всего запроса.
|
||||
@ -792,7 +792,7 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns()
|
||||
|
||||
transformStreams([&](auto & stream)
|
||||
{
|
||||
if (IProfilingBlockInputStream * p_stream = dynamic_cast<IProfilingBlockInputStream *>(&*stream))
|
||||
if (IProfilingBlockInputStream * p_stream = dynamic_cast<IProfilingBlockInputStream *>(stream.get()))
|
||||
{
|
||||
p_stream->setLimits(limits);
|
||||
p_stream->setQuota(quota);
|
||||
@ -1091,9 +1091,38 @@ void InterpreterSelectQuery::executeLimit()
|
||||
/// Если есть LIMIT
|
||||
if (query.limit_length)
|
||||
{
|
||||
/** Редкий случай:
|
||||
* если нет WITH TOTALS и есть подзапрос в FROM, и там на одном из уровней есть WITH TOTALS,
|
||||
* то при использовании LIMIT-а следует читать данные до конца, а не отменять выполнение запроса раньше,
|
||||
* потому что при отмене выполнения запроса, мы не получим данные для totals с удалённого сервера.
|
||||
*/
|
||||
bool always_read_till_end = false;
|
||||
if (!query.group_by_with_totals && query.table && typeid_cast<const ASTSelectQuery *>(query.table.get()))
|
||||
{
|
||||
const ASTSelectQuery * subquery = static_cast<const ASTSelectQuery *>(query.table.get());
|
||||
|
||||
while (subquery->table)
|
||||
{
|
||||
if (subquery->group_by_with_totals)
|
||||
{
|
||||
/** NOTE Можно ещё проверять, что таблица в подзапросе - распределённая, и что она смотрит только на один шард.
|
||||
* В остальных случаях totals будет вычислен на сервере-инициаторе запроса, и читать данные до конца не обязательно.
|
||||
*/
|
||||
|
||||
always_read_till_end = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (typeid_cast<const ASTSelectQuery *>(subquery->table.get()))
|
||||
subquery = static_cast<const ASTSelectQuery *>(subquery->table.get());
|
||||
else
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
transformStreams([&](auto & stream)
|
||||
{
|
||||
stream = new LimitBlockInputStream(stream, limit_length, limit_offset);
|
||||
stream = new LimitBlockInputStream(stream, limit_length, limit_offset, always_read_till_end);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
@ -376,13 +376,19 @@ void Join::setSampleBlock(const Block & block)
|
||||
|
||||
sample_block_with_columns_to_add = block;
|
||||
|
||||
/// Удаляем из sample_block_with_columns_to_add ключевые столбцы.
|
||||
for (const auto & name : key_names_right)
|
||||
/// Переносим из sample_block_with_columns_to_add ключевые столбцы в sample_block_with_keys, сохраняя порядок.
|
||||
size_t pos = 0;
|
||||
while (pos < sample_block_with_columns_to_add.columns())
|
||||
{
|
||||
const auto & name = sample_block_with_columns_to_add.unsafeGetByPosition(pos).name;
|
||||
if (key_names_right.end() != std::find(key_names_right.begin(), key_names_right.end(), name))
|
||||
{
|
||||
size_t pos = sample_block_with_columns_to_add.getPositionByName(name);
|
||||
sample_block_with_keys.insert(sample_block_with_columns_to_add.unsafeGetByPosition(pos));
|
||||
sample_block_with_columns_to_add.erase(pos);
|
||||
}
|
||||
else
|
||||
++pos;
|
||||
}
|
||||
|
||||
for (size_t i = 0, size = sample_block_with_columns_to_add.columns(); i < size; ++i)
|
||||
{
|
||||
@ -426,7 +432,9 @@ bool Join::insertFromBlock(const Block & block)
|
||||
|
||||
if (getFullness(kind))
|
||||
{
|
||||
/// Переносим ключевые столбцы в начало блока.
|
||||
/** Переносим ключевые столбцы в начало блока.
|
||||
* Именно там их будет ожидать NonJoinedBlockInputStream.
|
||||
*/
|
||||
size_t key_num = 0;
|
||||
for (const auto & name : key_names_right)
|
||||
{
|
||||
@ -810,6 +818,8 @@ void Join::checkTypesOfKeys(const Block & block_left, const Block & block_right)
|
||||
|
||||
void Join::joinBlock(Block & block) const
|
||||
{
|
||||
// std::cerr << "joinBlock: " << block.dumpStructure() << "\n";
|
||||
|
||||
Poco::ScopedReadRWLock lock(rwlock);
|
||||
|
||||
checkTypesOfKeys(block, sample_block_with_keys);
|
||||
@ -917,6 +927,8 @@ public:
|
||||
|
||||
result_sample_block = left_sample_block;
|
||||
|
||||
// std::cerr << result_sample_block.dumpStructure() << "\n";
|
||||
|
||||
/// Добавляем в блок новые столбцы.
|
||||
for (size_t i = 0; i < num_columns_right; ++i)
|
||||
{
|
||||
@ -932,10 +944,11 @@ public:
|
||||
{
|
||||
const String & name = left_sample_block.getByPosition(i).name;
|
||||
|
||||
if (parent.key_names_left.end() == std::find(parent.key_names_left.begin(), parent.key_names_left.end(), name))
|
||||
auto found_key_column = std::find(parent.key_names_left.begin(), parent.key_names_left.end(), name);
|
||||
if (parent.key_names_left.end() == found_key_column)
|
||||
column_numbers_left.push_back(i);
|
||||
else
|
||||
column_numbers_keys_and_right.push_back(i);
|
||||
column_numbers_keys_and_right.push_back(found_key_column - parent.key_names_left.begin());
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < num_columns_right; ++i)
|
||||
@ -1046,8 +1059,6 @@ private:
|
||||
|
||||
for (; it != end; ++it)
|
||||
{
|
||||
// std::cerr << it->second.getUsed() << "\n";
|
||||
|
||||
if (it->second.getUsed())
|
||||
continue;
|
||||
|
||||
|
@ -1,6 +1,7 @@
|
||||
#include <DB/Storages/MergeTree/MergeTreeDataSelectExecutor.h>
|
||||
#include <DB/Storages/MergeTree/MergeTreeBlockInputStream.h>
|
||||
#include <DB/Interpreters/ExpressionAnalyzer.h>
|
||||
#include <DB/Storages/MergeTree/MergeTreeReadPool.h>
|
||||
#include <DB/Storages/MergeTree/MergeTreeThreadBlockInputStream.h>
|
||||
#include <DB/Parsers/ASTIdentifier.h>
|
||||
#include <DB/DataStreams/ExpressionBlockInputStream.h>
|
||||
#include <DB/DataStreams/FilterBlockInputStream.h>
|
||||
@ -346,9 +347,9 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongThreads(
|
||||
const Names & virt_columns,
|
||||
const Settings & settings)
|
||||
{
|
||||
const size_t min_marks_for_concurrent_read =
|
||||
const std::size_t min_marks_for_concurrent_read =
|
||||
(settings.merge_tree_min_rows_for_concurrent_read + data.index_granularity - 1) / data.index_granularity;
|
||||
const size_t max_marks_to_use_cache =
|
||||
const std::size_t max_marks_to_use_cache =
|
||||
(settings.merge_tree_max_rows_to_use_cache + data.index_granularity - 1) / data.index_granularity;
|
||||
|
||||
/// Посчитаем засечки для каждого куска.
|
||||
@ -370,7 +371,27 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongThreads(
|
||||
|
||||
BlockInputStreams res;
|
||||
|
||||
if (sum_marks > 0)
|
||||
if (sum_marks > 0 && settings.merge_tree_uniform_read_distribution == 1)
|
||||
{
|
||||
MergeTreeReadPoolPtr pool = std::make_shared<MergeTreeReadPool>(
|
||||
threads, sum_marks, min_marks_for_concurrent_read, parts, data, prewhere_actions, prewhere_column, true,
|
||||
column_names);
|
||||
|
||||
for (std::size_t i = 0; i < threads; ++i)
|
||||
res.emplace_back(new MergeTreeThreadBlockInputStream{
|
||||
i, pool, min_marks_for_concurrent_read, max_block_size, data, use_uncompressed_cache, prewhere_actions,
|
||||
prewhere_column, settings, virt_columns
|
||||
});
|
||||
|
||||
/// Оценим общее количество строк - для прогресс-бара.
|
||||
const std::size_t total_rows = data.index_granularity * sum_marks;
|
||||
|
||||
/// Выставим приблизительное количество строк только для первого источника
|
||||
static_cast<IProfilingBlockInputStream &>(*res.front()).setTotalRowsApprox(total_rows);
|
||||
|
||||
LOG_TRACE(log, "Reading approx. " << total_rows);
|
||||
}
|
||||
else if (sum_marks > 0)
|
||||
{
|
||||
const size_t min_marks_per_thread = (sum_marks - 1) / threads + 1;
|
||||
|
||||
@ -468,7 +489,10 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongThreadsFinal
|
||||
const Settings & settings,
|
||||
const Context & context)
|
||||
{
|
||||
size_t max_marks_to_use_cache = (settings.merge_tree_max_rows_to_use_cache + data.index_granularity - 1) / data.index_granularity;
|
||||
const size_t max_marks_to_use_cache =
|
||||
(settings.merge_tree_max_rows_to_use_cache + data.index_granularity - 1) / data.index_granularity;
|
||||
const size_t min_marks_for_read_task =
|
||||
(settings.merge_tree_min_rows_for_concurrent_read + data.index_granularity - 1) / data.index_granularity;
|
||||
|
||||
size_t sum_marks = 0;
|
||||
for (size_t i = 0; i < parts.size(); ++i)
|
||||
@ -480,6 +504,34 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongThreadsFinal
|
||||
|
||||
BlockInputStreams to_merge;
|
||||
|
||||
if (settings.merge_tree_uniform_read_distribution == 1)
|
||||
{
|
||||
MergeTreeReadPoolPtr pool = std::make_shared<MergeTreeReadPool>(
|
||||
parts.size(), sum_marks, min_marks_for_read_task, parts, data, prewhere_actions, prewhere_column, true,
|
||||
column_names, true);
|
||||
|
||||
for (const auto i : ext::range(0, parts.size()))
|
||||
{
|
||||
BlockInputStreamPtr source_stream{
|
||||
new MergeTreeThreadBlockInputStream{
|
||||
i, pool, min_marks_for_read_task, max_block_size, data, use_uncompressed_cache, prewhere_actions,
|
||||
prewhere_column, settings, virt_columns
|
||||
}
|
||||
};
|
||||
|
||||
to_merge.push_back(new ExpressionBlockInputStream(source_stream, data.getPrimaryExpression()));
|
||||
}
|
||||
|
||||
/// Оценим общее количество строк - для прогресс-бара.
|
||||
const std::size_t total_rows = data.index_granularity * sum_marks;
|
||||
|
||||
/// Выставим приблизительное количество строк только для первого источника
|
||||
static_cast<IProfilingBlockInputStream &>(*to_merge.front()).setTotalRowsApprox(total_rows);
|
||||
|
||||
LOG_TRACE(log, "Reading approx. " << total_rows);
|
||||
}
|
||||
else
|
||||
{
|
||||
for (size_t part_index = 0; part_index < parts.size(); ++part_index)
|
||||
{
|
||||
RangesInDataPart & part = parts[part_index];
|
||||
@ -501,6 +553,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongThreadsFinal
|
||||
|
||||
to_merge.push_back(new ExpressionBlockInputStream(source_stream, data.getPrimaryExpression()));
|
||||
}
|
||||
}
|
||||
|
||||
BlockInputStreams res;
|
||||
if (to_merge.size() == 1)
|
||||
|
@ -149,6 +149,9 @@ static void appendBlock(const Block & from, Block & to)
|
||||
throw Exception("Cannot append block to another: different type of columns at index " + toString(column_no)
|
||||
+ ". Block 1: " + from.dumpStructure() + ". Block 2: " + to.dumpStructure(), ErrorCodes::BLOCKS_HAS_DIFFERENT_STRUCTURE);
|
||||
|
||||
if (col_to.empty())
|
||||
to.getByPosition(column_no).column = col_from.clone();
|
||||
else
|
||||
for (size_t row_no = 0; row_no < rows; ++row_no)
|
||||
col_to.insertFrom(col_from, row_no);
|
||||
}
|
||||
@ -243,23 +246,17 @@ private:
|
||||
buffer.data = sorted_block.cloneEmpty();
|
||||
}
|
||||
|
||||
/// Если после вставки в буфер, ограничения будут превышены, то будем сбрасывать буфер.
|
||||
/** Если после вставки в буфер, ограничения будут превышены, то будем сбрасывать буфер.
|
||||
* Это также защищает от неограниченного потребления оперативки, так как в случае невозможности записать в таблицу,
|
||||
* будет выкинуто исключение, а новые данные не будут добавлены в буфер.
|
||||
*/
|
||||
if (storage.checkThresholds(buffer, time(0), sorted_block.rowsInFirstColumn(), sorted_block.bytes()))
|
||||
{
|
||||
/// Вытащим из буфера блок, заменим буфер на пустой. После этого можно разблокировать mutex.
|
||||
Block block_to_write;
|
||||
buffer.data.swap(block_to_write);
|
||||
buffer.first_write_time = 0;
|
||||
lock.unlock();
|
||||
storage.flushBuffer(buffer, false);
|
||||
lock.lock();
|
||||
}
|
||||
|
||||
if (!storage.no_destination)
|
||||
{
|
||||
auto destination = storage.context.tryGetTable(storage.destination_database, storage.destination_table);
|
||||
appendBlock(sorted_block, block_to_write);
|
||||
storage.writeBlockToDestination(block_to_write, destination);
|
||||
}
|
||||
}
|
||||
else
|
||||
appendBlock(sorted_block, buffer.data);
|
||||
}
|
||||
};
|
||||
@ -278,8 +275,15 @@ void StorageBuffer::shutdown()
|
||||
if (flush_thread.joinable())
|
||||
flush_thread.join();
|
||||
|
||||
try
|
||||
{
|
||||
optimize(context.getSettings());
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
bool StorageBuffer::optimize(const Settings & settings)
|
||||
@ -331,8 +335,16 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds)
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(buffer.mutex);
|
||||
|
||||
if (check_thresholds && !checkThresholds(buffer, current_time))
|
||||
if (check_thresholds)
|
||||
{
|
||||
if (!checkThresholds(buffer, current_time))
|
||||
return;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (buffer.data.rowsInFirstColumn() == 0)
|
||||
return;
|
||||
}
|
||||
|
||||
buffer.data.swap(block_to_write);
|
||||
buffer.first_write_time = 0;
|
||||
@ -357,10 +369,11 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds)
|
||||
* Замечание: остаётся проблема - из-за того, что в разных попытках вставляются разные блоки,
|
||||
* теряется идемпотентность вставки в ReplicatedMergeTree.
|
||||
*/
|
||||
appendBlock(block_to_write, buffer.data);
|
||||
buffer.data.swap(block_to_write);
|
||||
appendBlock(buffer.data, block_to_write);
|
||||
}
|
||||
|
||||
buffer.data.swap(block_to_write);
|
||||
|
||||
if (!buffer.first_write_time)
|
||||
buffer.first_write_time = current_time;
|
||||
|
||||
|
@ -85,7 +85,7 @@ int main(int argc, char ** argv)
|
||||
|
||||
DB::WriteBufferFromOStream out_buf(std::cout);
|
||||
|
||||
DB::LimitBlockInputStream in_limit(in, 10);
|
||||
DB::LimitBlockInputStream in_limit(in, 10, 0);
|
||||
DB::RowOutputStreamPtr output_ = new DB::TabSeparatedRowOutputStream(out_buf, sample);
|
||||
DB::BlockOutputStreamFromRowOutputStream output(output_);
|
||||
|
||||
|
@ -16,3 +16,79 @@
|
||||
1
|
||||
1
|
||||
1
|
||||
00000000000000000000FFFF4D583737
|
||||
00000000000000000000FFFF4D583737
|
||||
00000000000000000000FFFF7F000001
|
||||
00000000000000000000FFFFC1FC110A
|
||||
2001:db8:ac10:fe01:feed:babe:cafe:f00d
|
||||
2001:db8:ac10:fe01:feed:babe:cafe:f000
|
||||
2001:db8:ac10:fe01:feed:babe:cafe:0
|
||||
2001:db8:ac10:fe01:feed:babe:ca00:0
|
||||
2001:db8:ac10:fe01:feed:babe::
|
||||
2001:db8:ac10:fe01:feed:ba00::
|
||||
2001:db8:ac10:fe01:feed::
|
||||
2001:db8:ac10:fe01:fe00::
|
||||
2001:db8:ac10:fe01::
|
||||
2001:db8:ac10:fe00::
|
||||
2001:db8:ac10::
|
||||
2001:db8:ac00::
|
||||
2001:db8::
|
||||
2001:d00::
|
||||
2001::
|
||||
2000::
|
||||
::
|
||||
2001:db8:ac10:fe01:feed:babe:cafe:f00d
|
||||
2001:db8:ac10:fe01:feed:babe:cafe:f00d
|
||||
2001:db8:ac10:fe01:feed:babe:cafe:f00d
|
||||
2001:db8:ac10:fe01:feed:babe:cafe:f00d
|
||||
2001:db8:ac10:fe01:feed:babe:cafe:f00d
|
||||
2001:db8:ac10:fe01:feed:babe:cafe:f00d
|
||||
2001:db8:ac10:fe01:feed:babe:cafe:f00d
|
||||
2001:db8:ac10:fe01:feed:babe:cafe:f00d
|
||||
2001:db8:ac10:fe01:feed:babe:cafe:f00d
|
||||
2001:db8:ac10:fe01:feed:babe:cafe:f00d
|
||||
2001:db8:ac10:fe01:feed:babe:cafe:f00d
|
||||
2001:db8:ac10:fe01:feed:babe:cafe:f00d
|
||||
2001:db8:ac10:fe01:feed:babe:cafe:f00d
|
||||
2001:db8:ac10:fe01:feed:babe:cafe:f00d
|
||||
2001:db8:ac10:fe01:feed:babe:cafe:f00d
|
||||
2001:db8:ac10:fe01:feed:babe:cafe:f00d
|
||||
::ffff:193.252.17.10
|
||||
::ffff:193.252.17.10
|
||||
::ffff:193.252.17.10
|
||||
::ffff:193.252.17.10
|
||||
::ffff:193.252.17.10
|
||||
::ffff:193.252.17.10
|
||||
::ffff:193.252.17.10
|
||||
::ffff:193.252.17.10
|
||||
::ffff:193.252.17.10
|
||||
::ffff:193.252.17.10
|
||||
::ffff:193.252.17.10
|
||||
::ffff:193.252.17.10
|
||||
::ffff:193.252.17.10
|
||||
::ffff:193.252.17.10
|
||||
::ffff:193.252.17.10
|
||||
::ffff:193.252.17.10
|
||||
::ffff:193.252.17.10
|
||||
::ffff:193.252.17.0
|
||||
::ffff:193.252.0.0
|
||||
::ffff:193.0.0.0
|
||||
::ffff:0.0.0.0
|
||||
::ff00:0:0
|
||||
::
|
||||
::
|
||||
::
|
||||
::
|
||||
::
|
||||
::
|
||||
::
|
||||
::
|
||||
::
|
||||
::
|
||||
::
|
||||
2001:db8:ac10:fe01:abad:babe:fa00:0
|
||||
2001:db8:ac10:fe01:dead:c0de:ca00:0
|
||||
2001:db8:ac10:fe01:feed:babe:ca00:0
|
||||
::ffff:77.0.0.0
|
||||
::ffff:127.0.0.0
|
||||
::ffff:193.0.0.0
|
||||
|
@ -17,3 +17,109 @@ select IPv6StringToNum('') == toFixedString(materialize(''), 16);
|
||||
select IPv6StringToNum(materialize('')) == toFixedString(materialize(''), 16);
|
||||
select IPv6StringToNum('not an ip string') == toFixedString(materialize(''), 16);
|
||||
select IPv6StringToNum(materialize('not an ip string')) == toFixedString(materialize(''), 16);
|
||||
|
||||
/* IPv4ToIPv6 */
|
||||
|
||||
SELECT hex(IPv4ToIPv6(1297626935));
|
||||
|
||||
/* Тест с таблицей */
|
||||
|
||||
DROP TABLE IF EXISTS test.addresses;
|
||||
CREATE TABLE test.addresses(addr UInt32) ENGINE = Memory;
|
||||
INSERT INTO test.addresses(addr) VALUES (1297626935), (2130706433), (3254522122);
|
||||
SELECT hex(IPv4ToIPv6(addr)) FROM test.addresses ORDER BY addr ASC;
|
||||
|
||||
/* cutIPv6 */
|
||||
|
||||
/* Реальный IPv6-адрес */
|
||||
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 0);
|
||||
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 1, 0);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 2, 0);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 3, 0);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 4, 0);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 5, 0);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 6, 0);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 7, 0);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 8, 0);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 9, 0);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 10, 0);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 11, 0);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 12, 0);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 13, 0);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 14, 0);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 15, 0);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 16, 0);
|
||||
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 1);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 2);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 3);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 4);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 5);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 6);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 7);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 8);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 9);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 10);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 11);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 12);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 13);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 14);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 15);
|
||||
SELECT cutIPv6(IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), 0, 16);
|
||||
|
||||
/* IPv4-mapped IPv6-адрес */
|
||||
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 0);
|
||||
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 1, 0);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 2, 0);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 3, 0);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 4, 0);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 5, 0);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 6, 0);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 7, 0);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 8, 0);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 9, 0);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 10, 0);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 11, 0);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 12, 0);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 13, 0);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 14, 0);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 15, 0);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 16, 0);
|
||||
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 1);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 2);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 3);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 4);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 5);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 6);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 7);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 8);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 9);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 10);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 11);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 12);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 13);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 14);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 15);
|
||||
SELECT cutIPv6(toFixedString(unhex('00000000000000000000FFFFC1FC110A'), 16), 0, 16);
|
||||
|
||||
/* Тест с таблицами */
|
||||
|
||||
/* Реальные IPv6-адреса */
|
||||
|
||||
DROP TABLE IF EXISTS test.addresses;
|
||||
CREATE TABLE test.addresses(addr String) ENGINE = Memory;
|
||||
INSERT INTO test.addresses(addr) VALUES ('20010DB8AC10FE01FEEDBABECAFEF00D'), ('20010DB8AC10FE01DEADC0DECAFED00D'), ('20010DB8AC10FE01ABADBABEFACEB00C');
|
||||
SELECT cutIPv6(toFixedString(unhex(addr), 16), 3, 0) FROM test.addresses ORDER BY addr ASC;
|
||||
|
||||
/* IPv4-mapped IPv6-адреса */
|
||||
|
||||
DROP TABLE IF EXISTS test.addresses;
|
||||
CREATE TABLE test.addresses(addr String) ENGINE = Memory;
|
||||
INSERT INTO test.addresses(addr) VALUES ('00000000000000000000FFFFC1FC110A'), ('00000000000000000000FFFF4D583737'), ('00000000000000000000FFFF7F000001');
|
||||
SELECT cutIPv6(toFixedString(unhex(addr), 16), 0, 3) FROM test.addresses ORDER BY addr ASC;
|
||||
|
||||
|
@ -239,25 +239,25 @@
|
||||
31 162
|
||||
35 162
|
||||
36 162
|
||||
0 54226
|
||||
1 54034
|
||||
3 54016
|
||||
6 53982
|
||||
7 54076
|
||||
9 54218
|
||||
10 27075
|
||||
11 54093
|
||||
13 54108
|
||||
14 54096
|
||||
17 54294
|
||||
19 54070
|
||||
20 54028
|
||||
21 54170
|
||||
22 54106
|
||||
26 54103
|
||||
31 54050
|
||||
35 54130
|
||||
36 53868
|
||||
0 53901
|
||||
1 54056
|
||||
3 53999
|
||||
6 54129
|
||||
7 54213
|
||||
9 53853
|
||||
10 26975
|
||||
11 54064
|
||||
13 53963
|
||||
14 53997
|
||||
17 54129
|
||||
19 53923
|
||||
20 53958
|
||||
21 54117
|
||||
22 54150
|
||||
26 54047
|
||||
31 53948
|
||||
35 53931
|
||||
36 53982
|
||||
0.125 1
|
||||
0.5 1
|
||||
0.05 1
|
||||
@ -291,25 +291,25 @@
|
||||
0.043 162
|
||||
0.037 162
|
||||
0.071 162
|
||||
0.045 54170
|
||||
0.125 54076
|
||||
0.5 54034
|
||||
0.05 54070
|
||||
0.143 53982
|
||||
0.091 27075
|
||||
0.056 54294
|
||||
0.048 54028
|
||||
0.083 54093
|
||||
0.25 54016
|
||||
1 54226
|
||||
0.1 54218
|
||||
0.028 54130
|
||||
0.027 53868
|
||||
0.031 54050
|
||||
0.067 54096
|
||||
0.043 54106
|
||||
0.037 54103
|
||||
0.071 54108
|
||||
0.045 54117
|
||||
0.125 54213
|
||||
0.5 54056
|
||||
0.05 53923
|
||||
0.143 54129
|
||||
0.091 26975
|
||||
0.056 54129
|
||||
0.048 53958
|
||||
0.083 54064
|
||||
0.25 53999
|
||||
1 53901
|
||||
0.1 53853
|
||||
0.028 53931
|
||||
0.027 53982
|
||||
0.031 53948
|
||||
0.067 53997
|
||||
0.043 54150
|
||||
0.037 54047
|
||||
0.071 53963
|
||||
0.5 1
|
||||
0.05 1
|
||||
0.25 1
|
||||
@ -343,25 +343,25 @@
|
||||
0.037 162
|
||||
0.1 163
|
||||
1 162
|
||||
0.5 54034
|
||||
0.05 54070
|
||||
0.25 54016
|
||||
0.048 54028
|
||||
0.091 27075
|
||||
0.043 54106
|
||||
0.071 54108
|
||||
0.083 54093
|
||||
0.125 54076
|
||||
0.031 54050
|
||||
0.143 53982
|
||||
0.028 54130
|
||||
0.067 54096
|
||||
0.045 54170
|
||||
0.027 53868
|
||||
0.056 54294
|
||||
0.037 54103
|
||||
0.1 54218
|
||||
1 54226
|
||||
0.5 54056
|
||||
0.05 53923
|
||||
0.25 53999
|
||||
0.048 53958
|
||||
0.091 26975
|
||||
0.043 54150
|
||||
0.071 53963
|
||||
0.083 54064
|
||||
0.125 54213
|
||||
0.031 53948
|
||||
0.143 54129
|
||||
0.028 53931
|
||||
0.067 53997
|
||||
0.045 54117
|
||||
0.027 53982
|
||||
0.056 54129
|
||||
0.037 54047
|
||||
0.1 53853
|
||||
1 53901
|
||||
1 1
|
||||
3 1
|
||||
6 1
|
||||
|
@ -0,0 +1,771 @@
|
||||
1 1 1 1
|
||||
1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
|
||||
1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
|
||||
1 0 0 0 0 0 0 0 0 0 0
|
||||
1 1 1 0 0 0 0 0 0 0 1
|
||||
1 2 2 0 0 0 0 0 0 1 0
|
||||
1 3 3 0 0 0 0 0 0 1 1
|
||||
1 4 4 0 0 0 0 0 1 0 0
|
||||
1 5 5 0 0 0 0 0 1 0 1
|
||||
1 6 6 0 0 0 0 0 1 1 0
|
||||
1 7 7 0 0 0 0 0 1 1 1
|
||||
1 8 8 0 0 0 0 1 0 0 0
|
||||
1 9 9 0 0 0 0 1 0 0 1
|
||||
1 10 10 0 0 0 0 1 0 1 0
|
||||
1 11 11 0 0 0 0 1 0 1 1
|
||||
1 12 12 0 0 0 0 1 1 0 0
|
||||
1 13 13 0 0 0 0 1 1 0 1
|
||||
1 14 14 0 0 0 0 1 1 1 0
|
||||
1 15 15 0 0 0 0 1 1 1 1
|
||||
1 16 16 0 0 0 1 0 0 0 0
|
||||
1 17 17 0 0 0 1 0 0 0 1
|
||||
1 18 18 0 0 0 1 0 0 1 0
|
||||
1 19 19 0 0 0 1 0 0 1 1
|
||||
1 20 20 0 0 0 1 0 1 0 0
|
||||
1 21 21 0 0 0 1 0 1 0 1
|
||||
1 22 22 0 0 0 1 0 1 1 0
|
||||
1 23 23 0 0 0 1 0 1 1 1
|
||||
1 24 24 0 0 0 1 1 0 0 0
|
||||
1 25 25 0 0 0 1 1 0 0 1
|
||||
1 26 26 0 0 0 1 1 0 1 0
|
||||
1 27 27 0 0 0 1 1 0 1 1
|
||||
1 28 28 0 0 0 1 1 1 0 0
|
||||
1 29 29 0 0 0 1 1 1 0 1
|
||||
1 30 30 0 0 0 1 1 1 1 0
|
||||
1 31 31 0 0 0 1 1 1 1 1
|
||||
1 32 32 0 0 1 0 0 0 0 0
|
||||
1 33 33 0 0 1 0 0 0 0 1
|
||||
1 34 34 0 0 1 0 0 0 1 0
|
||||
1 35 35 0 0 1 0 0 0 1 1
|
||||
1 36 36 0 0 1 0 0 1 0 0
|
||||
1 37 37 0 0 1 0 0 1 0 1
|
||||
1 38 38 0 0 1 0 0 1 1 0
|
||||
1 39 39 0 0 1 0 0 1 1 1
|
||||
1 40 40 0 0 1 0 1 0 0 0
|
||||
1 41 41 0 0 1 0 1 0 0 1
|
||||
1 42 42 0 0 1 0 1 0 1 0
|
||||
1 43 43 0 0 1 0 1 0 1 1
|
||||
1 44 44 0 0 1 0 1 1 0 0
|
||||
1 45 45 0 0 1 0 1 1 0 1
|
||||
1 46 46 0 0 1 0 1 1 1 0
|
||||
1 47 47 0 0 1 0 1 1 1 1
|
||||
1 48 48 0 0 1 1 0 0 0 0
|
||||
1 49 49 0 0 1 1 0 0 0 1
|
||||
1 50 50 0 0 1 1 0 0 1 0
|
||||
1 51 51 0 0 1 1 0 0 1 1
|
||||
1 52 52 0 0 1 1 0 1 0 0
|
||||
1 53 53 0 0 1 1 0 1 0 1
|
||||
1 54 54 0 0 1 1 0 1 1 0
|
||||
1 55 55 0 0 1 1 0 1 1 1
|
||||
1 56 56 0 0 1 1 1 0 0 0
|
||||
1 57 57 0 0 1 1 1 0 0 1
|
||||
1 58 58 0 0 1 1 1 0 1 0
|
||||
1 59 59 0 0 1 1 1 0 1 1
|
||||
1 60 60 0 0 1 1 1 1 0 0
|
||||
1 61 61 0 0 1 1 1 1 0 1
|
||||
1 62 62 0 0 1 1 1 1 1 0
|
||||
1 63 63 0 0 1 1 1 1 1 1
|
||||
1 64 64 0 1 0 0 0 0 0 0
|
||||
1 65 65 0 1 0 0 0 0 0 1
|
||||
1 66 66 0 1 0 0 0 0 1 0
|
||||
1 67 67 0 1 0 0 0 0 1 1
|
||||
1 68 68 0 1 0 0 0 1 0 0
|
||||
1 69 69 0 1 0 0 0 1 0 1
|
||||
1 70 70 0 1 0 0 0 1 1 0
|
||||
1 71 71 0 1 0 0 0 1 1 1
|
||||
1 72 72 0 1 0 0 1 0 0 0
|
||||
1 73 73 0 1 0 0 1 0 0 1
|
||||
1 74 74 0 1 0 0 1 0 1 0
|
||||
1 75 75 0 1 0 0 1 0 1 1
|
||||
1 76 76 0 1 0 0 1 1 0 0
|
||||
1 77 77 0 1 0 0 1 1 0 1
|
||||
1 78 78 0 1 0 0 1 1 1 0
|
||||
1 79 79 0 1 0 0 1 1 1 1
|
||||
1 80 80 0 1 0 1 0 0 0 0
|
||||
1 81 81 0 1 0 1 0 0 0 1
|
||||
1 82 82 0 1 0 1 0 0 1 0
|
||||
1 83 83 0 1 0 1 0 0 1 1
|
||||
1 84 84 0 1 0 1 0 1 0 0
|
||||
1 85 85 0 1 0 1 0 1 0 1
|
||||
1 86 86 0 1 0 1 0 1 1 0
|
||||
1 87 87 0 1 0 1 0 1 1 1
|
||||
1 88 88 0 1 0 1 1 0 0 0
|
||||
1 89 89 0 1 0 1 1 0 0 1
|
||||
1 90 90 0 1 0 1 1 0 1 0
|
||||
1 91 91 0 1 0 1 1 0 1 1
|
||||
1 92 92 0 1 0 1 1 1 0 0
|
||||
1 93 93 0 1 0 1 1 1 0 1
|
||||
1 94 94 0 1 0 1 1 1 1 0
|
||||
1 95 95 0 1 0 1 1 1 1 1
|
||||
1 96 96 0 1 1 0 0 0 0 0
|
||||
1 97 97 0 1 1 0 0 0 0 1
|
||||
1 98 98 0 1 1 0 0 0 1 0
|
||||
1 99 99 0 1 1 0 0 0 1 1
|
||||
1 100 100 0 1 1 0 0 1 0 0
|
||||
1 101 101 0 1 1 0 0 1 0 1
|
||||
1 102 102 0 1 1 0 0 1 1 0
|
||||
1 103 103 0 1 1 0 0 1 1 1
|
||||
1 104 104 0 1 1 0 1 0 0 0
|
||||
1 105 105 0 1 1 0 1 0 0 1
|
||||
1 106 106 0 1 1 0 1 0 1 0
|
||||
1 107 107 0 1 1 0 1 0 1 1
|
||||
1 108 108 0 1 1 0 1 1 0 0
|
||||
1 109 109 0 1 1 0 1 1 0 1
|
||||
1 110 110 0 1 1 0 1 1 1 0
|
||||
1 111 111 0 1 1 0 1 1 1 1
|
||||
1 112 112 0 1 1 1 0 0 0 0
|
||||
1 113 113 0 1 1 1 0 0 0 1
|
||||
1 114 114 0 1 1 1 0 0 1 0
|
||||
1 115 115 0 1 1 1 0 0 1 1
|
||||
1 116 116 0 1 1 1 0 1 0 0
|
||||
1 117 117 0 1 1 1 0 1 0 1
|
||||
1 118 118 0 1 1 1 0 1 1 0
|
||||
1 119 119 0 1 1 1 0 1 1 1
|
||||
1 120 120 0 1 1 1 1 0 0 0
|
||||
1 121 121 0 1 1 1 1 0 0 1
|
||||
1 122 122 0 1 1 1 1 0 1 0
|
||||
1 123 123 0 1 1 1 1 0 1 1
|
||||
1 124 124 0 1 1 1 1 1 0 0
|
||||
1 125 125 0 1 1 1 1 1 0 1
|
||||
1 126 126 0 1 1 1 1 1 1 0
|
||||
1 127 127 0 1 1 1 1 1 1 1
|
||||
1 128 128 1 0 0 0 0 0 0 0
|
||||
1 129 129 1 0 0 0 0 0 0 1
|
||||
1 130 130 1 0 0 0 0 0 1 0
|
||||
1 131 131 1 0 0 0 0 0 1 1
|
||||
1 132 132 1 0 0 0 0 1 0 0
|
||||
1 133 133 1 0 0 0 0 1 0 1
|
||||
1 134 134 1 0 0 0 0 1 1 0
|
||||
1 135 135 1 0 0 0 0 1 1 1
|
||||
1 136 136 1 0 0 0 1 0 0 0
|
||||
1 137 137 1 0 0 0 1 0 0 1
|
||||
1 138 138 1 0 0 0 1 0 1 0
|
||||
1 139 139 1 0 0 0 1 0 1 1
|
||||
1 140 140 1 0 0 0 1 1 0 0
|
||||
1 141 141 1 0 0 0 1 1 0 1
|
||||
1 142 142 1 0 0 0 1 1 1 0
|
||||
1 143 143 1 0 0 0 1 1 1 1
|
||||
1 144 144 1 0 0 1 0 0 0 0
|
||||
1 145 145 1 0 0 1 0 0 0 1
|
||||
1 146 146 1 0 0 1 0 0 1 0
|
||||
1 147 147 1 0 0 1 0 0 1 1
|
||||
1 148 148 1 0 0 1 0 1 0 0
|
||||
1 149 149 1 0 0 1 0 1 0 1
|
||||
1 150 150 1 0 0 1 0 1 1 0
|
||||
1 151 151 1 0 0 1 0 1 1 1
|
||||
1 152 152 1 0 0 1 1 0 0 0
|
||||
1 153 153 1 0 0 1 1 0 0 1
|
||||
1 154 154 1 0 0 1 1 0 1 0
|
||||
1 155 155 1 0 0 1 1 0 1 1
|
||||
1 156 156 1 0 0 1 1 1 0 0
|
||||
1 157 157 1 0 0 1 1 1 0 1
|
||||
1 158 158 1 0 0 1 1 1 1 0
|
||||
1 159 159 1 0 0 1 1 1 1 1
|
||||
1 160 160 1 0 1 0 0 0 0 0
|
||||
1 161 161 1 0 1 0 0 0 0 1
|
||||
1 162 162 1 0 1 0 0 0 1 0
|
||||
1 163 163 1 0 1 0 0 0 1 1
|
||||
1 164 164 1 0 1 0 0 1 0 0
|
||||
1 165 165 1 0 1 0 0 1 0 1
|
||||
1 166 166 1 0 1 0 0 1 1 0
|
||||
1 167 167 1 0 1 0 0 1 1 1
|
||||
1 168 168 1 0 1 0 1 0 0 0
|
||||
1 169 169 1 0 1 0 1 0 0 1
|
||||
1 170 170 1 0 1 0 1 0 1 0
|
||||
1 171 171 1 0 1 0 1 0 1 1
|
||||
1 172 172 1 0 1 0 1 1 0 0
|
||||
1 173 173 1 0 1 0 1 1 0 1
|
||||
1 174 174 1 0 1 0 1 1 1 0
|
||||
1 175 175 1 0 1 0 1 1 1 1
|
||||
1 176 176 1 0 1 1 0 0 0 0
|
||||
1 177 177 1 0 1 1 0 0 0 1
|
||||
1 178 178 1 0 1 1 0 0 1 0
|
||||
1 179 179 1 0 1 1 0 0 1 1
|
||||
1 180 180 1 0 1 1 0 1 0 0
|
||||
1 181 181 1 0 1 1 0 1 0 1
|
||||
1 182 182 1 0 1 1 0 1 1 0
|
||||
1 183 183 1 0 1 1 0 1 1 1
|
||||
1 184 184 1 0 1 1 1 0 0 0
|
||||
1 185 185 1 0 1 1 1 0 0 1
|
||||
1 186 186 1 0 1 1 1 0 1 0
|
||||
1 187 187 1 0 1 1 1 0 1 1
|
||||
1 188 188 1 0 1 1 1 1 0 0
|
||||
1 189 189 1 0 1 1 1 1 0 1
|
||||
1 190 190 1 0 1 1 1 1 1 0
|
||||
1 191 191 1 0 1 1 1 1 1 1
|
||||
1 192 192 1 1 0 0 0 0 0 0
|
||||
1 193 193 1 1 0 0 0 0 0 1
|
||||
1 194 194 1 1 0 0 0 0 1 0
|
||||
1 195 195 1 1 0 0 0 0 1 1
|
||||
1 196 196 1 1 0 0 0 1 0 0
|
||||
1 197 197 1 1 0 0 0 1 0 1
|
||||
1 198 198 1 1 0 0 0 1 1 0
|
||||
1 199 199 1 1 0 0 0 1 1 1
|
||||
1 200 200 1 1 0 0 1 0 0 0
|
||||
1 201 201 1 1 0 0 1 0 0 1
|
||||
1 202 202 1 1 0 0 1 0 1 0
|
||||
1 203 203 1 1 0 0 1 0 1 1
|
||||
1 204 204 1 1 0 0 1 1 0 0
|
||||
1 205 205 1 1 0 0 1 1 0 1
|
||||
1 206 206 1 1 0 0 1 1 1 0
|
||||
1 207 207 1 1 0 0 1 1 1 1
|
||||
1 208 208 1 1 0 1 0 0 0 0
|
||||
1 209 209 1 1 0 1 0 0 0 1
|
||||
1 210 210 1 1 0 1 0 0 1 0
|
||||
1 211 211 1 1 0 1 0 0 1 1
|
||||
1 212 212 1 1 0 1 0 1 0 0
|
||||
1 213 213 1 1 0 1 0 1 0 1
|
||||
1 214 214 1 1 0 1 0 1 1 0
|
||||
1 215 215 1 1 0 1 0 1 1 1
|
||||
1 216 216 1 1 0 1 1 0 0 0
|
||||
1 217 217 1 1 0 1 1 0 0 1
|
||||
1 218 218 1 1 0 1 1 0 1 0
|
||||
1 219 219 1 1 0 1 1 0 1 1
|
||||
1 220 220 1 1 0 1 1 1 0 0
|
||||
1 221 221 1 1 0 1 1 1 0 1
|
||||
1 222 222 1 1 0 1 1 1 1 0
|
||||
1 223 223 1 1 0 1 1 1 1 1
|
||||
1 224 224 1 1 1 0 0 0 0 0
|
||||
1 225 225 1 1 1 0 0 0 0 1
|
||||
1 226 226 1 1 1 0 0 0 1 0
|
||||
1 227 227 1 1 1 0 0 0 1 1
|
||||
1 228 228 1 1 1 0 0 1 0 0
|
||||
1 229 229 1 1 1 0 0 1 0 1
|
||||
1 230 230 1 1 1 0 0 1 1 0
|
||||
1 231 231 1 1 1 0 0 1 1 1
|
||||
1 232 232 1 1 1 0 1 0 0 0
|
||||
1 233 233 1 1 1 0 1 0 0 1
|
||||
1 234 234 1 1 1 0 1 0 1 0
|
||||
1 235 235 1 1 1 0 1 0 1 1
|
||||
1 236 236 1 1 1 0 1 1 0 0
|
||||
1 237 237 1 1 1 0 1 1 0 1
|
||||
1 238 238 1 1 1 0 1 1 1 0
|
||||
1 239 239 1 1 1 0 1 1 1 1
|
||||
1 240 240 1 1 1 1 0 0 0 0
|
||||
1 241 241 1 1 1 1 0 0 0 1
|
||||
1 242 242 1 1 1 1 0 0 1 0
|
||||
1 243 243 1 1 1 1 0 0 1 1
|
||||
1 244 244 1 1 1 1 0 1 0 0
|
||||
1 245 245 1 1 1 1 0 1 0 1
|
||||
1 246 246 1 1 1 1 0 1 1 0
|
||||
1 247 247 1 1 1 1 0 1 1 1
|
||||
1 248 248 1 1 1 1 1 0 0 0
|
||||
1 249 249 1 1 1 1 1 0 0 1
|
||||
1 250 250 1 1 1 1 1 0 1 0
|
||||
1 251 251 1 1 1 1 1 0 1 1
|
||||
1 252 252 1 1 1 1 1 1 0 0
|
||||
1 253 253 1 1 1 1 1 1 0 1
|
||||
1 254 254 1 1 1 1 1 1 1 0
|
||||
1 255 255 1 1 1 1 1 1 1 1
|
||||
1 0 0 0 0 0 0 0 0 0 0
|
||||
1 1 1 0 0 0 0 0 0 0 1
|
||||
1 2 2 0 0 0 0 0 0 1 0
|
||||
1 3 3 0 0 0 0 0 0 1 1
|
||||
1 4 4 0 0 0 0 0 1 0 0
|
||||
1 5 5 0 0 0 0 0 1 0 1
|
||||
1 6 6 0 0 0 0 0 1 1 0
|
||||
1 7 7 0 0 0 0 0 1 1 1
|
||||
1 8 8 0 0 0 0 1 0 0 0
|
||||
1 9 9 0 0 0 0 1 0 0 1
|
||||
1 10 10 0 0 0 0 1 0 1 0
|
||||
1 11 11 0 0 0 0 1 0 1 1
|
||||
1 12 12 0 0 0 0 1 1 0 0
|
||||
1 13 13 0 0 0 0 1 1 0 1
|
||||
1 14 14 0 0 0 0 1 1 1 0
|
||||
1 15 15 0 0 0 0 1 1 1 1
|
||||
1 16 16 0 0 0 1 0 0 0 0
|
||||
1 17 17 0 0 0 1 0 0 0 1
|
||||
1 18 18 0 0 0 1 0 0 1 0
|
||||
1 19 19 0 0 0 1 0 0 1 1
|
||||
1 20 20 0 0 0 1 0 1 0 0
|
||||
1 21 21 0 0 0 1 0 1 0 1
|
||||
1 22 22 0 0 0 1 0 1 1 0
|
||||
1 23 23 0 0 0 1 0 1 1 1
|
||||
1 24 24 0 0 0 1 1 0 0 0
|
||||
1 25 25 0 0 0 1 1 0 0 1
|
||||
1 26 26 0 0 0 1 1 0 1 0
|
||||
1 27 27 0 0 0 1 1 0 1 1
|
||||
1 28 28 0 0 0 1 1 1 0 0
|
||||
1 29 29 0 0 0 1 1 1 0 1
|
||||
1 30 30 0 0 0 1 1 1 1 0
|
||||
1 31 31 0 0 0 1 1 1 1 1
|
||||
1 32 32 0 0 1 0 0 0 0 0
|
||||
1 33 33 0 0 1 0 0 0 0 1
|
||||
1 34 34 0 0 1 0 0 0 1 0
|
||||
1 35 35 0 0 1 0 0 0 1 1
|
||||
1 36 36 0 0 1 0 0 1 0 0
|
||||
1 37 37 0 0 1 0 0 1 0 1
|
||||
1 38 38 0 0 1 0 0 1 1 0
|
||||
1 39 39 0 0 1 0 0 1 1 1
|
||||
1 40 40 0 0 1 0 1 0 0 0
|
||||
1 41 41 0 0 1 0 1 0 0 1
|
||||
1 42 42 0 0 1 0 1 0 1 0
|
||||
1 43 43 0 0 1 0 1 0 1 1
|
||||
1 44 44 0 0 1 0 1 1 0 0
|
||||
1 45 45 0 0 1 0 1 1 0 1
|
||||
1 46 46 0 0 1 0 1 1 1 0
|
||||
1 47 47 0 0 1 0 1 1 1 1
|
||||
1 48 48 0 0 1 1 0 0 0 0
|
||||
1 49 49 0 0 1 1 0 0 0 1
|
||||
1 50 50 0 0 1 1 0 0 1 0
|
||||
1 51 51 0 0 1 1 0 0 1 1
|
||||
1 52 52 0 0 1 1 0 1 0 0
|
||||
1 53 53 0 0 1 1 0 1 0 1
|
||||
1 54 54 0 0 1 1 0 1 1 0
|
||||
1 55 55 0 0 1 1 0 1 1 1
|
||||
1 56 56 0 0 1 1 1 0 0 0
|
||||
1 57 57 0 0 1 1 1 0 0 1
|
||||
1 58 58 0 0 1 1 1 0 1 0
|
||||
1 59 59 0 0 1 1 1 0 1 1
|
||||
1 60 60 0 0 1 1 1 1 0 0
|
||||
1 61 61 0 0 1 1 1 1 0 1
|
||||
1 62 62 0 0 1 1 1 1 1 0
|
||||
1 63 63 0 0 1 1 1 1 1 1
|
||||
1 64 64 0 1 0 0 0 0 0 0
|
||||
1 65 65 0 1 0 0 0 0 0 1
|
||||
1 66 66 0 1 0 0 0 0 1 0
|
||||
1 67 67 0 1 0 0 0 0 1 1
|
||||
1 68 68 0 1 0 0 0 1 0 0
|
||||
1 69 69 0 1 0 0 0 1 0 1
|
||||
1 70 70 0 1 0 0 0 1 1 0
|
||||
1 71 71 0 1 0 0 0 1 1 1
|
||||
1 72 72 0 1 0 0 1 0 0 0
|
||||
1 73 73 0 1 0 0 1 0 0 1
|
||||
1 74 74 0 1 0 0 1 0 1 0
|
||||
1 75 75 0 1 0 0 1 0 1 1
|
||||
1 76 76 0 1 0 0 1 1 0 0
|
||||
1 77 77 0 1 0 0 1 1 0 1
|
||||
1 78 78 0 1 0 0 1 1 1 0
|
||||
1 79 79 0 1 0 0 1 1 1 1
|
||||
1 80 80 0 1 0 1 0 0 0 0
|
||||
1 81 81 0 1 0 1 0 0 0 1
|
||||
1 82 82 0 1 0 1 0 0 1 0
|
||||
1 83 83 0 1 0 1 0 0 1 1
|
||||
1 84 84 0 1 0 1 0 1 0 0
|
||||
1 85 85 0 1 0 1 0 1 0 1
|
||||
1 86 86 0 1 0 1 0 1 1 0
|
||||
1 87 87 0 1 0 1 0 1 1 1
|
||||
1 88 88 0 1 0 1 1 0 0 0
|
||||
1 89 89 0 1 0 1 1 0 0 1
|
||||
1 90 90 0 1 0 1 1 0 1 0
|
||||
1 91 91 0 1 0 1 1 0 1 1
|
||||
1 92 92 0 1 0 1 1 1 0 0
|
||||
1 93 93 0 1 0 1 1 1 0 1
|
||||
1 94 94 0 1 0 1 1 1 1 0
|
||||
1 95 95 0 1 0 1 1 1 1 1
|
||||
1 96 96 0 1 1 0 0 0 0 0
|
||||
1 97 97 0 1 1 0 0 0 0 1
|
||||
1 98 98 0 1 1 0 0 0 1 0
|
||||
1 99 99 0 1 1 0 0 0 1 1
|
||||
1 100 100 0 1 1 0 0 1 0 0
|
||||
1 101 101 0 1 1 0 0 1 0 1
|
||||
1 102 102 0 1 1 0 0 1 1 0
|
||||
1 103 103 0 1 1 0 0 1 1 1
|
||||
1 104 104 0 1 1 0 1 0 0 0
|
||||
1 105 105 0 1 1 0 1 0 0 1
|
||||
1 106 106 0 1 1 0 1 0 1 0
|
||||
1 107 107 0 1 1 0 1 0 1 1
|
||||
1 108 108 0 1 1 0 1 1 0 0
|
||||
1 109 109 0 1 1 0 1 1 0 1
|
||||
1 110 110 0 1 1 0 1 1 1 0
|
||||
1 111 111 0 1 1 0 1 1 1 1
|
||||
1 112 112 0 1 1 1 0 0 0 0
|
||||
1 113 113 0 1 1 1 0 0 0 1
|
||||
1 114 114 0 1 1 1 0 0 1 0
|
||||
1 115 115 0 1 1 1 0 0 1 1
|
||||
1 116 116 0 1 1 1 0 1 0 0
|
||||
1 117 117 0 1 1 1 0 1 0 1
|
||||
1 118 118 0 1 1 1 0 1 1 0
|
||||
1 119 119 0 1 1 1 0 1 1 1
|
||||
1 120 120 0 1 1 1 1 0 0 0
|
||||
1 121 121 0 1 1 1 1 0 0 1
|
||||
1 122 122 0 1 1 1 1 0 1 0
|
||||
1 123 123 0 1 1 1 1 0 1 1
|
||||
1 124 124 0 1 1 1 1 1 0 0
|
||||
1 125 125 0 1 1 1 1 1 0 1
|
||||
1 126 126 0 1 1 1 1 1 1 0
|
||||
1 127 127 0 1 1 1 1 1 1 1
|
||||
1 128 128 1 0 0 0 0 0 0 0
|
||||
1 129 129 1 0 0 0 0 0 0 1
|
||||
1 130 130 1 0 0 0 0 0 1 0
|
||||
1 131 131 1 0 0 0 0 0 1 1
|
||||
1 132 132 1 0 0 0 0 1 0 0
|
||||
1 133 133 1 0 0 0 0 1 0 1
|
||||
1 134 134 1 0 0 0 0 1 1 0
|
||||
1 135 135 1 0 0 0 0 1 1 1
|
||||
1 136 136 1 0 0 0 1 0 0 0
|
||||
1 137 137 1 0 0 0 1 0 0 1
|
||||
1 138 138 1 0 0 0 1 0 1 0
|
||||
1 139 139 1 0 0 0 1 0 1 1
|
||||
1 140 140 1 0 0 0 1 1 0 0
|
||||
1 141 141 1 0 0 0 1 1 0 1
|
||||
1 142 142 1 0 0 0 1 1 1 0
|
||||
1 143 143 1 0 0 0 1 1 1 1
|
||||
1 144 144 1 0 0 1 0 0 0 0
|
||||
1 145 145 1 0 0 1 0 0 0 1
|
||||
1 146 146 1 0 0 1 0 0 1 0
|
||||
1 147 147 1 0 0 1 0 0 1 1
|
||||
1 148 148 1 0 0 1 0 1 0 0
|
||||
1 149 149 1 0 0 1 0 1 0 1
|
||||
1 150 150 1 0 0 1 0 1 1 0
|
||||
1 151 151 1 0 0 1 0 1 1 1
|
||||
1 152 152 1 0 0 1 1 0 0 0
|
||||
1 153 153 1 0 0 1 1 0 0 1
|
||||
1 154 154 1 0 0 1 1 0 1 0
|
||||
1 155 155 1 0 0 1 1 0 1 1
|
||||
1 156 156 1 0 0 1 1 1 0 0
|
||||
1 157 157 1 0 0 1 1 1 0 1
|
||||
1 158 158 1 0 0 1 1 1 1 0
|
||||
1 159 159 1 0 0 1 1 1 1 1
|
||||
1 160 160 1 0 1 0 0 0 0 0
|
||||
1 161 161 1 0 1 0 0 0 0 1
|
||||
1 162 162 1 0 1 0 0 0 1 0
|
||||
1 163 163 1 0 1 0 0 0 1 1
|
||||
1 164 164 1 0 1 0 0 1 0 0
|
||||
1 165 165 1 0 1 0 0 1 0 1
|
||||
1 166 166 1 0 1 0 0 1 1 0
|
||||
1 167 167 1 0 1 0 0 1 1 1
|
||||
1 168 168 1 0 1 0 1 0 0 0
|
||||
1 169 169 1 0 1 0 1 0 0 1
|
||||
1 170 170 1 0 1 0 1 0 1 0
|
||||
1 171 171 1 0 1 0 1 0 1 1
|
||||
1 172 172 1 0 1 0 1 1 0 0
|
||||
1 173 173 1 0 1 0 1 1 0 1
|
||||
1 174 174 1 0 1 0 1 1 1 0
|
||||
1 175 175 1 0 1 0 1 1 1 1
|
||||
1 176 176 1 0 1 1 0 0 0 0
|
||||
1 177 177 1 0 1 1 0 0 0 1
|
||||
1 178 178 1 0 1 1 0 0 1 0
|
||||
1 179 179 1 0 1 1 0 0 1 1
|
||||
1 180 180 1 0 1 1 0 1 0 0
|
||||
1 181 181 1 0 1 1 0 1 0 1
|
||||
1 182 182 1 0 1 1 0 1 1 0
|
||||
1 183 183 1 0 1 1 0 1 1 1
|
||||
1 184 184 1 0 1 1 1 0 0 0
|
||||
1 185 185 1 0 1 1 1 0 0 1
|
||||
1 186 186 1 0 1 1 1 0 1 0
|
||||
1 187 187 1 0 1 1 1 0 1 1
|
||||
1 188 188 1 0 1 1 1 1 0 0
|
||||
1 189 189 1 0 1 1 1 1 0 1
|
||||
1 190 190 1 0 1 1 1 1 1 0
|
||||
1 191 191 1 0 1 1 1 1 1 1
|
||||
1 192 192 1 1 0 0 0 0 0 0
|
||||
1 193 193 1 1 0 0 0 0 0 1
|
||||
1 194 194 1 1 0 0 0 0 1 0
|
||||
1 195 195 1 1 0 0 0 0 1 1
|
||||
1 196 196 1 1 0 0 0 1 0 0
|
||||
1 197 197 1 1 0 0 0 1 0 1
|
||||
1 198 198 1 1 0 0 0 1 1 0
|
||||
1 199 199 1 1 0 0 0 1 1 1
|
||||
1 200 200 1 1 0 0 1 0 0 0
|
||||
1 201 201 1 1 0 0 1 0 0 1
|
||||
1 202 202 1 1 0 0 1 0 1 0
|
||||
1 203 203 1 1 0 0 1 0 1 1
|
||||
1 204 204 1 1 0 0 1 1 0 0
|
||||
1 205 205 1 1 0 0 1 1 0 1
|
||||
1 206 206 1 1 0 0 1 1 1 0
|
||||
1 207 207 1 1 0 0 1 1 1 1
|
||||
1 208 208 1 1 0 1 0 0 0 0
|
||||
1 209 209 1 1 0 1 0 0 0 1
|
||||
1 210 210 1 1 0 1 0 0 1 0
|
||||
1 211 211 1 1 0 1 0 0 1 1
|
||||
1 212 212 1 1 0 1 0 1 0 0
|
||||
1 213 213 1 1 0 1 0 1 0 1
|
||||
1 214 214 1 1 0 1 0 1 1 0
|
||||
1 215 215 1 1 0 1 0 1 1 1
|
||||
1 216 216 1 1 0 1 1 0 0 0
|
||||
1 217 217 1 1 0 1 1 0 0 1
|
||||
1 218 218 1 1 0 1 1 0 1 0
|
||||
1 219 219 1 1 0 1 1 0 1 1
|
||||
1 220 220 1 1 0 1 1 1 0 0
|
||||
1 221 221 1 1 0 1 1 1 0 1
|
||||
1 222 222 1 1 0 1 1 1 1 0
|
||||
1 223 223 1 1 0 1 1 1 1 1
|
||||
1 224 224 1 1 1 0 0 0 0 0
|
||||
1 225 225 1 1 1 0 0 0 0 1
|
||||
1 226 226 1 1 1 0 0 0 1 0
|
||||
1 227 227 1 1 1 0 0 0 1 1
|
||||
1 228 228 1 1 1 0 0 1 0 0
|
||||
1 229 229 1 1 1 0 0 1 0 1
|
||||
1 230 230 1 1 1 0 0 1 1 0
|
||||
1 231 231 1 1 1 0 0 1 1 1
|
||||
1 232 232 1 1 1 0 1 0 0 0
|
||||
1 233 233 1 1 1 0 1 0 0 1
|
||||
1 234 234 1 1 1 0 1 0 1 0
|
||||
1 235 235 1 1 1 0 1 0 1 1
|
||||
1 236 236 1 1 1 0 1 1 0 0
|
||||
1 237 237 1 1 1 0 1 1 0 1
|
||||
1 238 238 1 1 1 0 1 1 1 0
|
||||
1 239 239 1 1 1 0 1 1 1 1
|
||||
1 240 240 1 1 1 1 0 0 0 0
|
||||
1 241 241 1 1 1 1 0 0 0 1
|
||||
1 242 242 1 1 1 1 0 0 1 0
|
||||
1 243 243 1 1 1 1 0 0 1 1
|
||||
1 244 244 1 1 1 1 0 1 0 0
|
||||
1 245 245 1 1 1 1 0 1 0 1
|
||||
1 246 246 1 1 1 1 0 1 1 0
|
||||
1 247 247 1 1 1 1 0 1 1 1
|
||||
1 248 248 1 1 1 1 1 0 0 0
|
||||
1 249 249 1 1 1 1 1 0 0 1
|
||||
1 250 250 1 1 1 1 1 0 1 0
|
||||
1 251 251 1 1 1 1 1 0 1 1
|
||||
1 252 252 1 1 1 1 1 1 0 0
|
||||
1 253 253 1 1 1 1 1 1 0 1
|
||||
1 254 254 1 1 1 1 1 1 1 0
|
||||
1 255 255 1 1 1 1 1 1 1 1
|
||||
1 0 0 0 0 0 0 0 0 0 0
|
||||
1 1 1 0 0 0 0 0 0 0 1
|
||||
1 2 2 0 0 0 0 0 0 1 0
|
||||
1 3 3 0 0 0 0 0 0 1 1
|
||||
1 4 4 0 0 0 0 0 1 0 0
|
||||
1 5 5 0 0 0 0 0 1 0 1
|
||||
1 6 6 0 0 0 0 0 1 1 0
|
||||
1 7 7 0 0 0 0 0 1 1 1
|
||||
1 8 8 0 0 0 0 1 0 0 0
|
||||
1 9 9 0 0 0 0 1 0 0 1
|
||||
1 10 10 0 0 0 0 1 0 1 0
|
||||
1 11 11 0 0 0 0 1 0 1 1
|
||||
1 12 12 0 0 0 0 1 1 0 0
|
||||
1 13 13 0 0 0 0 1 1 0 1
|
||||
1 14 14 0 0 0 0 1 1 1 0
|
||||
1 15 15 0 0 0 0 1 1 1 1
|
||||
1 16 16 0 0 0 1 0 0 0 0
|
||||
1 17 17 0 0 0 1 0 0 0 1
|
||||
1 18 18 0 0 0 1 0 0 1 0
|
||||
1 19 19 0 0 0 1 0 0 1 1
|
||||
1 20 20 0 0 0 1 0 1 0 0
|
||||
1 21 21 0 0 0 1 0 1 0 1
|
||||
1 22 22 0 0 0 1 0 1 1 0
|
||||
1 23 23 0 0 0 1 0 1 1 1
|
||||
1 24 24 0 0 0 1 1 0 0 0
|
||||
1 25 25 0 0 0 1 1 0 0 1
|
||||
1 26 26 0 0 0 1 1 0 1 0
|
||||
1 27 27 0 0 0 1 1 0 1 1
|
||||
1 28 28 0 0 0 1 1 1 0 0
|
||||
1 29 29 0 0 0 1 1 1 0 1
|
||||
1 30 30 0 0 0 1 1 1 1 0
|
||||
1 31 31 0 0 0 1 1 1 1 1
|
||||
1 32 32 0 0 1 0 0 0 0 0
|
||||
1 33 33 0 0 1 0 0 0 0 1
|
||||
1 34 34 0 0 1 0 0 0 1 0
|
||||
1 35 35 0 0 1 0 0 0 1 1
|
||||
1 36 36 0 0 1 0 0 1 0 0
|
||||
1 37 37 0 0 1 0 0 1 0 1
|
||||
1 38 38 0 0 1 0 0 1 1 0
|
||||
1 39 39 0 0 1 0 0 1 1 1
|
||||
1 40 40 0 0 1 0 1 0 0 0
|
||||
1 41 41 0 0 1 0 1 0 0 1
|
||||
1 42 42 0 0 1 0 1 0 1 0
|
||||
1 43 43 0 0 1 0 1 0 1 1
|
||||
1 44 44 0 0 1 0 1 1 0 0
|
||||
1 45 45 0 0 1 0 1 1 0 1
|
||||
1 46 46 0 0 1 0 1 1 1 0
|
||||
1 47 47 0 0 1 0 1 1 1 1
|
||||
1 48 48 0 0 1 1 0 0 0 0
|
||||
1 49 49 0 0 1 1 0 0 0 1
|
||||
1 50 50 0 0 1 1 0 0 1 0
|
||||
1 51 51 0 0 1 1 0 0 1 1
|
||||
1 52 52 0 0 1 1 0 1 0 0
|
||||
1 53 53 0 0 1 1 0 1 0 1
|
||||
1 54 54 0 0 1 1 0 1 1 0
|
||||
1 55 55 0 0 1 1 0 1 1 1
|
||||
1 56 56 0 0 1 1 1 0 0 0
|
||||
1 57 57 0 0 1 1 1 0 0 1
|
||||
1 58 58 0 0 1 1 1 0 1 0
|
||||
1 59 59 0 0 1 1 1 0 1 1
|
||||
1 60 60 0 0 1 1 1 1 0 0
|
||||
1 61 61 0 0 1 1 1 1 0 1
|
||||
1 62 62 0 0 1 1 1 1 1 0
|
||||
1 63 63 0 0 1 1 1 1 1 1
|
||||
1 64 64 0 1 0 0 0 0 0 0
|
||||
1 65 65 0 1 0 0 0 0 0 1
|
||||
1 66 66 0 1 0 0 0 0 1 0
|
||||
1 67 67 0 1 0 0 0 0 1 1
|
||||
1 68 68 0 1 0 0 0 1 0 0
|
||||
1 69 69 0 1 0 0 0 1 0 1
|
||||
1 70 70 0 1 0 0 0 1 1 0
|
||||
1 71 71 0 1 0 0 0 1 1 1
|
||||
1 72 72 0 1 0 0 1 0 0 0
|
||||
1 73 73 0 1 0 0 1 0 0 1
|
||||
1 74 74 0 1 0 0 1 0 1 0
|
||||
1 75 75 0 1 0 0 1 0 1 1
|
||||
1 76 76 0 1 0 0 1 1 0 0
|
||||
1 77 77 0 1 0 0 1 1 0 1
|
||||
1 78 78 0 1 0 0 1 1 1 0
|
||||
1 79 79 0 1 0 0 1 1 1 1
|
||||
1 80 80 0 1 0 1 0 0 0 0
|
||||
1 81 81 0 1 0 1 0 0 0 1
|
||||
1 82 82 0 1 0 1 0 0 1 0
|
||||
1 83 83 0 1 0 1 0 0 1 1
|
||||
1 84 84 0 1 0 1 0 1 0 0
|
||||
1 85 85 0 1 0 1 0 1 0 1
|
||||
1 86 86 0 1 0 1 0 1 1 0
|
||||
1 87 87 0 1 0 1 0 1 1 1
|
||||
1 88 88 0 1 0 1 1 0 0 0
|
||||
1 89 89 0 1 0 1 1 0 0 1
|
||||
1 90 90 0 1 0 1 1 0 1 0
|
||||
1 91 91 0 1 0 1 1 0 1 1
|
||||
1 92 92 0 1 0 1 1 1 0 0
|
||||
1 93 93 0 1 0 1 1 1 0 1
|
||||
1 94 94 0 1 0 1 1 1 1 0
|
||||
1 95 95 0 1 0 1 1 1 1 1
|
||||
1 96 96 0 1 1 0 0 0 0 0
|
||||
1 97 97 0 1 1 0 0 0 0 1
|
||||
1 98 98 0 1 1 0 0 0 1 0
|
||||
1 99 99 0 1 1 0 0 0 1 1
|
||||
1 100 100 0 1 1 0 0 1 0 0
|
||||
1 101 101 0 1 1 0 0 1 0 1
|
||||
1 102 102 0 1 1 0 0 1 1 0
|
||||
1 103 103 0 1 1 0 0 1 1 1
|
||||
1 104 104 0 1 1 0 1 0 0 0
|
||||
1 105 105 0 1 1 0 1 0 0 1
|
||||
1 106 106 0 1 1 0 1 0 1 0
|
||||
1 107 107 0 1 1 0 1 0 1 1
|
||||
1 108 108 0 1 1 0 1 1 0 0
|
||||
1 109 109 0 1 1 0 1 1 0 1
|
||||
1 110 110 0 1 1 0 1 1 1 0
|
||||
1 111 111 0 1 1 0 1 1 1 1
|
||||
1 112 112 0 1 1 1 0 0 0 0
|
||||
1 113 113 0 1 1 1 0 0 0 1
|
||||
1 114 114 0 1 1 1 0 0 1 0
|
||||
1 115 115 0 1 1 1 0 0 1 1
|
||||
1 116 116 0 1 1 1 0 1 0 0
|
||||
1 117 117 0 1 1 1 0 1 0 1
|
||||
1 118 118 0 1 1 1 0 1 1 0
|
||||
1 119 119 0 1 1 1 0 1 1 1
|
||||
1 120 120 0 1 1 1 1 0 0 0
|
||||
1 121 121 0 1 1 1 1 0 0 1
|
||||
1 122 122 0 1 1 1 1 0 1 0
|
||||
1 123 123 0 1 1 1 1 0 1 1
|
||||
1 124 124 0 1 1 1 1 1 0 0
|
||||
1 125 125 0 1 1 1 1 1 0 1
|
||||
1 126 126 0 1 1 1 1 1 1 0
|
||||
1 127 127 0 1 1 1 1 1 1 1
|
||||
1 128 128 1 0 0 0 0 0 0 0
|
||||
1 129 129 1 0 0 0 0 0 0 1
|
||||
1 130 130 1 0 0 0 0 0 1 0
|
||||
1 131 131 1 0 0 0 0 0 1 1
|
||||
1 132 132 1 0 0 0 0 1 0 0
|
||||
1 133 133 1 0 0 0 0 1 0 1
|
||||
1 134 134 1 0 0 0 0 1 1 0
|
||||
1 135 135 1 0 0 0 0 1 1 1
|
||||
1 136 136 1 0 0 0 1 0 0 0
|
||||
1 137 137 1 0 0 0 1 0 0 1
|
||||
1 138 138 1 0 0 0 1 0 1 0
|
||||
1 139 139 1 0 0 0 1 0 1 1
|
||||
1 140 140 1 0 0 0 1 1 0 0
|
||||
1 141 141 1 0 0 0 1 1 0 1
|
||||
1 142 142 1 0 0 0 1 1 1 0
|
||||
1 143 143 1 0 0 0 1 1 1 1
|
||||
1 144 144 1 0 0 1 0 0 0 0
|
||||
1 145 145 1 0 0 1 0 0 0 1
|
||||
1 146 146 1 0 0 1 0 0 1 0
|
||||
1 147 147 1 0 0 1 0 0 1 1
|
||||
1 148 148 1 0 0 1 0 1 0 0
|
||||
1 149 149 1 0 0 1 0 1 0 1
|
||||
1 150 150 1 0 0 1 0 1 1 0
|
||||
1 151 151 1 0 0 1 0 1 1 1
|
||||
1 152 152 1 0 0 1 1 0 0 0
|
||||
1 153 153 1 0 0 1 1 0 0 1
|
||||
1 154 154 1 0 0 1 1 0 1 0
|
||||
1 155 155 1 0 0 1 1 0 1 1
|
||||
1 156 156 1 0 0 1 1 1 0 0
|
||||
1 157 157 1 0 0 1 1 1 0 1
|
||||
1 158 158 1 0 0 1 1 1 1 0
|
||||
1 159 159 1 0 0 1 1 1 1 1
|
||||
1 160 160 1 0 1 0 0 0 0 0
|
||||
1 161 161 1 0 1 0 0 0 0 1
|
||||
1 162 162 1 0 1 0 0 0 1 0
|
||||
1 163 163 1 0 1 0 0 0 1 1
|
||||
1 164 164 1 0 1 0 0 1 0 0
|
||||
1 165 165 1 0 1 0 0 1 0 1
|
||||
1 166 166 1 0 1 0 0 1 1 0
|
||||
1 167 167 1 0 1 0 0 1 1 1
|
||||
1 168 168 1 0 1 0 1 0 0 0
|
||||
1 169 169 1 0 1 0 1 0 0 1
|
||||
1 170 170 1 0 1 0 1 0 1 0
|
||||
1 171 171 1 0 1 0 1 0 1 1
|
||||
1 172 172 1 0 1 0 1 1 0 0
|
||||
1 173 173 1 0 1 0 1 1 0 1
|
||||
1 174 174 1 0 1 0 1 1 1 0
|
||||
1 175 175 1 0 1 0 1 1 1 1
|
||||
1 176 176 1 0 1 1 0 0 0 0
|
||||
1 177 177 1 0 1 1 0 0 0 1
|
||||
1 178 178 1 0 1 1 0 0 1 0
|
||||
1 179 179 1 0 1 1 0 0 1 1
|
||||
1 180 180 1 0 1 1 0 1 0 0
|
||||
1 181 181 1 0 1 1 0 1 0 1
|
||||
1 182 182 1 0 1 1 0 1 1 0
|
||||
1 183 183 1 0 1 1 0 1 1 1
|
||||
1 184 184 1 0 1 1 1 0 0 0
|
||||
1 185 185 1 0 1 1 1 0 0 1
|
||||
1 186 186 1 0 1 1 1 0 1 0
|
||||
1 187 187 1 0 1 1 1 0 1 1
|
||||
1 188 188 1 0 1 1 1 1 0 0
|
||||
1 189 189 1 0 1 1 1 1 0 1
|
||||
1 190 190 1 0 1 1 1 1 1 0
|
||||
1 191 191 1 0 1 1 1 1 1 1
|
||||
1 192 192 1 1 0 0 0 0 0 0
|
||||
1 193 193 1 1 0 0 0 0 0 1
|
||||
1 194 194 1 1 0 0 0 0 1 0
|
||||
1 195 195 1 1 0 0 0 0 1 1
|
||||
1 196 196 1 1 0 0 0 1 0 0
|
||||
1 197 197 1 1 0 0 0 1 0 1
|
||||
1 198 198 1 1 0 0 0 1 1 0
|
||||
1 199 199 1 1 0 0 0 1 1 1
|
||||
1 200 200 1 1 0 0 1 0 0 0
|
||||
1 201 201 1 1 0 0 1 0 0 1
|
||||
1 202 202 1 1 0 0 1 0 1 0
|
||||
1 203 203 1 1 0 0 1 0 1 1
|
||||
1 204 204 1 1 0 0 1 1 0 0
|
||||
1 205 205 1 1 0 0 1 1 0 1
|
||||
1 206 206 1 1 0 0 1 1 1 0
|
||||
1 207 207 1 1 0 0 1 1 1 1
|
||||
1 208 208 1 1 0 1 0 0 0 0
|
||||
1 209 209 1 1 0 1 0 0 0 1
|
||||
1 210 210 1 1 0 1 0 0 1 0
|
||||
1 211 211 1 1 0 1 0 0 1 1
|
||||
1 212 212 1 1 0 1 0 1 0 0
|
||||
1 213 213 1 1 0 1 0 1 0 1
|
||||
1 214 214 1 1 0 1 0 1 1 0
|
||||
1 215 215 1 1 0 1 0 1 1 1
|
||||
1 216 216 1 1 0 1 1 0 0 0
|
||||
1 217 217 1 1 0 1 1 0 0 1
|
||||
1 218 218 1 1 0 1 1 0 1 0
|
||||
1 219 219 1 1 0 1 1 0 1 1
|
||||
1 220 220 1 1 0 1 1 1 0 0
|
||||
1 221 221 1 1 0 1 1 1 0 1
|
||||
1 222 222 1 1 0 1 1 1 1 0
|
||||
1 223 223 1 1 0 1 1 1 1 1
|
||||
1 224 224 1 1 1 0 0 0 0 0
|
||||
1 225 225 1 1 1 0 0 0 0 1
|
||||
1 226 226 1 1 1 0 0 0 1 0
|
||||
1 227 227 1 1 1 0 0 0 1 1
|
||||
1 228 228 1 1 1 0 0 1 0 0
|
||||
1 229 229 1 1 1 0 0 1 0 1
|
||||
1 230 230 1 1 1 0 0 1 1 0
|
||||
1 231 231 1 1 1 0 0 1 1 1
|
||||
1 232 232 1 1 1 0 1 0 0 0
|
||||
1 233 233 1 1 1 0 1 0 0 1
|
||||
1 234 234 1 1 1 0 1 0 1 0
|
||||
1 235 235 1 1 1 0 1 0 1 1
|
||||
1 236 236 1 1 1 0 1 1 0 0
|
||||
1 237 237 1 1 1 0 1 1 0 1
|
||||
1 238 238 1 1 1 0 1 1 1 0
|
||||
1 239 239 1 1 1 0 1 1 1 1
|
||||
1 240 240 1 1 1 1 0 0 0 0
|
||||
1 241 241 1 1 1 1 0 0 0 1
|
||||
1 242 242 1 1 1 1 0 0 1 0
|
||||
1 243 243 1 1 1 1 0 0 1 1
|
||||
1 244 244 1 1 1 1 0 1 0 0
|
||||
1 245 245 1 1 1 1 0 1 0 1
|
||||
1 246 246 1 1 1 1 0 1 1 0
|
||||
1 247 247 1 1 1 1 0 1 1 1
|
||||
1 248 248 1 1 1 1 1 0 0 0
|
||||
1 249 249 1 1 1 1 1 0 0 1
|
||||
1 250 250 1 1 1 1 1 0 1 0
|
||||
1 251 251 1 1 1 1 1 0 1 1
|
||||
1 252 252 1 1 1 1 1 1 0 0
|
||||
1 253 253 1 1 1 1 1 1 0 1
|
||||
1 254 254 1 1 1 1 1 1 1 0
|
||||
1 255 255 1 1 1 1 1 1 1 1
|
@ -0,0 +1,115 @@
|
||||
select
|
||||
bitTest(0, 0) = 0,
|
||||
bitTest(1, 0) = 1,
|
||||
bitTest(1, 1) = 0,
|
||||
bitTest(0xff, 7) = 1;
|
||||
|
||||
select
|
||||
bitTestAll(0, 0) = 0,
|
||||
bitTestAll(1, 0) = 1,
|
||||
bitTestAll(1, 1) = 0,
|
||||
bitTestAll(0xff, 0) = 1,
|
||||
bitTestAll(0xff, 1) = 1,
|
||||
bitTestAll(0xff, 2) = 1,
|
||||
bitTestAll(0xff, 3) = 1,
|
||||
bitTestAll(0xff, 4) = 1,
|
||||
bitTestAll(0xff, 5) = 1,
|
||||
bitTestAll(0xff, 6) = 1,
|
||||
bitTestAll(0xff, 7) = 1,
|
||||
bitTestAll(0xff, 0, 1) = 1,
|
||||
bitTestAll(0xff, 2, 3) = 1,
|
||||
bitTestAll(0xff, 4, 5) = 1,
|
||||
bitTestAll(0xff, 6, 7) = 1,
|
||||
bitTestAll(0xff, 0, 1, 2, 3) = 1,
|
||||
bitTestAll(0xff, 4, 5, 6, 7) = 1,
|
||||
bitTestAll(0xff, 0, 1, 2, 3, 4, 5, 6, 7) = 1,
|
||||
bitTestAll(0x81, 0) = 1,
|
||||
bitTestAll(0x81, 1) = 0,
|
||||
bitTestAll(0x81, 2) = 0,
|
||||
bitTestAll(0x81, 3) = 0,
|
||||
bitTestAll(0x81, 4) = 0,
|
||||
bitTestAll(0x81, 5) = 0,
|
||||
bitTestAll(0x81, 6) = 0,
|
||||
bitTestAll(0x81, 7) = 1,
|
||||
bitTestAll(0x81, 0, 1) = 0,
|
||||
bitTestAll(0x81, 2, 3) = 0,
|
||||
bitTestAll(0x81, 4, 5) = 0,
|
||||
bitTestAll(0x81, 6, 7) = 0,
|
||||
bitTestAll(0x81, 0, 1, 2, 3) = 0,
|
||||
bitTestAll(0x81, 4, 5, 6, 7) = 0,
|
||||
bitTestAll(0x81, 0, 1, 2, 3, 4, 5, 6, 7) = 0,
|
||||
bitTestAll(0x81, 0, 7) = 1;
|
||||
|
||||
select
|
||||
bitTestAny(0, 0) = 0,
|
||||
bitTestAny(1, 0) = 1,
|
||||
bitTestAny(1, 1) = 0,
|
||||
bitTestAny(0xff, 0) = 1,
|
||||
bitTestAny(0xff, 1) = 1,
|
||||
bitTestAny(0xff, 2) = 1,
|
||||
bitTestAny(0xff, 3) = 1,
|
||||
bitTestAny(0xff, 4) = 1,
|
||||
bitTestAny(0xff, 5) = 1,
|
||||
bitTestAny(0xff, 6) = 1,
|
||||
bitTestAny(0xff, 7) = 1,
|
||||
bitTestAny(0xff, 0, 1) = 1,
|
||||
bitTestAny(0xff, 2, 3) = 1,
|
||||
bitTestAny(0xff, 4, 5) = 1,
|
||||
bitTestAny(0xff, 6, 7) = 1,
|
||||
bitTestAny(0xff, 0, 1, 2, 3) = 1,
|
||||
bitTestAny(0xff, 4, 5, 6, 7) = 1,
|
||||
bitTestAny(0xff, 0, 1, 2, 3, 4, 5, 6, 7) = 1,
|
||||
bitTestAny(0x81, 0) = 1,
|
||||
bitTestAny(0x81, 1) = 0,
|
||||
bitTestAny(0x81, 2) = 0,
|
||||
bitTestAny(0x81, 3) = 0,
|
||||
bitTestAny(0x81, 4) = 0,
|
||||
bitTestAny(0x81, 5) = 0,
|
||||
bitTestAny(0x81, 6) = 0,
|
||||
bitTestAny(0x81, 7) = 1,
|
||||
bitTestAny(0x81, 0, 1) = 1,
|
||||
bitTestAny(0x81, 2, 3) = 0,
|
||||
bitTestAny(0x81, 4, 5) = 0,
|
||||
bitTestAny(0x81, 6, 7) = 1,
|
||||
bitTestAny(0x81, 0, 1, 2, 3) = 1,
|
||||
bitTestAny(0x81, 4, 5, 6, 7) = 1,
|
||||
bitTestAny(0x81, 0, 1, 2, 3, 4, 5, 6, 7) = 1;
|
||||
|
||||
select n = n_,
|
||||
number as n,
|
||||
bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(b7, 1), b6), 1), b5), 1), b4), 1), b3), 1), b2), 1), b1), 1), b0) as n_,
|
||||
bitTest(n, 7) as b7,
|
||||
bitTest(n, 6) as b6,
|
||||
bitTest(n, 5) as b5,
|
||||
bitTest(n, 4) as b4,
|
||||
bitTest(n, 3) as b3,
|
||||
bitTest(n, 2) as b2,
|
||||
bitTest(n, 1) as b1,
|
||||
bitTest(n, 0) as b0
|
||||
from system.numbers limit 256;
|
||||
|
||||
select n = n_,
|
||||
number as n,
|
||||
bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(b7, 1), b6), 1), b5), 1), b4), 1), b3), 1), b2), 1), b1), 1), b0) as n_,
|
||||
bitTestAll(n, 7) as b7,
|
||||
bitTestAll(n, 6) as b6,
|
||||
bitTestAll(n, 5) as b5,
|
||||
bitTestAll(n, 4) as b4,
|
||||
bitTestAll(n, 3) as b3,
|
||||
bitTestAll(n, 2) as b2,
|
||||
bitTestAll(n, 1) as b1,
|
||||
bitTestAll(n, 0) as b0
|
||||
from system.numbers limit 256;
|
||||
|
||||
select n = n_,
|
||||
number as n,
|
||||
bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(bitOr(bitShiftLeft(b7, 1), b6), 1), b5), 1), b4), 1), b3), 1), b2), 1), b1), 1), b0) as n_,
|
||||
bitTestAny(n, 7) as b7,
|
||||
bitTestAny(n, 6) as b6,
|
||||
bitTestAny(n, 5) as b5,
|
||||
bitTestAny(n, 4) as b4,
|
||||
bitTestAny(n, 3) as b3,
|
||||
bitTestAny(n, 2) as b2,
|
||||
bitTestAny(n, 1) as b1,
|
||||
bitTestAny(n, 0) as b0
|
||||
from system.numbers limit 256;
|
@ -0,0 +1,2 @@
|
||||
42 1
|
||||
1
|
@ -0,0 +1,2 @@
|
||||
SELECT k, a FROM (SELECT 42 AS k FROM remote('127.0.0.1', system.one)) GLOBAL ALL FULL OUTER JOIN (SELECT 42 AS k, 1 AS a, a) USING k;
|
||||
SELECT 1 FROM remote('127.0.0.1', system.one) WHERE (1, 1) GLOBAL IN (SELECT 1 AS a, a);
|
@ -0,0 +1,11 @@
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
0
|
||||
0
|
11
dbms/tests/queries/0_stateless/00218_like_regexp_newline.sql
Normal file
11
dbms/tests/queries/0_stateless/00218_like_regexp_newline.sql
Normal file
@ -0,0 +1,11 @@
|
||||
SELECT 'abcdef' LIKE '%abc%def%';
|
||||
SELECT 'abctdef' LIKE '%abc%def%';
|
||||
SELECT 'abc\ndef' LIKE '%abc%def%';
|
||||
SELECT 'abc\ntdef' LIKE '%abc%def%';
|
||||
SELECT 'abct\ndef' LIKE '%abc%def%';
|
||||
SELECT 'abc\n\ndef' LIKE '%abc%def%';
|
||||
SELECT 'abc\n\ntdef' LIKE '%abc%def%';
|
||||
SELECT 'abc\nt\ndef' LIKE '%abc%def%';
|
||||
SELECT 'abct\n\ndef' LIKE '%abc%def%';
|
||||
SELECT 'ab\ndef' LIKE '%abc%def%';
|
||||
SELECT 'abc\nef' LIKE '%abc%def%';
|
@ -0,0 +1,6 @@
|
||||
2 3000
|
||||
2 3000
|
||||
1 2000
|
||||
2 3000
|
||||
1 2000
|
||||
2 3000
|
@ -0,0 +1,6 @@
|
||||
SELECT a, b FROM (SELECT 1 AS a, 2000 AS b) ANY RIGHT JOIN (SELECT 2 AS a, 3000 AS b) USING a, b;
|
||||
SELECT a, b FROM (SELECT 1 AS a, 2000 AS b) ANY RIGHT JOIN (SELECT 2 AS a, 3000 AS b) USING b, a;
|
||||
|
||||
SELECT a, b FROM (SELECT 1 AS a, 2000 AS b) ANY RIGHT JOIN (SELECT 2 AS a, 3000 AS b UNION ALL SELECT 1 AS a, 2000 AS b) USING a, b;
|
||||
SELECT a, b FROM (SELECT 1 AS a, 2000 AS b) ANY RIGHT JOIN (SELECT 2 AS a, 3000 AS b UNION ALL SELECT 1 AS a, 2000 AS b) USING b, a;
|
||||
|
@ -0,0 +1,3 @@
|
||||
1
|
||||
|
||||
1
|
@ -0,0 +1 @@
|
||||
SELECT x FROM (SELECT count() AS x FROM remote('127.0.0.1', system.one) WITH TOTALS) LIMIT 1;
|
Loading…
Reference in New Issue
Block a user