mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-21 09:10:48 +00:00
reverted accidentally committed changes. [#METR-8818]
This commit is contained in:
Michael Kolupaev
parent
228b2f6f8a
commit
25a33cdeed
@ -220,7 +220,6 @@ namespace ErrorCodes
|
||||
NETWORK_ERROR,
|
||||
EMPTY_QUERY,
|
||||
UNKNOWN_LOAD_BALANCING,
|
||||
UNKNOWN_TOTALS_MODE,
|
||||
CANNOT_STATVFS,
|
||||
NOT_AN_AGGREGATE,
|
||||
QUERY_ID_ALREADY_RUNNING,
|
||||
|
||||
@ -23,25 +23,6 @@ namespace LoadBalancing
|
||||
};
|
||||
}
|
||||
|
||||
/// Какие строки включать в TOTALS.
|
||||
namespace TotalsMode
|
||||
{
|
||||
enum TotalsMode
|
||||
{
|
||||
BEFORE_HAVING = 0, /// Считать HAVING по всем прочитанным строкам;
|
||||
/// включая не попавшие в max_rows_to_group_by
|
||||
/// и не прошедшие HAVING после группировки.
|
||||
AFTER_HAVING_INCLUSIVE = 1, /// Считать по всем строкам, кроме не прошедших HAVING;
|
||||
/// то есть, включать в TOTALS все строки, не прошедшие max_rows_to_group_by.
|
||||
AFTER_HAVING_EXCLUSIVE = 2, /// Включать только строки, прошедшие и max_rows_to_group_by, и HAVING.
|
||||
AFTER_HAVING_AUTO = 3, /// Автоматически выбирать между INCLUSIVE и EXCLUSIVE,
|
||||
/// учитывая долю строк, прошедших HAVING.
|
||||
};
|
||||
|
||||
String toString(TotalsMode mode);
|
||||
TotalsMode parse(const String & s);
|
||||
};
|
||||
|
||||
/** Настройки выполнения запроса.
|
||||
*/
|
||||
struct Settings
|
||||
@ -87,8 +68,6 @@ struct Settings
|
||||
/// Сэмплирование по умолчанию. Если равно 1, то отключено
|
||||
float default_sample;
|
||||
|
||||
TotalsMode::TotalsMode totals_mode;
|
||||
|
||||
/// Всевозможные ограничения на выполнение запроса.
|
||||
Limits limits;
|
||||
|
||||
@ -108,8 +87,7 @@ struct Settings
|
||||
distributed_connections_pool_size(DBMS_DEFAULT_DISTRIBUTED_CONNECTIONS_POOL_SIZE),
|
||||
connections_with_failover_max_tries(DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES),
|
||||
sign_rewrite(false), extremes(false), use_uncompressed_cache(true), use_splitting_aggregator(false),
|
||||
replace_running_query(false), load_balancing(LoadBalancing::RANDOM), default_sample(DBMS_DEFAULT_SAMPLE),
|
||||
totals_mode(TotalsMode::BEFORE_HAVING)
|
||||
replace_running_query(false), load_balancing(LoadBalancing::RANDOM), default_sample(DBMS_DEFAULT_SAMPLE)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
@ -1,369 +0,0 @@
|
||||
#include <DB/DataTypes/DataTypesNumberFixed.h>
|
||||
#include <DB/Functions/IFunction.h>
|
||||
#include <statdaemons/Stopwatch.h>
|
||||
#include <iomanip>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
template<typename B>
|
||||
struct AndImpl
|
||||
{
|
||||
static inline UInt8 apply(UInt8 a, B b)
|
||||
{
|
||||
return a && b;
|
||||
}
|
||||
};
|
||||
|
||||
template<typename B>
|
||||
struct OrImpl
|
||||
{
|
||||
static inline UInt8 apply(UInt8 a, B b)
|
||||
{
|
||||
return a || b;
|
||||
}
|
||||
};
|
||||
|
||||
template<typename B>
|
||||
struct XorImpl
|
||||
{
|
||||
static inline UInt8 apply(UInt8 a, B b)
|
||||
{
|
||||
return (!a) != (!b);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
typedef ColumnVector<UInt8>::Container_t UInt8Container;
|
||||
typedef std::vector<const ColumnVector<UInt8> *> UInt8ColumnPtrs;
|
||||
|
||||
|
||||
template <typename Op, size_t N>
|
||||
struct AssociativeOperationImpl
|
||||
{
|
||||
/// Выбрасывает N последних столбцов из in (если их меньше, то все) и кладет в result их комбинацию.
|
||||
static void execute(UInt8ColumnPtrs & in, UInt8Container & result)
|
||||
{
|
||||
if (N > in.size()){
|
||||
AssociativeOperationImpl<Op, N - 1>::execute(in, result);
|
||||
return;
|
||||
}
|
||||
|
||||
AssociativeOperationImpl<Op, N> operation(in);
|
||||
in.erase(in.end() - N, in.end());
|
||||
|
||||
size_t n = result.size();
|
||||
for (size_t i = 0; i < n; ++i)
|
||||
{
|
||||
result[i] = operation.apply(i);
|
||||
}
|
||||
}
|
||||
|
||||
const UInt8Container & vec;
|
||||
AssociativeOperationImpl<Op, N - 1> continuation;
|
||||
|
||||
/// Запоминает последние N столбцов из in.
|
||||
AssociativeOperationImpl(UInt8ColumnPtrs & in)
|
||||
: vec(in[in.size() - N]->getData()), continuation(in) {}
|
||||
|
||||
/// Возвращает комбинацию значений в i-й строке всех столбцов, запомненных в конструкторе.
|
||||
inline UInt8 apply(size_t i) const
|
||||
{
|
||||
return Op::apply(vec[i], continuation.apply(i));
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Op>
|
||||
struct AssociativeOperationImpl<Op, 1>
|
||||
{
|
||||
static void execute(UInt8ColumnPtrs & in, UInt8Container & result)
|
||||
{
|
||||
throw Exception("Logical error: AssociativeOperationImpl<Op, 1>::execute called", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
const UInt8Container & vec;
|
||||
|
||||
AssociativeOperationImpl(UInt8ColumnPtrs & in)
|
||||
: vec(in[in.size() - 1]->getData()) {}
|
||||
|
||||
inline UInt8 apply(size_t i) const
|
||||
{
|
||||
return vec[i];
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
template <template <typename> class Impl, typename Name>
|
||||
class FunctionAnyArityLogical : public IFunction
|
||||
{
|
||||
private:
|
||||
bool extractConstColumns(ColumnPlainPtrs & in, UInt8 & res)
|
||||
{
|
||||
bool has_res = false;
|
||||
for (int i = static_cast<int>(in.size()) - 1; i >= 0; --i)
|
||||
{
|
||||
if (in[i]->isConst())
|
||||
{
|
||||
Field val = (*in[i])[0];
|
||||
UInt8 x = !!val.get<UInt64>();
|
||||
if (has_res)
|
||||
{
|
||||
res = Impl<UInt8>::apply(res, x);
|
||||
}
|
||||
else
|
||||
{
|
||||
res = x;
|
||||
has_res = true;
|
||||
}
|
||||
|
||||
in.erase(in.begin() + i);
|
||||
}
|
||||
}
|
||||
return has_res;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool convertTypeToUInt8(const IColumn * column, UInt8Container & res)
|
||||
{
|
||||
auto col = dynamic_cast<const ColumnVector<T> *>(column);
|
||||
if (!col)
|
||||
return false;
|
||||
const typename ColumnVector<T>::Container_t & vec = col->getData();
|
||||
size_t n = res.size();
|
||||
for (size_t i = 0; i < n; ++i)
|
||||
{
|
||||
res[i] = !!vec[i];
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void convertToUInt8(const IColumn * column, UInt8Container & res)
|
||||
{
|
||||
if (!convertTypeToUInt8< Int8 >(column, res) &&
|
||||
!convertTypeToUInt8< Int16>(column, res) &&
|
||||
!convertTypeToUInt8< Int32>(column, res) &&
|
||||
!convertTypeToUInt8< Int64>(column, res) &&
|
||||
!convertTypeToUInt8< UInt16>(column, res) &&
|
||||
!convertTypeToUInt8< UInt32>(column, res) &&
|
||||
!convertTypeToUInt8< UInt64>(column, res) &&
|
||||
!convertTypeToUInt8<Float32>(column, res) &&
|
||||
!convertTypeToUInt8<Float64>(column, res))
|
||||
throw Exception("Unexpected type of column: " + column->getName(), ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool executeUInt8Type(const UInt8Container & uint8_vec, IColumn * column, UInt8Container & res)
|
||||
{
|
||||
auto col = dynamic_cast<const ColumnVector<T> *>(column);
|
||||
if (!col)
|
||||
return false;
|
||||
const typename ColumnVector<T>::Container_t & other_vec = col->getData();
|
||||
size_t n = res.size();
|
||||
for (size_t i = 0; i < n; ++i)
|
||||
{
|
||||
res[i] = Impl<T>::apply(uint8_vec[i], other_vec[i]);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void executeUInt8Other(const UInt8Container & uint8_vec, IColumn * column, UInt8Container & res)
|
||||
{
|
||||
if (!executeUInt8Type< Int8 >(uint8_vec, column, res) &&
|
||||
!executeUInt8Type< Int16>(uint8_vec, column, res) &&
|
||||
!executeUInt8Type< Int32>(uint8_vec, column, res) &&
|
||||
!executeUInt8Type< Int64>(uint8_vec, column, res) &&
|
||||
!executeUInt8Type< UInt16>(uint8_vec, column, res) &&
|
||||
!executeUInt8Type< UInt32>(uint8_vec, column, res) &&
|
||||
!executeUInt8Type< UInt64>(uint8_vec, column, res) &&
|
||||
!executeUInt8Type<Float32>(uint8_vec, column, res) &&
|
||||
!executeUInt8Type<Float64>(uint8_vec, column, res))
|
||||
throw Exception("Unexpected type of column: " + column->getName(), ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
|
||||
public:
|
||||
/// Получить имя функции.
|
||||
String getName() const
|
||||
{
|
||||
return Name::get();
|
||||
}
|
||||
|
||||
/// Получить типы результата по типам аргументов. Если функция неприменима для данных аргументов - кинуть исключение.
|
||||
DataTypePtr getReturnType(const DataTypes & arguments) const
|
||||
{
|
||||
if (arguments.size() < 2)
|
||||
throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
|
||||
+ toString(arguments.size()) + ", should be at least 2.",
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
for (size_t i = 0; i < arguments.size(); ++i)
|
||||
{
|
||||
if (!arguments[i]->isNumeric())
|
||||
throw Exception("Illegal type ("
|
||||
+ arguments[i]->getName()
|
||||
+ ") of " + toString(i + 1) + " argument of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
|
||||
return new DataTypeUInt8;
|
||||
}
|
||||
|
||||
/// Выполнить функцию над блоком.
|
||||
void execute(Block & block, const ColumnNumbers & arguments, size_t result)
|
||||
{
|
||||
ColumnPlainPtrs in(arguments.size());
|
||||
for (size_t i = 0; i < arguments.size(); ++i)
|
||||
{
|
||||
in[i] = &*block.getByPosition(arguments[i]).column;
|
||||
}
|
||||
size_t n = in[0]->size();
|
||||
|
||||
/// Скомбинируем все константные столбцы в одно значение.
|
||||
UInt8 const_val = 0;
|
||||
bool has_consts = extractConstColumns(in, const_val);
|
||||
|
||||
// Если это значение однозначно определяет результат, вернем его.
|
||||
if (has_consts && (in.empty() || Impl<UInt8>::apply(const_val, 0) == Impl<UInt8>::apply(const_val, 1)))
|
||||
{
|
||||
if (!in.empty())
|
||||
const_val = Impl<UInt8>::apply(const_val, 0);
|
||||
ColumnConst<UInt8> * col_res = new ColumnConst<UInt8>(n, const_val);
|
||||
block.getByPosition(result).column = col_res;
|
||||
return;
|
||||
}
|
||||
|
||||
/// Если это значение - нейтральный элемент, забудем про него.
|
||||
if (has_consts && Impl<UInt8>::apply(const_val, 0) == 0 && Impl<UInt8>::apply(const_val, 1) == 1)
|
||||
has_consts = false;
|
||||
|
||||
ColumnVector<UInt8> * col_res = new ColumnVector<UInt8>;
|
||||
block.getByPosition(result).column = col_res;
|
||||
UInt8Container & vec_res = col_res->getData();
|
||||
|
||||
if (has_consts)
|
||||
{
|
||||
vec_res.assign(n, const_val);
|
||||
in.push_back(col_res);
|
||||
}
|
||||
else
|
||||
{
|
||||
vec_res.resize(n);
|
||||
}
|
||||
|
||||
/// Разделим входные столбцы на UInt8 и остальные. Первые обработаем более эффективно.
|
||||
/// col_res в каждый момент будет либо находится в конце uint8_in, либо не содержаться в uint8_in.
|
||||
UInt8ColumnPtrs uint8_in;
|
||||
ColumnPlainPtrs other_in;
|
||||
for (IColumn * column : in)
|
||||
{
|
||||
if (auto uint8_column = dynamic_cast<const ColumnVector<UInt8> *>(column))
|
||||
uint8_in.push_back(uint8_column);
|
||||
else
|
||||
other_in.push_back(column);
|
||||
}
|
||||
|
||||
/// Нужен хотя бы один столбец в uint8_in, чтобы было с кем комбинировать столбцы из other_in.
|
||||
if (uint8_in.empty())
|
||||
{
|
||||
if (other_in.empty())
|
||||
throw Exception("Hello, I'm a bug", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
convertToUInt8(other_in.back(), vec_res);
|
||||
other_in.pop_back();
|
||||
uint8_in.push_back(col_res);
|
||||
}
|
||||
|
||||
/// Эффективно скомбинируем все столбцы правильного типа.
|
||||
while (uint8_in.size() > 1)
|
||||
{
|
||||
AssociativeOperationImpl<Impl<UInt8>, 6>::execute(uint8_in, vec_res);
|
||||
uint8_in.push_back(col_res);
|
||||
}
|
||||
|
||||
/// По одному добавим все столбцы неправильного типа.
|
||||
while (!other_in.empty())
|
||||
{
|
||||
executeUInt8Other(uint8_in[0]->getData(), other_in.back(), vec_res);
|
||||
other_in.pop_back();
|
||||
uint8_in[0] = col_res;
|
||||
}
|
||||
|
||||
/// Такое возможно, если среди аргументов ровно один неконстантный, и он имеет тип UInt8.
|
||||
if (uint8_in[0] != col_res)
|
||||
{
|
||||
vec_res.assign(uint8_in[0]->getData());
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
struct NameAnd { static const char * get() { return "and"; } };
|
||||
struct NameOr { static const char * get() { return "or"; } };
|
||||
struct NameXor { static const char * get() { return "xor"; } };
|
||||
|
||||
typedef FunctionAnyArityLogical <AndImpl, NameAnd> FunctionAnd;
|
||||
typedef FunctionAnyArityLogical <OrImpl, NameOr> FunctionOr;
|
||||
typedef FunctionAnyArityLogical <XorImpl, NameXor> FunctionXor;
|
||||
}
|
||||
|
||||
using namespace DB;
|
||||
|
||||
|
||||
int main()
|
||||
{
|
||||
try
|
||||
{
|
||||
Block block;
|
||||
size_t size = 100000000;
|
||||
size_t columns = 10;
|
||||
size_t repeats = 3;
|
||||
|
||||
for (size_t i = 0; i < columns; ++i)
|
||||
{
|
||||
ColumnVector<UInt8> * column = new ColumnVector<UInt8>(size);
|
||||
block.insert(ColumnWithNameAndType(column, new DataTypeUInt8, "v" + toString(i)));
|
||||
|
||||
ColumnVector<UInt8>::Container_t & vec = column->getData();
|
||||
vec.resize(size);
|
||||
|
||||
for (size_t j = 0; j < size; ++j)
|
||||
{
|
||||
vec[j] = rand() % 2;
|
||||
}
|
||||
}
|
||||
|
||||
ColumnVector<UInt8> * result_column = new ColumnVector<UInt8>;
|
||||
block.insert(ColumnWithNameAndType(result_column, new DataTypeUInt8, "x"));
|
||||
result_column->getData().resize(size);
|
||||
|
||||
for (size_t arity = 2; arity <= columns; ++arity)
|
||||
{
|
||||
FunctionPtr function = new FunctionAnd;
|
||||
function->getReturnType(DataTypes(arity, DataTypePtr(new DataTypeUInt8)));
|
||||
|
||||
ColumnNumbers arguments(arity);
|
||||
for (size_t i = 0; i < arity; ++i)
|
||||
{
|
||||
arguments[i] = i;
|
||||
}
|
||||
|
||||
std::cerr << arity << "-ary:" << std::endl;
|
||||
|
||||
Stopwatch outer_sw;
|
||||
for (size_t i = 0; i < repeats; ++i)
|
||||
{
|
||||
Stopwatch inner_sw;
|
||||
function->execute(block, arguments, columns);
|
||||
std::cerr << "iteration " << i + 1 << ": " << std::setprecision(3) << inner_sw.elapsedSeconds() << " seconds." << std::endl;
|
||||
}
|
||||
double seconds = outer_sw.elapsedSeconds();
|
||||
std::cout << arity << "-ary average: " << std::setprecision(3) << seconds / repeats << " seconds, " << seconds / repeats / (arity - 1) << " seconds per operation." << std::endl;
|
||||
}
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
std::cerr << "exception" << std::endl;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -12,30 +12,6 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
String TotalsMode::toString(TotalsMode mode)
|
||||
{
|
||||
switch (mode)
|
||||
{
|
||||
case BEFORE_HAVING: return "before_having";
|
||||
case AFTER_HAVING_EXCLUSIVE: return "after_having_exclusive";
|
||||
case AFTER_HAVING_INCLUSIVE: return "after_having_inclusive";
|
||||
case AFTER_HAVING_AUTO: return "after_having_auto";
|
||||
|
||||
default:
|
||||
throw Exception("Unknown TotalsMode enum value: " + toString(mode), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
}
|
||||
}
|
||||
|
||||
auto TotalsMode::parse(const String & s) -> TotalsMode
|
||||
{
|
||||
if (s == "before_having") return BEFORE_HAVING;
|
||||
if (s == "after_having_exclusive") return AFTER_HAVING_EXCLUSIVE;
|
||||
if (s == "after_having_inclusive") return AFTER_HAVING_INCLUSIVE;
|
||||
if (s == "after_having_auto") return AFTER_HAVING_AUTO;
|
||||
|
||||
throw Exception("Unknown totals mode: '" + s + "', must be one of 'before_having', 'after_having_exclusive', 'after_having_inclusive', 'after_having_auto'", ErrorCodes::UNKNOWN_TOTALS_MODE);
|
||||
}
|
||||
|
||||
static LoadBalancing::LoadBalancing getLoadBalancing(const String & s)
|
||||
{
|
||||
if (s == "random") return LoadBalancing::RANDOM;
|
||||
@ -67,7 +43,6 @@ void Settings::set(const String & name, const Field & value)
|
||||
else if (name == "use_splitting_aggregator") use_splitting_aggregator = safeGet<UInt64>(value);
|
||||
else if (name == "replace_running_query") replace_running_query = safeGet<UInt64>(value);
|
||||
else if (name == "load_balancing") load_balancing = getLoadBalancing(safeGet<const String &>(value));
|
||||
else if (name == "totals_mode") totals_mode = TotalsMode::parse(safeGet<const String &>(value));
|
||||
else if (name == "default_sample")
|
||||
{
|
||||
if (value.getType() == Field::Types::UInt64)
|
||||
@ -111,8 +86,7 @@ void Settings::set(const String & name, ReadBuffer & buf)
|
||||
readVarUInt(value, buf);
|
||||
set(name, value);
|
||||
}
|
||||
else if ( name == "load_balancing"
|
||||
|| name == "totals_mode")
|
||||
else if (name == "load_balancing")
|
||||
{
|
||||
String value;
|
||||
readBinary(value, buf);
|
||||
@ -156,8 +130,7 @@ void Settings::set(const String & name, const String & value)
|
||||
{
|
||||
set(name, parse<Float64>(value));
|
||||
}
|
||||
else if ( name == "load_balancing"
|
||||
|| name == "totals_mode")
|
||||
else if (name == "load_balancing")
|
||||
{
|
||||
set(name, Field(value));
|
||||
}
|
||||
@ -216,7 +189,6 @@ void Settings::serialize(WriteBuffer & buf) const
|
||||
writeStringBinary("use_splitting_aggregator", buf); writeVarUInt(use_splitting_aggregator, buf);
|
||||
writeStringBinary("replace_running_query", buf); writeVarUInt(replace_running_query, buf);
|
||||
writeStringBinary("load_balancing", buf); writeStringBinary(toString(load_balancing), buf);
|
||||
writeStringBinary("totals_mode", buf); writeStringBinary(TotalsMode::toString(totals_mode), buf);
|
||||
writeStringBinary("default_sample", buf); writeStringBinary(DB::toString(default_sample), buf);
|
||||
|
||||
limits.serialize(buf);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user