2012-12-21 19:48:47 +00:00
|
|
|
|
#pragma once
|
|
|
|
|
|
|
|
|
|
#include <DB/DataTypes/DataTypesNumberFixed.h>
|
2013-03-06 12:03:52 +00:00
|
|
|
|
#include <DB/DataTypes/DataTypeArray.h>
|
2013-06-05 13:40:14 +00:00
|
|
|
|
#include <DB/DataTypes/DataTypeString.h>
|
2012-12-21 19:48:47 +00:00
|
|
|
|
|
|
|
|
|
#include <DB/Columns/ColumnVector.h>
|
2013-03-06 12:03:52 +00:00
|
|
|
|
#include <DB/Columns/ColumnArray.h>
|
2013-06-05 13:40:14 +00:00
|
|
|
|
#include <DB/Columns/ColumnString.h>
|
2012-12-21 19:48:47 +00:00
|
|
|
|
|
|
|
|
|
#include <DB/Interpreters/Context.h>
|
|
|
|
|
|
|
|
|
|
#include <DB/Functions/IFunction.h>
|
2013-04-17 11:37:34 +00:00
|
|
|
|
#include <statdaemons/CategoriesHierarchy.h>
|
2012-12-21 19:48:47 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
|
{
|
|
|
|
|
|
|
|
|
|
/** Функции, использующие словари Яндекс.Метрики
|
|
|
|
|
* - словари регионов, операционных систем, поисковых систем.
|
|
|
|
|
*
|
2013-03-18 10:49:31 +00:00
|
|
|
|
* Подняться по дереву до определенного уровня.
|
|
|
|
|
* regionToCity, regionToArea, regionToCountry,
|
|
|
|
|
* OSToRoot,
|
|
|
|
|
* SEToRoot,
|
2013-04-17 11:37:34 +00:00
|
|
|
|
* categoryToRoot,
|
|
|
|
|
* categoryToSecondLevel
|
2014-06-26 00:58:14 +00:00
|
|
|
|
*
|
2013-06-05 13:40:14 +00:00
|
|
|
|
* Преобразовать значения в столбце
|
|
|
|
|
* regionToName
|
2012-12-21 19:48:47 +00:00
|
|
|
|
*
|
2013-03-18 10:49:31 +00:00
|
|
|
|
* Является ли первый идентификатор потомком второго.
|
2013-04-17 11:37:34 +00:00
|
|
|
|
* regionIn, SEIn, OSIn, categoryIn.
|
2014-06-26 00:58:14 +00:00
|
|
|
|
*
|
2013-03-18 10:49:31 +00:00
|
|
|
|
* Получить массив идентификаторов регионов, состоящий из исходного и цепочки родителей. Порядок implementation defined.
|
2013-04-17 11:37:34 +00:00
|
|
|
|
* regionHierarchy, OSHierarchy, SEHierarchy, categoryHierarchy.
|
2012-12-21 19:48:47 +00:00
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
struct RegionToCityImpl
|
|
|
|
|
{
|
|
|
|
|
static UInt32 apply(UInt32 x, const RegionsHierarchy & hierarchy) { return hierarchy.toCity(x); }
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct RegionToAreaImpl
|
|
|
|
|
{
|
|
|
|
|
static UInt32 apply(UInt32 x, const RegionsHierarchy & hierarchy) { return hierarchy.toArea(x); }
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct RegionToCountryImpl
|
|
|
|
|
{
|
|
|
|
|
static UInt32 apply(UInt32 x, const RegionsHierarchy & hierarchy) { return hierarchy.toCountry(x); }
|
|
|
|
|
};
|
|
|
|
|
|
2014-01-22 12:27:55 +00:00
|
|
|
|
struct RegionToContinentImpl
|
|
|
|
|
{
|
|
|
|
|
static UInt32 apply(UInt32 x, const RegionsHierarchy & hierarchy) { return hierarchy.toContinent(x); }
|
|
|
|
|
};
|
|
|
|
|
|
2012-12-21 19:48:47 +00:00
|
|
|
|
struct OSToRootImpl
|
|
|
|
|
{
|
|
|
|
|
static UInt8 apply(UInt8 x, const TechDataHierarchy & hierarchy) { return hierarchy.OSToMostAncestor(x); }
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct SEToRootImpl
|
|
|
|
|
{
|
|
|
|
|
static UInt8 apply(UInt8 x, const TechDataHierarchy & hierarchy) { return hierarchy.SEToMostAncestor(x); }
|
|
|
|
|
};
|
|
|
|
|
|
2013-04-17 11:37:34 +00:00
|
|
|
|
struct CategoryToRootImpl
|
|
|
|
|
{
|
|
|
|
|
static UInt16 apply(UInt16 x, const CategoriesHierarchy & hierarchy) { return hierarchy.toMostAncestor(x); }
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct CategoryToSecondLevelImpl
|
|
|
|
|
{
|
|
|
|
|
static UInt16 apply(UInt16 x, const CategoriesHierarchy & hierarchy) { return hierarchy.toSecondLevel(x); }
|
|
|
|
|
};
|
|
|
|
|
|
2012-12-21 19:48:47 +00:00
|
|
|
|
struct RegionInImpl
|
|
|
|
|
{
|
|
|
|
|
static bool apply(UInt32 x, UInt32 y, const RegionsHierarchy & hierarchy) { return hierarchy.in(x, y); }
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct OSInImpl
|
|
|
|
|
{
|
|
|
|
|
static bool apply(UInt32 x, UInt32 y, const TechDataHierarchy & hierarchy) { return hierarchy.isOSIn(x, y); }
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct SEInImpl
|
|
|
|
|
{
|
|
|
|
|
static bool apply(UInt32 x, UInt32 y, const TechDataHierarchy & hierarchy) { return hierarchy.isSEIn(x, y); }
|
|
|
|
|
};
|
|
|
|
|
|
2013-04-17 11:37:34 +00:00
|
|
|
|
struct CategoryInImpl
|
|
|
|
|
{
|
|
|
|
|
static bool apply(UInt16 x, UInt16 y, const CategoriesHierarchy & hierarchy) { return hierarchy.in(x, y); }
|
|
|
|
|
};
|
|
|
|
|
|
2013-03-06 12:03:52 +00:00
|
|
|
|
struct RegionHierarchyImpl
|
|
|
|
|
{
|
|
|
|
|
static UInt32 toParent(UInt32 x, const RegionsHierarchy & hierarchy) { return hierarchy.toParent(x); }
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct OSHierarchyImpl
|
|
|
|
|
{
|
|
|
|
|
static UInt8 toParent(UInt8 x, const TechDataHierarchy & hierarchy) { return hierarchy.OSToParent(x); }
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct SEHierarchyImpl
|
|
|
|
|
{
|
|
|
|
|
static UInt8 toParent(UInt8 x, const TechDataHierarchy & hierarchy) { return hierarchy.SEToParent(x); }
|
|
|
|
|
};
|
|
|
|
|
|
2013-04-17 11:37:34 +00:00
|
|
|
|
struct CategoryHierarchyImpl
|
|
|
|
|
{
|
|
|
|
|
static UInt16 toParent(UInt16 x, const CategoriesHierarchy & hierarchy) { return hierarchy.toParent(x); }
|
|
|
|
|
};
|
|
|
|
|
|
2012-12-21 19:48:47 +00:00
|
|
|
|
|
2014-04-04 18:08:01 +00:00
|
|
|
|
/** Вспомогательная вещь, позволяющая достать из словаря конкретный словарь, соответствующий точке зрения
|
|
|
|
|
* (ключу словаря, передаваемому в аргументе функции).
|
|
|
|
|
* Пример: при вызове regionToCountry(x, 'ua'), может быть использован словарь, в котором Крым относится к Украине.
|
|
|
|
|
*/
|
|
|
|
|
struct RegionsHierarchyGetter
|
|
|
|
|
{
|
|
|
|
|
typedef RegionsHierarchies Src;
|
|
|
|
|
typedef RegionsHierarchy Dst;
|
|
|
|
|
|
|
|
|
|
static const Dst & get(const Src & src, const std::string & key)
|
|
|
|
|
{
|
|
|
|
|
return src.get(key);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
/** Для словарей без поддержки ключей. Ничего не делает.
|
|
|
|
|
*/
|
|
|
|
|
template <typename Dict>
|
|
|
|
|
struct IdentityDictionaryGetter
|
|
|
|
|
{
|
|
|
|
|
typedef Dict Src;
|
|
|
|
|
typedef Dict Dst;
|
|
|
|
|
|
|
|
|
|
static const Dst & get(const Src & src, const std::string & key)
|
|
|
|
|
{
|
|
|
|
|
if (key.empty())
|
|
|
|
|
return src;
|
|
|
|
|
else
|
|
|
|
|
throw Exception("Dictionary doesn't support 'point of view' keys.", ErrorCodes::BAD_ARGUMENTS);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
2012-12-21 19:48:47 +00:00
|
|
|
|
/// Преобразует идентификатор, используя словарь.
|
2014-04-04 23:41:54 +00:00
|
|
|
|
template <typename T, typename Transform, typename DictGetter, typename Name>
|
2012-12-21 19:48:47 +00:00
|
|
|
|
class FunctionTransformWithDictionary : public IFunction
|
|
|
|
|
{
|
|
|
|
|
private:
|
2014-04-04 23:41:54 +00:00
|
|
|
|
const SharedPtr<typename DictGetter::Src> owned_dict;
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2012-12-21 19:48:47 +00:00
|
|
|
|
public:
|
2014-04-04 23:41:54 +00:00
|
|
|
|
FunctionTransformWithDictionary(const SharedPtr<typename DictGetter::Src> & owned_dict_)
|
2012-12-21 19:48:47 +00:00
|
|
|
|
: owned_dict(owned_dict_)
|
|
|
|
|
{
|
|
|
|
|
if (!owned_dict)
|
|
|
|
|
throw Exception("Dictionaries was not loaded. You need to check configuration file.", ErrorCodes::DICTIONARIES_WAS_NOT_LOADED);
|
|
|
|
|
}
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2012-12-21 19:48:47 +00:00
|
|
|
|
/// Получить имя функции.
|
|
|
|
|
String getName() const
|
|
|
|
|
{
|
|
|
|
|
return Name::get();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Получить тип результата по типам аргументов. Если функция неприменима для данных аргументов - кинуть исключение.
|
|
|
|
|
DataTypePtr getReturnType(const DataTypes & arguments) const
|
|
|
|
|
{
|
2014-04-04 18:08:01 +00:00
|
|
|
|
if (arguments.size() != 1 && arguments.size() != 2)
|
2012-12-21 19:48:47 +00:00
|
|
|
|
throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
|
2014-04-04 18:08:01 +00:00
|
|
|
|
+ toString(arguments.size()) + ", should be 1 or 2.",
|
2012-12-21 19:48:47 +00:00
|
|
|
|
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
|
|
|
|
|
|
|
|
|
if (arguments[0]->getName() != TypeName<T>::get())
|
|
|
|
|
throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName()
|
|
|
|
|
+ " (must be " + TypeName<T>::get() + ")",
|
|
|
|
|
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
|
|
|
|
|
2014-04-04 18:08:01 +00:00
|
|
|
|
if (arguments.size() == 2 && arguments[1]->getName() != TypeName<String>::get())
|
|
|
|
|
throw Exception("Illegal type " + arguments[1]->getName() + " of the second ('point of view') argument of function " + getName()
|
|
|
|
|
+ " (must be " + TypeName<String>::get() + ")",
|
|
|
|
|
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
|
|
|
|
|
2012-12-21 19:48:47 +00:00
|
|
|
|
return arguments[0];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Выполнить функцию над блоком.
|
|
|
|
|
void execute(Block & block, const ColumnNumbers & arguments, size_t result)
|
|
|
|
|
{
|
2014-04-04 18:08:01 +00:00
|
|
|
|
/// Ключ словаря, определяющий "точку зрения".
|
|
|
|
|
std::string dict_key;
|
|
|
|
|
|
|
|
|
|
if (arguments.size() == 2)
|
|
|
|
|
{
|
2014-06-26 00:58:14 +00:00
|
|
|
|
const ColumnConstString * key_col = typeid_cast<const ColumnConstString *>(&*block.getByPosition(arguments[1]).column);
|
2014-04-04 18:08:01 +00:00
|
|
|
|
|
|
|
|
|
if (!key_col)
|
|
|
|
|
throw Exception("Illegal column " + block.getByPosition(arguments[1]).column->getName()
|
|
|
|
|
+ " of second ('point of view') argument of function " + Name::get()
|
|
|
|
|
+ ". Must be constant string.",
|
|
|
|
|
ErrorCodes::ILLEGAL_COLUMN);
|
|
|
|
|
|
|
|
|
|
dict_key = key_col->getData();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const typename DictGetter::Dst & dict = DictGetter::get(*owned_dict, dict_key);
|
|
|
|
|
|
2014-06-26 00:58:14 +00:00
|
|
|
|
if (const ColumnVector<T> * col_from = typeid_cast<const ColumnVector<T> *>(&*block.getByPosition(arguments[0]).column))
|
2012-12-21 19:48:47 +00:00
|
|
|
|
{
|
|
|
|
|
ColumnVector<T> * col_to = new ColumnVector<T>;
|
|
|
|
|
block.getByPosition(result).column = col_to;
|
|
|
|
|
|
|
|
|
|
const typename ColumnVector<T>::Container_t & vec_from = col_from->getData();
|
|
|
|
|
typename ColumnVector<T>::Container_t & vec_to = col_to->getData();
|
|
|
|
|
size_t size = vec_from.size();
|
|
|
|
|
vec_to.resize(size);
|
|
|
|
|
|
|
|
|
|
for (size_t i = 0; i < size; ++i)
|
|
|
|
|
vec_to[i] = Transform::apply(vec_from[i], dict);
|
|
|
|
|
}
|
2014-06-26 00:58:14 +00:00
|
|
|
|
else if (const ColumnConst<T> * col_from = typeid_cast<const ColumnConst<T> *>(&*block.getByPosition(arguments[0]).column))
|
2012-12-21 19:48:47 +00:00
|
|
|
|
{
|
2014-04-04 18:08:01 +00:00
|
|
|
|
block.getByPosition(result).column = new ColumnConst<T>(col_from->size(), Transform::apply(col_from->getData(), dict));
|
2012-12-21 19:48:47 +00:00
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName()
|
|
|
|
|
+ " of first argument of function " + Name::get(),
|
|
|
|
|
ErrorCodes::ILLEGAL_COLUMN);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/// Проверяет принадлежность, используя словарь.
|
2014-04-04 23:41:54 +00:00
|
|
|
|
template <typename T, typename Transform, typename DictGetter, typename Name>
|
2012-12-21 19:48:47 +00:00
|
|
|
|
class FunctionIsInWithDictionary : public IFunction
|
|
|
|
|
{
|
|
|
|
|
private:
|
2014-04-04 23:41:54 +00:00
|
|
|
|
const SharedPtr<typename DictGetter::Src> owned_dict;
|
2012-12-21 19:48:47 +00:00
|
|
|
|
|
|
|
|
|
public:
|
2014-04-04 23:41:54 +00:00
|
|
|
|
FunctionIsInWithDictionary(const SharedPtr<typename DictGetter::Src> & owned_dict_)
|
2012-12-21 19:48:47 +00:00
|
|
|
|
: owned_dict(owned_dict_)
|
|
|
|
|
{
|
|
|
|
|
if (!owned_dict)
|
|
|
|
|
throw Exception("Dictionaries was not loaded. You need to check configuration file.", ErrorCodes::DICTIONARIES_WAS_NOT_LOADED);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Получить имя функции.
|
|
|
|
|
String getName() const
|
|
|
|
|
{
|
|
|
|
|
return Name::get();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Получить тип результата по типам аргументов. Если функция неприменима для данных аргументов - кинуть исключение.
|
|
|
|
|
DataTypePtr getReturnType(const DataTypes & arguments) const
|
|
|
|
|
{
|
2014-04-04 18:08:01 +00:00
|
|
|
|
if (arguments.size() != 2 && arguments.size() != 3)
|
2012-12-21 19:48:47 +00:00
|
|
|
|
throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
|
2014-04-04 18:08:01 +00:00
|
|
|
|
+ toString(arguments.size()) + ", should be 2 or 3.",
|
2012-12-21 19:48:47 +00:00
|
|
|
|
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
|
|
|
|
|
|
|
|
|
if (arguments[0]->getName() != TypeName<T>::get())
|
|
|
|
|
throw Exception("Illegal type " + arguments[0]->getName() + " of first argument of function " + getName()
|
|
|
|
|
+ " (must be " + TypeName<T>::get() + ")",
|
|
|
|
|
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
|
|
|
|
|
|
|
|
|
if (arguments[1]->getName() != TypeName<T>::get())
|
|
|
|
|
throw Exception("Illegal type " + arguments[1]->getName() + " of second argument of function " + getName()
|
|
|
|
|
+ " (must be " + TypeName<T>::get() + ")",
|
|
|
|
|
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
|
|
|
|
|
2014-04-04 18:08:01 +00:00
|
|
|
|
if (arguments.size() == 3 && arguments[2]->getName() != TypeName<String>::get())
|
|
|
|
|
throw Exception("Illegal type " + arguments[2]->getName() + " of the third ('point of view') argument of function " + getName()
|
|
|
|
|
+ " (must be " + TypeName<String>::get() + ")",
|
|
|
|
|
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
|
|
|
|
|
2012-12-21 19:48:47 +00:00
|
|
|
|
return new DataTypeUInt8;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Выполнить функцию над блоком.
|
|
|
|
|
void execute(Block & block, const ColumnNumbers & arguments, size_t result)
|
|
|
|
|
{
|
2014-04-04 18:08:01 +00:00
|
|
|
|
/// Ключ словаря, определяющий "точку зрения".
|
|
|
|
|
std::string dict_key;
|
|
|
|
|
|
|
|
|
|
if (arguments.size() == 3)
|
|
|
|
|
{
|
2014-06-26 00:58:14 +00:00
|
|
|
|
const ColumnConstString * key_col = typeid_cast<const ColumnConstString *>(&*block.getByPosition(arguments[2]).column);
|
2014-04-04 18:08:01 +00:00
|
|
|
|
|
|
|
|
|
if (!key_col)
|
|
|
|
|
throw Exception("Illegal column " + block.getByPosition(arguments[2]).column->getName()
|
|
|
|
|
+ " of third ('point of view') argument of function " + Name::get()
|
|
|
|
|
+ ". Must be constant string.",
|
|
|
|
|
ErrorCodes::ILLEGAL_COLUMN);
|
|
|
|
|
|
|
|
|
|
dict_key = key_col->getData();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const typename DictGetter::Dst & dict = DictGetter::get(*owned_dict, dict_key);
|
|
|
|
|
|
2014-06-26 00:58:14 +00:00
|
|
|
|
const ColumnVector<T> * col_vec1 = typeid_cast<const ColumnVector<T> *>(&*block.getByPosition(arguments[0]).column);
|
|
|
|
|
const ColumnVector<T> * col_vec2 = typeid_cast<const ColumnVector<T> *>(&*block.getByPosition(arguments[1]).column);
|
|
|
|
|
const ColumnConst<T> * col_const1 = typeid_cast<const ColumnConst<T> *>(&*block.getByPosition(arguments[0]).column);
|
|
|
|
|
const ColumnConst<T> * col_const2 = typeid_cast<const ColumnConst<T> *>(&*block.getByPosition(arguments[1]).column);
|
|
|
|
|
|
2012-12-21 19:48:47 +00:00
|
|
|
|
if (col_vec1 && col_vec2)
|
|
|
|
|
{
|
|
|
|
|
ColumnVector<UInt8> * col_to = new ColumnVector<UInt8>;
|
|
|
|
|
block.getByPosition(result).column = col_to;
|
|
|
|
|
|
|
|
|
|
const typename ColumnVector<T>::Container_t & vec_from1 = col_vec1->getData();
|
|
|
|
|
const typename ColumnVector<T>::Container_t & vec_from2 = col_vec2->getData();
|
|
|
|
|
typename ColumnVector<UInt8>::Container_t & vec_to = col_to->getData();
|
|
|
|
|
size_t size = vec_from1.size();
|
|
|
|
|
vec_to.resize(size);
|
|
|
|
|
|
|
|
|
|
for (size_t i = 0; i < size; ++i)
|
|
|
|
|
vec_to[i] = Transform::apply(vec_from1[i], vec_from2[i], dict);
|
|
|
|
|
}
|
|
|
|
|
else if (col_vec1 && col_const2)
|
|
|
|
|
{
|
|
|
|
|
ColumnVector<UInt8> * col_to = new ColumnVector<UInt8>;
|
|
|
|
|
block.getByPosition(result).column = col_to;
|
|
|
|
|
|
|
|
|
|
const typename ColumnVector<T>::Container_t & vec_from1 = col_vec1->getData();
|
|
|
|
|
const T const_from2 = col_const2->getData();
|
|
|
|
|
typename ColumnVector<UInt8>::Container_t & vec_to = col_to->getData();
|
|
|
|
|
size_t size = vec_from1.size();
|
|
|
|
|
vec_to.resize(size);
|
|
|
|
|
|
|
|
|
|
for (size_t i = 0; i < size; ++i)
|
|
|
|
|
vec_to[i] = Transform::apply(vec_from1[i], const_from2, dict);
|
|
|
|
|
}
|
|
|
|
|
else if (col_const1 && col_vec2)
|
|
|
|
|
{
|
|
|
|
|
ColumnVector<UInt8> * col_to = new ColumnVector<UInt8>;
|
|
|
|
|
block.getByPosition(result).column = col_to;
|
|
|
|
|
|
|
|
|
|
const T const_from1 = col_const1->getData();
|
|
|
|
|
const typename ColumnVector<T>::Container_t & vec_from2 = col_vec2->getData();
|
|
|
|
|
typename ColumnVector<UInt8>::Container_t & vec_to = col_to->getData();
|
|
|
|
|
size_t size = vec_from2.size();
|
|
|
|
|
vec_to.resize(size);
|
|
|
|
|
|
|
|
|
|
for (size_t i = 0; i < size; ++i)
|
|
|
|
|
vec_to[i] = Transform::apply(const_from1, vec_from2[i], dict);
|
|
|
|
|
}
|
|
|
|
|
else if (col_const1 && col_const2)
|
|
|
|
|
{
|
2013-04-10 10:52:38 +00:00
|
|
|
|
block.getByPosition(result).column = new ColumnConst<UInt8>(col_const1->size(),
|
2014-04-04 18:08:01 +00:00
|
|
|
|
Transform::apply(col_const1->getData(), col_const2->getData(), dict));
|
2012-12-21 19:48:47 +00:00
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
throw Exception("Illegal columns " + block.getByPosition(arguments[0]).column->getName()
|
|
|
|
|
+ " and " + block.getByPosition(arguments[1]).column->getName()
|
|
|
|
|
+ " of arguments of function " + Name::get(),
|
|
|
|
|
ErrorCodes::ILLEGAL_COLUMN);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
2013-03-06 12:03:52 +00:00
|
|
|
|
/// Получает массив идентификаторов, состоящий из исходного и цепочки родителей.
|
2014-04-04 23:41:54 +00:00
|
|
|
|
template <typename T, typename Transform, typename DictGetter, typename Name>
|
2013-03-06 12:03:52 +00:00
|
|
|
|
class FunctionHierarchyWithDictionary : public IFunction
|
|
|
|
|
{
|
|
|
|
|
private:
|
2014-04-04 23:41:54 +00:00
|
|
|
|
const SharedPtr<typename DictGetter::Src> owned_dict;
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-03-06 12:03:52 +00:00
|
|
|
|
public:
|
2014-04-04 23:41:54 +00:00
|
|
|
|
FunctionHierarchyWithDictionary(const SharedPtr<typename DictGetter::Src> & owned_dict_)
|
2013-03-06 12:03:52 +00:00
|
|
|
|
: owned_dict(owned_dict_)
|
|
|
|
|
{
|
|
|
|
|
if (!owned_dict)
|
|
|
|
|
throw Exception("Dictionaries was not loaded. You need to check configuration file.", ErrorCodes::DICTIONARIES_WAS_NOT_LOADED);
|
|
|
|
|
}
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-03-06 12:03:52 +00:00
|
|
|
|
/// Получить имя функции.
|
|
|
|
|
String getName() const
|
|
|
|
|
{
|
|
|
|
|
return Name::get();
|
|
|
|
|
}
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-03-06 12:03:52 +00:00
|
|
|
|
/// Получить тип результата по типам аргументов. Если функция неприменима для данных аргументов - кинуть исключение.
|
|
|
|
|
DataTypePtr getReturnType(const DataTypes & arguments) const
|
|
|
|
|
{
|
2014-04-04 18:08:01 +00:00
|
|
|
|
if (arguments.size() != 1 && arguments.size() != 2)
|
2013-03-06 12:03:52 +00:00
|
|
|
|
throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
|
2014-04-04 18:08:01 +00:00
|
|
|
|
+ toString(arguments.size()) + ", should be 1 or 2.",
|
|
|
|
|
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-03-06 12:03:52 +00:00
|
|
|
|
if (arguments[0]->getName() != TypeName<T>::get())
|
|
|
|
|
throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName()
|
|
|
|
|
+ " (must be " + TypeName<T>::get() + ")",
|
|
|
|
|
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
2014-04-04 18:08:01 +00:00
|
|
|
|
|
|
|
|
|
if (arguments.size() == 2 && arguments[1]->getName() != TypeName<String>::get())
|
|
|
|
|
throw Exception("Illegal type " + arguments[1]->getName() + " of the second ('point of view') argument of function " + getName()
|
|
|
|
|
+ " (must be " + TypeName<String>::get() + ")",
|
|
|
|
|
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-03-06 12:03:52 +00:00
|
|
|
|
return new DataTypeArray(arguments[0]);
|
|
|
|
|
}
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-03-06 12:03:52 +00:00
|
|
|
|
/// Выполнить функцию над блоком.
|
|
|
|
|
void execute(Block & block, const ColumnNumbers & arguments, size_t result)
|
|
|
|
|
{
|
2014-04-04 18:08:01 +00:00
|
|
|
|
/// Ключ словаря, определяющий "точку зрения".
|
|
|
|
|
std::string dict_key;
|
|
|
|
|
|
|
|
|
|
if (arguments.size() == 2)
|
|
|
|
|
{
|
2014-06-26 00:58:14 +00:00
|
|
|
|
const ColumnConstString * key_col = typeid_cast<const ColumnConstString *>(&*block.getByPosition(arguments[1]).column);
|
2014-04-04 18:08:01 +00:00
|
|
|
|
|
|
|
|
|
if (!key_col)
|
|
|
|
|
throw Exception("Illegal column " + block.getByPosition(arguments[1]).column->getName()
|
|
|
|
|
+ " of second ('point of view') argument of function " + Name::get()
|
|
|
|
|
+ ". Must be constant string.",
|
|
|
|
|
ErrorCodes::ILLEGAL_COLUMN);
|
|
|
|
|
|
|
|
|
|
dict_key = key_col->getData();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const typename DictGetter::Dst & dict = DictGetter::get(*owned_dict, dict_key);
|
|
|
|
|
|
2014-06-26 00:58:14 +00:00
|
|
|
|
if (const ColumnVector<T> * col_from = typeid_cast<const ColumnVector<T> *>(&*block.getByPosition(arguments[0]).column))
|
2013-03-06 12:03:52 +00:00
|
|
|
|
{
|
|
|
|
|
ColumnVector<T> * col_values = new ColumnVector<T>;
|
|
|
|
|
ColumnArray * col_array = new ColumnArray(col_values);
|
|
|
|
|
block.getByPosition(result).column = col_array;
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-03-06 12:03:52 +00:00
|
|
|
|
ColumnArray::Offsets_t & res_offsets = col_array->getOffsets();
|
|
|
|
|
typename ColumnVector<T>::Container_t & res_values = col_values->getData();
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-03-06 12:03:52 +00:00
|
|
|
|
const typename ColumnVector<T>::Container_t & vec_from = col_from->getData();
|
|
|
|
|
size_t size = vec_from.size();
|
|
|
|
|
res_offsets.resize(size);
|
|
|
|
|
res_values.reserve(size * 4);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-03-06 12:03:52 +00:00
|
|
|
|
for (size_t i = 0; i < size; ++i)
|
|
|
|
|
{
|
|
|
|
|
T cur = vec_from[i];
|
|
|
|
|
while (cur)
|
|
|
|
|
{
|
|
|
|
|
res_values.push_back(cur);
|
|
|
|
|
cur = Transform::toParent(cur, dict);
|
|
|
|
|
}
|
|
|
|
|
res_offsets[i] = res_values.size();
|
|
|
|
|
}
|
|
|
|
|
}
|
2014-06-26 00:58:14 +00:00
|
|
|
|
else if (const ColumnConst<T> * col_from = typeid_cast<const ColumnConst<T> *>(&*block.getByPosition(arguments[0]).column))
|
2013-03-06 12:03:52 +00:00
|
|
|
|
{
|
|
|
|
|
Array res;
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-03-06 12:03:52 +00:00
|
|
|
|
T cur = col_from->getData();
|
|
|
|
|
while (cur)
|
|
|
|
|
{
|
2013-03-06 12:27:05 +00:00
|
|
|
|
res.push_back(static_cast<typename NearestFieldType<T>::Type>(cur));
|
2013-03-06 12:03:52 +00:00
|
|
|
|
cur = Transform::toParent(cur, dict);
|
|
|
|
|
}
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-03-27 10:07:23 +00:00
|
|
|
|
block.getByPosition(result).column = new ColumnConstArray(col_from->size(), res, new DataTypeArray(new typename DataTypeFromFieldType<T>::Type));
|
2013-03-06 12:03:52 +00:00
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName()
|
|
|
|
|
+ " of first argument of function " + Name::get(),
|
|
|
|
|
ErrorCodes::ILLEGAL_COLUMN);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
2012-12-21 19:48:47 +00:00
|
|
|
|
struct NameRegionToCity { static const char * get() { return "regionToCity"; } };
|
|
|
|
|
struct NameRegionToArea { static const char * get() { return "regionToArea"; } };
|
|
|
|
|
struct NameRegionToCountry { static const char * get() { return "regionToCountry"; } };
|
2014-01-22 12:27:55 +00:00
|
|
|
|
struct NameRegionToContinent{ static const char * get() { return "regionToContient"; } };
|
2012-12-21 19:48:47 +00:00
|
|
|
|
struct NameOSToRoot { static const char * get() { return "OSToRoot"; } };
|
|
|
|
|
struct NameSEToRoot { static const char * get() { return "SEToRoot"; } };
|
2013-04-17 11:37:34 +00:00
|
|
|
|
struct NameCategoryToRoot { static const char * get() { return "categoryToRoot"; } };
|
|
|
|
|
struct NameCategoryToSecondLevel { static const char * get() { return "categoryToSecondLevel"; } };
|
2012-12-21 19:48:47 +00:00
|
|
|
|
|
|
|
|
|
struct NameRegionIn { static const char * get() { return "regionIn"; } };
|
|
|
|
|
struct NameOSIn { static const char * get() { return "OSIn"; } };
|
|
|
|
|
struct NameSEIn { static const char * get() { return "SEIn"; } };
|
2013-04-17 11:37:34 +00:00
|
|
|
|
struct NameCategoryIn { static const char * get() { return "categoryIn"; } };
|
2012-12-21 19:48:47 +00:00
|
|
|
|
|
2013-03-06 12:03:52 +00:00
|
|
|
|
struct NameRegionHierarchy { static const char * get() { return "regionHierarchy"; } };
|
2013-04-17 11:37:34 +00:00
|
|
|
|
struct NameOSHierarchy { static const char * get() { return "OSHierarchy"; } };
|
|
|
|
|
struct NameSEHierarchy { static const char * get() { return "SEHierarchy"; } };
|
|
|
|
|
struct NameCategoryHierarchy{ static const char * get() { return "categoryHierarchy"; } };
|
|
|
|
|
|
|
|
|
|
|
2014-04-04 18:08:01 +00:00
|
|
|
|
typedef FunctionTransformWithDictionary
|
2014-04-04 23:41:54 +00:00
|
|
|
|
<UInt32, RegionToCityImpl, RegionsHierarchyGetter, NameRegionToCity> FunctionRegionToCity;
|
2014-04-04 18:08:01 +00:00
|
|
|
|
|
|
|
|
|
typedef FunctionTransformWithDictionary
|
2014-04-04 23:41:54 +00:00
|
|
|
|
<UInt32, RegionToAreaImpl, RegionsHierarchyGetter, NameRegionToArea> FunctionRegionToArea;
|
2014-04-04 18:08:01 +00:00
|
|
|
|
|
|
|
|
|
typedef FunctionTransformWithDictionary
|
2014-04-04 23:41:54 +00:00
|
|
|
|
<UInt32, RegionToCountryImpl, RegionsHierarchyGetter, NameRegionToCountry> FunctionRegionToCountry;
|
2014-04-04 18:08:01 +00:00
|
|
|
|
|
|
|
|
|
typedef FunctionTransformWithDictionary
|
2014-04-04 23:41:54 +00:00
|
|
|
|
<UInt32, RegionToContinentImpl, RegionsHierarchyGetter, NameRegionToContinent> FunctionRegionToContinent;
|
2014-04-04 18:08:01 +00:00
|
|
|
|
|
|
|
|
|
typedef FunctionTransformWithDictionary
|
2014-04-04 23:41:54 +00:00
|
|
|
|
<UInt8, OSToRootImpl, IdentityDictionaryGetter<TechDataHierarchy>, NameOSToRoot> FunctionOSToRoot;
|
2014-04-04 18:08:01 +00:00
|
|
|
|
|
|
|
|
|
typedef FunctionTransformWithDictionary
|
2014-04-04 23:41:54 +00:00
|
|
|
|
<UInt8, SEToRootImpl, IdentityDictionaryGetter<TechDataHierarchy>, NameSEToRoot> FunctionSEToRoot;
|
2014-04-04 18:08:01 +00:00
|
|
|
|
|
|
|
|
|
typedef FunctionTransformWithDictionary
|
2014-04-04 23:41:54 +00:00
|
|
|
|
<UInt16, CategoryToRootImpl, IdentityDictionaryGetter<CategoriesHierarchy>, NameCategoryToRoot> FunctionCategoryToRoot;
|
2014-04-04 18:08:01 +00:00
|
|
|
|
|
|
|
|
|
typedef FunctionTransformWithDictionary
|
2014-04-04 23:41:54 +00:00
|
|
|
|
<UInt16, CategoryToSecondLevelImpl, IdentityDictionaryGetter<CategoriesHierarchy>, NameCategoryToSecondLevel> FunctionCategoryToSecondLevel;
|
2013-04-17 11:37:34 +00:00
|
|
|
|
|
2014-04-04 18:08:01 +00:00
|
|
|
|
typedef FunctionIsInWithDictionary
|
2014-04-04 23:41:54 +00:00
|
|
|
|
<UInt32, RegionInImpl, RegionsHierarchyGetter, NameRegionIn> FunctionRegionIn;
|
2013-04-17 11:37:34 +00:00
|
|
|
|
|
2014-04-04 18:08:01 +00:00
|
|
|
|
typedef FunctionIsInWithDictionary
|
2014-04-04 23:41:54 +00:00
|
|
|
|
<UInt8, OSInImpl, IdentityDictionaryGetter<TechDataHierarchy>, NameOSIn> FunctionOSIn;
|
2014-04-04 18:08:01 +00:00
|
|
|
|
|
|
|
|
|
typedef FunctionIsInWithDictionary
|
2014-04-04 23:41:54 +00:00
|
|
|
|
<UInt8, SEInImpl, IdentityDictionaryGetter<TechDataHierarchy>, NameSEIn> FunctionSEIn;
|
2014-04-04 18:08:01 +00:00
|
|
|
|
|
|
|
|
|
typedef FunctionIsInWithDictionary
|
2014-04-04 23:41:54 +00:00
|
|
|
|
<UInt16, CategoryInImpl, IdentityDictionaryGetter<CategoriesHierarchy>, NameCategoryIn> FunctionCategoryIn;
|
2014-04-04 18:08:01 +00:00
|
|
|
|
|
|
|
|
|
typedef FunctionHierarchyWithDictionary
|
2014-04-04 23:41:54 +00:00
|
|
|
|
<UInt32, RegionHierarchyImpl, RegionsHierarchyGetter, NameRegionHierarchy> FunctionRegionHierarchy;
|
2014-04-04 18:08:01 +00:00
|
|
|
|
|
|
|
|
|
typedef FunctionHierarchyWithDictionary
|
2014-04-04 23:41:54 +00:00
|
|
|
|
<UInt8, OSHierarchyImpl, IdentityDictionaryGetter<TechDataHierarchy>, NameOSHierarchy> FunctionOSHierarchy;
|
2014-04-04 18:08:01 +00:00
|
|
|
|
|
|
|
|
|
typedef FunctionHierarchyWithDictionary
|
2014-04-04 23:41:54 +00:00
|
|
|
|
<UInt8, SEHierarchyImpl, IdentityDictionaryGetter<TechDataHierarchy>, NameSEHierarchy> FunctionSEHierarchy;
|
2014-04-04 18:08:01 +00:00
|
|
|
|
|
|
|
|
|
typedef FunctionHierarchyWithDictionary
|
2014-04-04 23:41:54 +00:00
|
|
|
|
<UInt16, CategoryHierarchyImpl, IdentityDictionaryGetter<CategoriesHierarchy>, NameCategoryHierarchy> FunctionCategoryHierarchy;
|
2013-03-06 12:03:52 +00:00
|
|
|
|
|
2013-06-05 13:40:14 +00:00
|
|
|
|
|
|
|
|
|
/// Преобразует числовой идентификатор региона в имя на заданном языке, используя словарь.
|
|
|
|
|
class FunctionRegionToName : public IFunction
|
|
|
|
|
{
|
|
|
|
|
private:
|
|
|
|
|
const SharedPtr<RegionsNames> owned_dict;
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-06-05 13:40:14 +00:00
|
|
|
|
public:
|
|
|
|
|
FunctionRegionToName(const SharedPtr<RegionsNames> & owned_dict_)
|
|
|
|
|
: owned_dict(owned_dict_)
|
|
|
|
|
{
|
|
|
|
|
if (!owned_dict)
|
|
|
|
|
throw Exception("Dictionaries was not loaded. You need to check configuration file.", ErrorCodes::DICTIONARIES_WAS_NOT_LOADED);
|
|
|
|
|
}
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-06-05 13:40:14 +00:00
|
|
|
|
/// Получить имя функции.
|
|
|
|
|
String getName() const
|
|
|
|
|
{
|
|
|
|
|
return "regionToName";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Получить тип результата по типам аргументов. Если функция неприменима для данных аргументов - кинуть исключение.
|
|
|
|
|
DataTypePtr getReturnType(const DataTypes & arguments) const
|
|
|
|
|
{
|
2013-06-05 13:54:05 +00:00
|
|
|
|
if (arguments.size() != 1 && arguments.size() != 2)
|
2013-06-05 13:40:14 +00:00
|
|
|
|
throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
|
2013-06-21 20:34:19 +00:00
|
|
|
|
+ toString(arguments.size()) + ", should be 1 or 2.",
|
2013-06-05 13:40:14 +00:00
|
|
|
|
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
|
|
|
|
|
|
|
|
|
if (arguments[0]->getName() != TypeName<UInt32>::get())
|
|
|
|
|
throw Exception("Illegal type " + arguments[0]->getName() + " of the first argument of function " + getName()
|
|
|
|
|
+ " (must be " + TypeName<UInt32>::get() + ")",
|
|
|
|
|
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-06-05 13:40:14 +00:00
|
|
|
|
if (arguments.size() == 2 && arguments[1]->getName() != TypeName<String>::get())
|
|
|
|
|
throw Exception("Illegal type " + arguments[0]->getName() + " of the second argument of function " + getName()
|
|
|
|
|
+ " (must be " + TypeName<String>::get() + ")",
|
|
|
|
|
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
|
|
|
|
|
|
|
|
|
return new DataTypeString;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Выполнить функцию над блоком.
|
|
|
|
|
void execute(Block & block, const ColumnNumbers & arguments, size_t result)
|
|
|
|
|
{
|
|
|
|
|
RegionsNames::SupportedLanguages::Enum language = RegionsNames::SupportedLanguages::RU;
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-06-05 13:40:14 +00:00
|
|
|
|
/// Если указан язык результата
|
|
|
|
|
if (arguments.size() == 2)
|
|
|
|
|
{
|
2014-06-26 00:58:14 +00:00
|
|
|
|
if (const ColumnConstString * col_language = typeid_cast<const ColumnConstString *>(&*block.getByPosition(arguments[1]).column))
|
2013-06-05 13:40:14 +00:00
|
|
|
|
language = RegionsNames::getLanguageEnum(col_language->getData());
|
|
|
|
|
else
|
|
|
|
|
throw Exception("Illegal column " + block.getByPosition(arguments[1]).column->getName()
|
|
|
|
|
+ " of the second argument of function " + getName(),
|
|
|
|
|
ErrorCodes::ILLEGAL_COLUMN);
|
|
|
|
|
}
|
2014-04-04 18:08:01 +00:00
|
|
|
|
|
|
|
|
|
const RegionsNames & dict = *owned_dict;
|
|
|
|
|
|
2014-06-26 00:58:14 +00:00
|
|
|
|
if (const ColumnVector<UInt32> * col_from = typeid_cast<const ColumnVector<UInt32> *>(&*block.getByPosition(arguments[0]).column))
|
2013-06-05 13:40:14 +00:00
|
|
|
|
{
|
|
|
|
|
ColumnString * col_to = new ColumnString;
|
|
|
|
|
block.getByPosition(result).column = col_to;
|
|
|
|
|
|
|
|
|
|
const ColumnVector<UInt32>::Container_t & region_ids = col_from->getData();
|
|
|
|
|
|
|
|
|
|
for (size_t i = 0; i < region_ids.size(); ++i)
|
|
|
|
|
{
|
2014-04-28 01:48:24 +00:00
|
|
|
|
const StringRef & name_ref = dict.getRegionName(region_ids[i], language);
|
2013-06-05 13:40:14 +00:00
|
|
|
|
col_to->insertDataWithTerminatingZero(name_ref.data, name_ref.size + 1);
|
|
|
|
|
}
|
|
|
|
|
}
|
2014-06-26 00:58:14 +00:00
|
|
|
|
else if (const ColumnConst<UInt32> * col_from = typeid_cast<const ColumnConst<UInt32> *>(&*block.getByPosition(arguments[0]).column))
|
2013-06-05 13:40:14 +00:00
|
|
|
|
{
|
|
|
|
|
UInt32 region_id = col_from->getData();
|
2014-04-28 01:48:24 +00:00
|
|
|
|
const StringRef & name_ref = dict.getRegionName(region_id, language);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-06-05 13:40:14 +00:00
|
|
|
|
block.getByPosition(result).column = new ColumnConstString(col_from->size(), name_ref.toString());
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName()
|
|
|
|
|
+ " of the first argument of function " + getName(),
|
|
|
|
|
ErrorCodes::ILLEGAL_COLUMN);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
2012-12-21 19:48:47 +00:00
|
|
|
|
}
|