ClickHouse/dbms/src/Functions/convertCharset.cpp

221 lines
7.6 KiB
C++
Raw Normal View History

#include "config_core.h"
#if USE_ICU
#include <Functions/IFunction.h>
#include <Functions/FunctionFactory.h>
ColumnConst unification (#1011) * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * Fixed error in ColumnArray::replicateGeneric [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150].
2017-07-21 06:35:58 +00:00
#include <Functions/FunctionHelpers.h>
#include <IO/WriteHelpers.h>
#include <DataTypes/DataTypeString.h>
#include <Columns/ColumnString.h>
#include <Columns/ColumnConst.h>
2017-07-13 20:58:19 +00:00
#include <Common/typeid_cast.h>
2018-08-30 18:40:46 +00:00
#include <Common/ObjectPool.h>
2017-06-06 17:18:32 +00:00
#include <ext/range.h>
#include <unicode/ucnv.h>
2018-04-24 07:16:39 +00:00
#include <string>
#include <memory>
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int LOGICAL_ERROR;
extern const int CANNOT_CREATE_CHARSET_CONVERTER;
extern const int CANNOT_CONVERT_CHARSET;
2017-06-13 02:06:53 +00:00
extern const int ILLEGAL_COLUMN;
}
/** convertCharset(s, from, to)
*
* Assuming string 's' contains bytes in charset 'from',
* returns another string with bytes, representing same content in charset 'to'.
* from and to must be constants.
*
* When bytes are illegal in 'from' charset or are not representable in 'to' charset,
* behavior is implementation specific.
*/
class FunctionConvertCharset : public IFunction
{
private:
struct Converter : private boost::noncopyable
{
UConverter * impl;
explicit Converter(const String & charset)
{
UErrorCode status = U_ZERO_ERROR;
impl = ucnv_open(charset.data(), &status);
if (U_SUCCESS(status))
ucnv_setToUCallBack(impl,
UCNV_TO_U_CALLBACK_SUBSTITUTE,
nullptr,
nullptr, nullptr,
&status);
if (U_SUCCESS(status))
ucnv_setFromUCallBack(impl,
UCNV_FROM_U_CALLBACK_SUBSTITUTE,
nullptr,
nullptr, nullptr,
&status);
if (!U_SUCCESS(status))
throw Exception("Cannot create UConverter with charset " + charset + ", error: " + String(u_errorName(status)),
ErrorCodes::CANNOT_CREATE_CHARSET_CONVERTER);
}
~Converter()
{
ucnv_close(impl);
}
};
/// Separate converter is created for each thread.
using Pool = ObjectPoolMap<Converter, String>;
Pool::Pointer getConverter(const String & charset)
{
static Pool pool;
return pool.get(charset, [&charset] { return new Converter(charset); });
}
void convert(const String & from_charset, const String & to_charset,
const ColumnString::Chars & from_chars, const ColumnString::Offsets & from_offsets,
ColumnString::Chars & to_chars, ColumnString::Offsets & to_offsets)
{
auto converter_from = getConverter(from_charset);
auto converter_to = getConverter(to_charset);
ColumnString::Offset current_from_offset = 0;
ColumnString::Offset current_to_offset = 0;
size_t size = from_offsets.size();
to_offsets.resize(size);
PODArray<UChar> uchars;
for (size_t i = 0; i < size; ++i)
{
size_t from_string_size = from_offsets[i] - current_from_offset - 1;
/// We assume that empty string is empty in every charset.
if (0 != from_string_size)
{
/// reset state of converter
ucnv_reset(converter_from->impl);
ucnv_reset(converter_to->impl);
/// maximum number of code points is number of bytes in input string plus one for terminating zero
uchars.resize(from_string_size + 1);
UErrorCode status = U_ZERO_ERROR;
int32_t res = ucnv_toUChars(
converter_from->impl,
uchars.data(), uchars.size(),
reinterpret_cast<const char *>(&from_chars[current_from_offset]), from_string_size,
&status);
if (!U_SUCCESS(status))
throw Exception("Cannot convert from charset " + from_charset + ", error: " + String(u_errorName(status)),
ErrorCodes::CANNOT_CONVERT_CHARSET);
auto max_to_char_size = ucnv_getMaxCharSize(converter_to->impl);
auto max_to_size = UCNV_GET_MAX_BYTES_FOR_STRING(res, max_to_char_size);
to_chars.resize(current_to_offset + max_to_size);
res = ucnv_fromUChars(
converter_to->impl,
reinterpret_cast<char *>(&to_chars[current_to_offset]), max_to_size,
uchars.data(), res,
&status);
if (!U_SUCCESS(status))
throw Exception("Cannot convert to charset " + to_charset + ", error: " + String(u_errorName(status)),
ErrorCodes::CANNOT_CONVERT_CHARSET);
current_to_offset += res;
}
if (to_chars.size() < current_to_offset + 1)
to_chars.resize(current_to_offset + 1);
to_chars[current_to_offset] = 0;
++current_to_offset;
to_offsets[i] = current_to_offset;
current_from_offset = from_offsets[i];
}
to_chars.resize(current_to_offset);
}
public:
static constexpr auto name = "convertCharset";
2017-12-02 02:47:12 +00:00
static FunctionPtr create(const Context &) { return std::make_shared<FunctionConvertCharset>(); }
String getName() const override
{
return name;
}
size_t getNumberOfArguments() const override { return 3; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
for (size_t i : ext::range(0, 3))
if (!isString(arguments[i]))
throw Exception("Illegal type " + arguments[i]->getName() + " of argument of function " + getName()
+ ", must be String", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
return std::make_shared<DataTypeString>();
}
bool useDefaultImplementationForConstants() const override { return true; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; }
2018-04-24 07:16:39 +00:00
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override
{
const ColumnWithTypeAndName & arg_from = block.getByPosition(arguments[0]);
const ColumnWithTypeAndName & arg_charset_from = block.getByPosition(arguments[1]);
const ColumnWithTypeAndName & arg_charset_to = block.getByPosition(arguments[2]);
ColumnConst unification (#1011) * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * Fixed error in ColumnArray::replicateGeneric [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150].
2017-07-21 06:35:58 +00:00
const ColumnConst * col_charset_from = checkAndGetColumnConstStringOrFixedString(arg_charset_from.column.get());
const ColumnConst * col_charset_to = checkAndGetColumnConstStringOrFixedString(arg_charset_to.column.get());
if (!col_charset_from || !col_charset_to)
throw Exception("2nd and 3rd arguments of function " + getName() + " (source charset and destination charset) must be constant strings.",
ErrorCodes::ILLEGAL_COLUMN);
ColumnConst unification (#1011) * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * Fixed error in ColumnArray::replicateGeneric [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150].
2017-07-21 06:35:58 +00:00
String charset_from = col_charset_from->getValue<String>();
String charset_to = col_charset_to->getValue<String>();
ColumnConst unification (#1011) * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * Fixed error in ColumnArray::replicateGeneric [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150].
2017-07-21 06:35:58 +00:00
if (const ColumnString * col_from = checkAndGetColumn<ColumnString>(arg_from.column.get()))
{
auto col_to = ColumnString::create();
convert(charset_from, charset_to, col_from->getChars(), col_from->getOffsets(), col_to->getChars(), col_to->getOffsets());
block.getByPosition(result).column = std::move(col_to);
}
else
throw Exception("Illegal column passed as first argument of function " + getName() + " (must be ColumnString).",
ErrorCodes::ILLEGAL_COLUMN);
}
};
void registerFunctionConvertCharset(FunctionFactory & factory)
{
factory.registerFunction<FunctionConvertCharset>();
}
}
#endif