mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-22 09:40:49 +00:00
Merge branch 'master' into fix-fuzz-test6
This commit is contained in:
commit
deb483eee5
@ -121,3 +121,18 @@ template <> struct is_big_int<bInt256> { static constexpr bool value = true; };
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_big_int_v = is_big_int<T>::value;
|
||||
|
||||
template <typename T>
|
||||
inline std::string bigintToString(const T & x)
|
||||
{
|
||||
return x.str();
|
||||
}
|
||||
|
||||
template <typename To, typename From>
|
||||
inline To bigint_cast(const From & x [[maybe_unused]])
|
||||
{
|
||||
if constexpr ((is_big_int_v<From> && std::is_same_v<To, UInt8>) || (is_big_int_v<To> && std::is_same_v<From, UInt8>))
|
||||
return static_cast<uint8_t>(x);
|
||||
else
|
||||
return static_cast<To>(x);
|
||||
}
|
||||
|
@ -2,6 +2,15 @@
|
||||
<profiles>
|
||||
<default>
|
||||
<max_execution_time>10</max_execution_time>
|
||||
<!--
|
||||
Don't let the fuzzer change this setting (I've actually seen it
|
||||
do this before).
|
||||
-->
|
||||
<constraints>
|
||||
<max_execution_time>
|
||||
<max>10</max>
|
||||
</max_execution_time>
|
||||
</constraints>
|
||||
</default>
|
||||
</profiles>
|
||||
</yandex>
|
||||
|
@ -91,7 +91,7 @@ function fuzz
|
||||
|
||||
./clickhouse-client --query "select elapsed, query from system.processes" ||:
|
||||
killall clickhouse-server ||:
|
||||
for x in {1..10}
|
||||
for _ in {1..10}
|
||||
do
|
||||
if ! pgrep -f clickhouse-server
|
||||
then
|
||||
@ -172,8 +172,60 @@ case "$stage" in
|
||||
echo "failure" > status.txt
|
||||
echo "Fuzzer failed ($fuzzer_exit_code). See the logs" > description.txt
|
||||
fi
|
||||
;&
|
||||
"report")
|
||||
cat > report.html <<EOF ||:
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<link rel="preload" as="font" href="https://yastatic.net/adv-www/_/sUYVCPUAQE7ExrvMS7FoISoO83s.woff2" type="font/woff2" crossorigin="anonymous"/>
|
||||
<style>
|
||||
@font-face {
|
||||
font-family:'Yandex Sans Display Web';
|
||||
src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot);
|
||||
src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot?#iefix) format('embedded-opentype'),
|
||||
url(https://yastatic.net/adv-www/_/sUYVCPUAQE7ExrvMS7FoISoO83s.woff2) format('woff2'),
|
||||
url(https://yastatic.net/adv-www/_/v2Sve_obH3rKm6rKrtSQpf-eB7U.woff) format('woff'),
|
||||
url(https://yastatic.net/adv-www/_/PzD8hWLMunow5i3RfJ6WQJAL7aI.ttf) format('truetype'),
|
||||
url(https://yastatic.net/adv-www/_/lF_KG5g4tpQNlYIgA0e77fBSZ5s.svg#YandexSansDisplayWeb-Regular) format('svg');
|
||||
font-weight:400;
|
||||
font-style:normal;
|
||||
font-stretch:normal
|
||||
}
|
||||
|
||||
exit $task_exit_code
|
||||
body { font-family: "Yandex Sans Display Web", Arial, sans-serif; background: #EEE; }
|
||||
h1 { margin-left: 10px; }
|
||||
th, td { border: 0; padding: 5px 10px 5px 10px; text-align: left; vertical-align: top; line-height: 1.5; background-color: #FFF;
|
||||
td { white-space: pre; font-family: Monospace, Courier New; }
|
||||
border: 0; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }
|
||||
a { color: #06F; text-decoration: none; }
|
||||
a:hover, a:active { color: #F40; text-decoration: underline; }
|
||||
table { border: 0; }
|
||||
.main { margin-left: 10%; }
|
||||
p.links a { padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-space: nowrap; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }
|
||||
th { cursor: pointer; }
|
||||
|
||||
</style>
|
||||
<title>AST Fuzzer for PR #${PR_TO_TEST} @ ${SHA_TO_TEST}</title>
|
||||
</head>
|
||||
<body>
|
||||
<div class="main">
|
||||
|
||||
<h1>AST Fuzzer for PR #${PR_TO_TEST} @ ${SHA_TO_TEST}</h1>
|
||||
<p class="links">
|
||||
<a href="fuzzer.log">fuzzer.log</a>
|
||||
<a href="server.log">server.log</a>
|
||||
<a href="main.log">main.log</a>
|
||||
</p>
|
||||
<table>
|
||||
<tr><th>Test name</th><th>Test status</th><th>Description</th></tr>
|
||||
<tr><td>AST Fuzzer</td><td>$(cat status.txt)</td><td>$(cat description.txt)</td></tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
EOF
|
||||
;&
|
||||
esac
|
||||
|
||||
exit $task_exit_code
|
||||
|
||||
|
@ -5,3 +5,4 @@ services:
|
||||
restart: always
|
||||
ports:
|
||||
- 6380:6379
|
||||
command: redis-server --requirepass "clickhouse"
|
||||
|
@ -536,7 +536,9 @@ create table queries engine File(TSVWithNamesAndTypes, 'report/queries.tsv')
|
||||
left join query_display_names
|
||||
on query_metric_stats.test = query_display_names.test
|
||||
and query_metric_stats.query_index = query_display_names.query_index
|
||||
where metric_name = 'server_time'
|
||||
-- 'server_time' is rounded down to ms, which might be bad for very short queries.
|
||||
-- Use 'client_time' instead.
|
||||
where metric_name = 'client_time'
|
||||
order by test, query_index, metric_name
|
||||
;
|
||||
|
||||
@ -888,7 +890,10 @@ for log in *-err.log
|
||||
do
|
||||
test=$(basename "$log" "-err.log")
|
||||
{
|
||||
grep -H -m2 -i '\(Exception\|Error\):[^:]' "$log" \
|
||||
# The second grep is a heuristic for error messages like
|
||||
# "socket.timeout: timed out".
|
||||
grep -h -m2 -i '\(Exception\|Error\):[^:]' "$log" \
|
||||
|| grep -h -m2 -i '^[^ ]\+: ' "$log" \
|
||||
|| head -2 "$log"
|
||||
} | sed "s/^/$test\t/" >> run-errors.tsv ||:
|
||||
done
|
||||
|
@ -168,12 +168,6 @@ def nextRowAnchor():
|
||||
global table_anchor
|
||||
return f'{table_anchor}.{row_anchor + 1}'
|
||||
|
||||
def setRowAnchor(anchor_row_part):
|
||||
global row_anchor
|
||||
global table_anchor
|
||||
row_anchor = anchor_row_part
|
||||
return currentRowAnchor()
|
||||
|
||||
def advanceRowAnchor():
|
||||
global row_anchor
|
||||
global table_anchor
|
||||
@ -480,11 +474,12 @@ if args.report == 'main':
|
||||
total_runs = (nominal_runs + 1) * 2 # one prewarm run, two servers
|
||||
attrs = ['' for c in columns]
|
||||
for r in rows:
|
||||
anchor = f'{currentTableAnchor()}.{r[0]}'
|
||||
if float(r[6]) > 1.5 * total_runs:
|
||||
# FIXME should be 15s max -- investigate parallel_insert
|
||||
slow_average_tests += 1
|
||||
attrs[6] = f'style="background: {color_bad}"'
|
||||
errors_explained.append([f'<a href="./all-queries.html#all-query-times.{r[0]}.0">The test \'{r[0]}\' is too slow to run as a whole. Investigate whether the create and fill queries can be sped up'])
|
||||
errors_explained.append([f'<a href="#{anchor}">The test \'{r[0]}\' is too slow to run as a whole. Investigate whether the create and fill queries can be sped up'])
|
||||
else:
|
||||
attrs[6] = ''
|
||||
|
||||
@ -495,7 +490,7 @@ if args.report == 'main':
|
||||
else:
|
||||
attrs[5] = ''
|
||||
|
||||
text += tableRow(r, attrs)
|
||||
text += tableRow(r, attrs, anchor)
|
||||
|
||||
text += tableEnd()
|
||||
tables.append(text)
|
||||
|
@ -31,7 +31,7 @@ For a description of request parameters, see [statement description](../../../sq
|
||||
|
||||
**ReplacingMergeTree Parameters**
|
||||
|
||||
- `ver` — column with version. Type `UInt*`, `Date`, `DateTime` or `DateTime64`. Optional parameter.
|
||||
- `ver` — column with version. Type `UInt*`, `Date` or `DateTime`. Optional parameter.
|
||||
|
||||
When merging, `ReplacingMergeTree` from all the rows with the same sorting key leaves only one:
|
||||
|
||||
|
@ -136,7 +136,7 @@ ENGINE = <Engine>
|
||||
...
|
||||
```
|
||||
|
||||
If a codec is specified, the default codec doesn’t apply. Codecs can be combined in a pipeline, for example, `CODEC(Delta, ZSTD)`. To select the best codec combination for you project, pass benchmarks similar to described in the Altinity [New Encodings to Improve ClickHouse Efficiency](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse) article.
|
||||
If a codec is specified, the default codec doesn’t apply. Codecs can be combined in a pipeline, for example, `CODEC(Delta, ZSTD)`. To select the best codec combination for you project, pass benchmarks similar to described in the Altinity [New Encodings to Improve ClickHouse Efficiency](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse) article. One thing to note is that codec can't be applied for ALIAS column type.
|
||||
|
||||
!!! warning "Warning"
|
||||
You can’t decompress ClickHouse database files with external utilities like `lz4`. Instead, use the special [clickhouse-compressor](https://github.com/ClickHouse/ClickHouse/tree/master/programs/compressor) utility.
|
||||
|
@ -33,7 +33,7 @@ Para obtener una descripción de los parámetros de solicitud, consulte [descrip
|
||||
|
||||
**ReplacingMergeTree Parámetros**
|
||||
|
||||
- `ver` — column with version. Type `UInt*`, `Date`, `DateTime` o `DateTime64`. Parámetro opcional.
|
||||
- `ver` — column with version. Type `UInt*`, `Date` o `DateTime`. Parámetro opcional.
|
||||
|
||||
Al fusionar, `ReplacingMergeTree` de todas las filas con la misma clave primaria deja solo una:
|
||||
|
||||
|
@ -33,7 +33,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
|
||||
**پارامترهای جایگزین**
|
||||
|
||||
- `ver` — column with version. Type `UInt*`, `Date`, `DateTime` یا `DateTime64`. پارامتر اختیاری.
|
||||
- `ver` — column with version. Type `UInt*`, `Date` یا `DateTime`. پارامتر اختیاری.
|
||||
|
||||
هنگام ادغام, `ReplacingMergeTree` از تمام ردیف ها با همان کلید اصلی تنها یک برگ دارد:
|
||||
|
||||
|
@ -33,7 +33,7 @@ Pour une description des paramètres de requête, voir [demande de description](
|
||||
|
||||
**ReplacingMergeTree Paramètres**
|
||||
|
||||
- `ver` — column with version. Type `UInt*`, `Date`, `DateTime` ou `DateTime64`. Paramètre facultatif.
|
||||
- `ver` — column with version. Type `UInt*`, `Date` ou `DateTime`. Paramètre facultatif.
|
||||
|
||||
Lors de la fusion, `ReplacingMergeTree` de toutes les lignes avec la même clé primaire ne laisse qu'un:
|
||||
|
||||
|
@ -33,7 +33,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
|
||||
**ReplacingMergeTreeパラメータ**
|
||||
|
||||
- `ver` — column with version. Type `UInt*`, `Date`, `DateTime` または `DateTime64`. 任意パラメータ。
|
||||
- `ver` — column with version. Type `UInt*`, `Date` または `DateTime`. 任意パラメータ。
|
||||
|
||||
マージ時, `ReplacingMergeTree` 同じ主キーを持つすべての行から、一つだけを残します:
|
||||
|
||||
|
@ -25,7 +25,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
|
||||
**Параметры ReplacingMergeTree**
|
||||
|
||||
- `ver` — столбец с версией, тип `UInt*`, `Date`, `DateTime` или `DateTime64`. Необязательный параметр.
|
||||
- `ver` — столбец с версией, тип `UInt*`, `Date` или `DateTime`. Необязательный параметр.
|
||||
|
||||
При слиянии, из всех строк с одинаковым значением ключа сортировки `ReplacingMergeTree` оставляет только одну:
|
||||
|
||||
|
@ -33,7 +33,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
|
||||
**ReplacingMergeTree Parametreleri**
|
||||
|
||||
- `ver` — column with version. Type `UInt*`, `Date`, `DateTime` veya `DateTime64`. İsteğe bağlı parametre.
|
||||
- `ver` — column with version. Type `UInt*`, `Date` veya `DateTime`. İsteğe bağlı parametre.
|
||||
|
||||
Birleş whenirken, `ReplacingMergeTree` aynı birincil anahtara sahip tüm satırlardan sadece bir tane bırakır:
|
||||
|
||||
|
@ -25,7 +25,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
|
||||
**参数**
|
||||
|
||||
- `ver` — 版本列。类型为 `UInt*`, `Date`, `DateTime` 或 `DateTime64`。可选参数。
|
||||
- `ver` — 版本列。类型为 `UInt*`, `Date` 或 `DateTime`。可选参数。
|
||||
|
||||
合并的时候,`ReplacingMergeTree` 从所有具有相同主键的行中选择一行留下:
|
||||
- 如果 `ver` 列未指定,选择最后一条。
|
||||
|
@ -847,32 +847,26 @@ private:
|
||||
}
|
||||
|
||||
// Parse and execute what we've read.
|
||||
fprintf(stderr, "will now parse '%s'\n", text.c_str());
|
||||
|
||||
const auto * new_end = processWithFuzzing(text);
|
||||
|
||||
if (new_end > &text[0])
|
||||
{
|
||||
const auto rest_size = text.size() - (new_end - &text[0]);
|
||||
|
||||
fprintf(stderr, "total %zd, rest %zd\n", text.size(), rest_size);
|
||||
|
||||
memcpy(&text[0], new_end, rest_size);
|
||||
text.resize(rest_size);
|
||||
}
|
||||
else
|
||||
{
|
||||
fprintf(stderr, "total %zd, can't parse\n", text.size());
|
||||
// We didn't read enough text to parse a query. Will read more.
|
||||
}
|
||||
|
||||
if (!connection->isConnected())
|
||||
{
|
||||
// Uh-oh...
|
||||
std::cerr << "Lost connection to the server." << std::endl;
|
||||
last_exception_received_from_server
|
||||
= std::make_unique<Exception>(210, "~");
|
||||
return;
|
||||
}
|
||||
// Ensure that we're still connected to the server. If the server died,
|
||||
// the reconnect is going to fail with an exception, and the fuzzer
|
||||
// will exit. The ping() would be the best match here, but it's
|
||||
// private, probably for a good reason that the protocol doesn't allow
|
||||
// pings at any possible moment.
|
||||
connection->forceConnected(connection_parameters.timeouts);
|
||||
|
||||
if (text.size() > 4 * 1024)
|
||||
{
|
||||
@ -880,9 +874,6 @@ private:
|
||||
// and we still cannot parse a single query in it. Abort.
|
||||
std::cerr << "Read too much text and still can't parse a query."
|
||||
" Aborting." << std::endl;
|
||||
last_exception_received_from_server
|
||||
= std::make_unique<Exception>(1, "~");
|
||||
// return;
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
@ -106,8 +106,8 @@ AggregateFunctionPtr createAggregateFunctionQuantile(const std::string & name, c
|
||||
if constexpr (supportBigInt<Function>())
|
||||
{
|
||||
if (which.idx == TypeIndex::Int128) return std::make_shared<Function<Int128, true>>(argument_types, params);
|
||||
if (which.idx == TypeIndex::bInt256) return std::make_shared<Function<bInt256, true>>(argument_types, params);
|
||||
if (which.idx == TypeIndex::bUInt256) return std::make_shared<Function<bUInt256, true>>(argument_types, params);
|
||||
if (which.idx == TypeIndex::Int256) return std::make_shared<Function<Int256, true>>(argument_types, params);
|
||||
if (which.idx == TypeIndex::UInt256) return std::make_shared<Function<UInt256, true>>(argument_types, params);
|
||||
}
|
||||
|
||||
throw Exception("Illegal type " + argument_type->getName() + " of argument for aggregate function " + name,
|
||||
|
@ -137,23 +137,23 @@ struct AggregateFunctionUniqUpToData<UInt128> : AggregateFunctionUniqUpToData<UI
|
||||
};
|
||||
|
||||
template <>
|
||||
struct AggregateFunctionUniqUpToData<bUInt256> : AggregateFunctionUniqUpToData<UInt64>
|
||||
struct AggregateFunctionUniqUpToData<UInt256> : AggregateFunctionUniqUpToData<UInt64>
|
||||
{
|
||||
/// ALWAYS_INLINE is required to have better code layout for uniqUpTo function
|
||||
void ALWAYS_INLINE add(const IColumn & column, size_t row_num, UInt8 threshold)
|
||||
{
|
||||
bUInt256 value = assert_cast<const ColumnVector<bUInt256> &>(column).getData()[row_num];
|
||||
UInt256 value = assert_cast<const ColumnVector<UInt256> &>(column).getData()[row_num];
|
||||
insert(sipHash64(value), threshold);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct AggregateFunctionUniqUpToData<bInt256> : AggregateFunctionUniqUpToData<UInt64>
|
||||
struct AggregateFunctionUniqUpToData<Int256> : AggregateFunctionUniqUpToData<UInt64>
|
||||
{
|
||||
/// ALWAYS_INLINE is required to have better code layout for uniqUpTo function
|
||||
void ALWAYS_INLINE add(const IColumn & column, size_t row_num, UInt8 threshold)
|
||||
{
|
||||
bInt256 value = assert_cast<const ColumnVector<bInt256> &>(column).getData()[row_num];
|
||||
Int256 value = assert_cast<const ColumnVector<Int256> &>(column).getData()[row_num];
|
||||
insert(sipHash64(value), threshold);
|
||||
}
|
||||
};
|
||||
|
@ -20,13 +20,13 @@
|
||||
M(UInt16) \
|
||||
M(UInt32) \
|
||||
M(UInt64) \
|
||||
M(bUInt256) \
|
||||
M(UInt256) \
|
||||
M(Int8) \
|
||||
M(Int16) \
|
||||
M(Int32) \
|
||||
M(Int64) \
|
||||
M(Int128) \
|
||||
M(bInt256) \
|
||||
M(Int256) \
|
||||
M(Float32) \
|
||||
M(Float64)
|
||||
|
||||
|
@ -61,7 +61,7 @@ StringRef ColumnDecimal<T>::serializeValueIntoArena(size_t n, Arena & arena, cha
|
||||
else
|
||||
{
|
||||
char * pos = arena.allocContinue(BigInt<T>::size, begin);
|
||||
return BigInt<bInt256>::serialize(data[n], pos);
|
||||
return BigInt<Int256>::serialize(data[n], pos);
|
||||
}
|
||||
}
|
||||
|
||||
@ -75,8 +75,8 @@ const char * ColumnDecimal<T>::deserializeAndInsertFromArena(const char * pos)
|
||||
}
|
||||
else
|
||||
{
|
||||
data.push_back(BigInt<bInt256>::deserialize(pos));
|
||||
return pos + BigInt<bInt256>::size;
|
||||
data.push_back(BigInt<Int256>::deserialize(pos));
|
||||
return pos + BigInt<Int256>::size;
|
||||
}
|
||||
}
|
||||
|
||||
@ -273,7 +273,7 @@ void ColumnDecimal<T>::insertData(const char * src, size_t /*length*/)
|
||||
}
|
||||
else
|
||||
{
|
||||
data.push_back(BigInt<bInt256>::deserialize(src));
|
||||
data.push_back(BigInt<Int256>::deserialize(src));
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -598,13 +598,13 @@ template class ColumnVector<UInt16>;
|
||||
template class ColumnVector<UInt32>;
|
||||
template class ColumnVector<UInt64>;
|
||||
template class ColumnVector<UInt128>;
|
||||
template class ColumnVector<bUInt256>;
|
||||
template class ColumnVector<UInt256>;
|
||||
template class ColumnVector<Int8>;
|
||||
template class ColumnVector<Int16>;
|
||||
template class ColumnVector<Int32>;
|
||||
template class ColumnVector<Int64>;
|
||||
template class ColumnVector<Int128>;
|
||||
template class ColumnVector<bInt256>;
|
||||
template class ColumnVector<Int256>;
|
||||
template class ColumnVector<Float32>;
|
||||
template class ColumnVector<Float64>;
|
||||
}
|
||||
|
@ -14,14 +14,14 @@ using ColumnUInt16 = ColumnVector<UInt16>;
|
||||
using ColumnUInt32 = ColumnVector<UInt32>;
|
||||
using ColumnUInt64 = ColumnVector<UInt64>;
|
||||
using ColumnUInt128 = ColumnVector<UInt128>;
|
||||
using ColumnUInt256 = ColumnVector<bUInt256>;
|
||||
using ColumnUInt256 = ColumnVector<UInt256>;
|
||||
|
||||
using ColumnInt8 = ColumnVector<Int8>;
|
||||
using ColumnInt16 = ColumnVector<Int16>;
|
||||
using ColumnInt32 = ColumnVector<Int32>;
|
||||
using ColumnInt64 = ColumnVector<Int64>;
|
||||
using ColumnInt128 = ColumnVector<Int128>;
|
||||
using ColumnInt256 = ColumnVector<bInt256>;
|
||||
using ColumnInt256 = ColumnVector<Int256>;
|
||||
|
||||
using ColumnFloat32 = ColumnVector<Float32>;
|
||||
using ColumnFloat64 = ColumnVector<Float64>;
|
||||
|
@ -48,8 +48,8 @@ String FieldVisitorDump::operator() (const DecimalField<Decimal32> & x) const {
|
||||
String FieldVisitorDump::operator() (const DecimalField<Decimal64> & x) const { return formatQuotedWithPrefix(x, "Decimal64_"); }
|
||||
String FieldVisitorDump::operator() (const DecimalField<Decimal128> & x) const { return formatQuotedWithPrefix(x, "Decimal128_"); }
|
||||
String FieldVisitorDump::operator() (const DecimalField<Decimal256> & x) const { return formatQuotedWithPrefix(x, "Decimal256_"); }
|
||||
String FieldVisitorDump::operator() (const bUInt256 & x) const { return formatQuotedWithPrefix(x, "UInt256_"); }
|
||||
String FieldVisitorDump::operator() (const bInt256 & x) const { return formatQuotedWithPrefix(x, "Int256_"); }
|
||||
String FieldVisitorDump::operator() (const UInt256 & x) const { return formatQuotedWithPrefix(x, "UInt256_"); }
|
||||
String FieldVisitorDump::operator() (const Int256 & x) const { return formatQuotedWithPrefix(x, "Int256_"); }
|
||||
String FieldVisitorDump::operator() (const Int128 & x) const { return formatQuotedWithPrefix(x, "Int128_"); }
|
||||
String FieldVisitorDump::operator() (const UInt128 & x) const { return formatQuotedWithPrefix(UUID(x), "UUID_"); }
|
||||
|
||||
@ -141,8 +141,8 @@ String FieldVisitorToString::operator() (const AggregateFunctionStateData & x) c
|
||||
{
|
||||
return formatQuoted(x.data);
|
||||
}
|
||||
String FieldVisitorToString::operator() (const bUInt256 & x) const { return formatQuoted(x); }
|
||||
String FieldVisitorToString::operator() (const bInt256 & x) const { return formatQuoted(x); }
|
||||
String FieldVisitorToString::operator() (const UInt256 & x) const { return formatQuoted(x); }
|
||||
String FieldVisitorToString::operator() (const Int256 & x) const { return formatQuoted(x); }
|
||||
|
||||
String FieldVisitorToString::operator() (const Array & x) const
|
||||
{
|
||||
@ -286,16 +286,16 @@ void FieldVisitorHash::operator() (const AggregateFunctionStateData & x) const
|
||||
hash.update(x.data.data(), x.data.size());
|
||||
}
|
||||
|
||||
void FieldVisitorHash::operator() (const bUInt256 & x) const
|
||||
void FieldVisitorHash::operator() (const UInt256 & x) const
|
||||
{
|
||||
UInt8 type = Field::Types::bUInt256;
|
||||
UInt8 type = Field::Types::UInt256;
|
||||
hash.update(type);
|
||||
hash.update(x);
|
||||
}
|
||||
|
||||
void FieldVisitorHash::operator() (const bInt256 & x) const
|
||||
void FieldVisitorHash::operator() (const Int256 & x) const
|
||||
{
|
||||
UInt8 type = Field::Types::bInt256;
|
||||
UInt8 type = Field::Types::Int256;
|
||||
hash.update(type);
|
||||
hash.update(x);
|
||||
}
|
||||
|
@ -82,8 +82,8 @@ public:
|
||||
String operator() (const DecimalField<Decimal256> & x) const;
|
||||
String operator() (const AggregateFunctionStateData & x) const;
|
||||
|
||||
String operator() (const bUInt256 & x) const;
|
||||
String operator() (const bInt256 & x) const;
|
||||
String operator() (const UInt256 & x) const;
|
||||
String operator() (const Int256 & x) const;
|
||||
};
|
||||
|
||||
|
||||
@ -106,8 +106,8 @@ public:
|
||||
String operator() (const DecimalField<Decimal256> & x) const;
|
||||
String operator() (const AggregateFunctionStateData & x) const;
|
||||
|
||||
String operator() (const bUInt256 & x) const;
|
||||
String operator() (const bInt256 & x) const;
|
||||
String operator() (const UInt256 & x) const;
|
||||
String operator() (const Int256 & x) const;
|
||||
};
|
||||
|
||||
|
||||
@ -143,7 +143,7 @@ public:
|
||||
T operator() (const Float64 & x) const
|
||||
{
|
||||
if constexpr (std::is_same_v<Decimal256, T>)
|
||||
return bInt256(x);
|
||||
return Int256(x);
|
||||
else
|
||||
return T(x);
|
||||
}
|
||||
@ -187,12 +187,10 @@ public:
|
||||
{
|
||||
if constexpr (IsDecimalNumber<T>)
|
||||
return static_cast<T>(static_cast<typename T::NativeType>(x));
|
||||
else if constexpr (std::is_same_v<T, UInt8>)
|
||||
return static_cast<T>(static_cast<UInt16>(x));
|
||||
else if constexpr (std::is_same_v<T, UInt128>)
|
||||
throw Exception("No conversion to old UInt128 from " + demangle(typeid(U).name()), ErrorCodes::NOT_IMPLEMENTED);
|
||||
else
|
||||
return static_cast<T>(x);
|
||||
return bigint_cast<T>(x);
|
||||
}
|
||||
};
|
||||
|
||||
@ -220,8 +218,8 @@ public:
|
||||
void operator() (const DecimalField<Decimal256> & x) const;
|
||||
void operator() (const AggregateFunctionStateData & x) const;
|
||||
|
||||
void operator() (const bUInt256 & x) const;
|
||||
void operator() (const bInt256 & x) const;
|
||||
void operator() (const UInt256 & x) const;
|
||||
void operator() (const Int256 & x) const;
|
||||
};
|
||||
|
||||
|
||||
|
@ -280,13 +280,13 @@ DEFINE_HASH(DB::UInt16)
|
||||
DEFINE_HASH(DB::UInt32)
|
||||
DEFINE_HASH(DB::UInt64)
|
||||
DEFINE_HASH(DB::UInt128)
|
||||
DEFINE_HASH(DB::bUInt256)
|
||||
DEFINE_HASH(DB::UInt256)
|
||||
DEFINE_HASH(DB::Int8)
|
||||
DEFINE_HASH(DB::Int16)
|
||||
DEFINE_HASH(DB::Int32)
|
||||
DEFINE_HASH(DB::Int64)
|
||||
DEFINE_HASH(DB::Int128)
|
||||
DEFINE_HASH(DB::bInt256)
|
||||
DEFINE_HASH(DB::Int256)
|
||||
DEFINE_HASH(DB::Float32)
|
||||
DEFINE_HASH(DB::Float64)
|
||||
|
||||
@ -297,7 +297,7 @@ template <>
|
||||
struct DefaultHash<DB::UInt128> : public DB::UInt128Hash {};
|
||||
|
||||
template <>
|
||||
struct DefaultHash<DB::UInt256> : public DB::UInt256Hash {};
|
||||
struct DefaultHash<DB::DummyUInt256> : public DB::UInt256Hash {};
|
||||
|
||||
|
||||
/// It is reasonable to use for UInt8, UInt16 with sufficient hash table size.
|
||||
|
@ -136,7 +136,7 @@ struct UInt128TrivialHash
|
||||
|
||||
/** Used for aggregation, for putting a large number of constant-length keys in a hash table.
|
||||
*/
|
||||
struct UInt256
|
||||
struct DummyUInt256
|
||||
{
|
||||
|
||||
/// Suppress gcc7 warnings: 'prev_key.DB::UInt256::a' may be used uninitialized in this function
|
||||
@ -150,7 +150,7 @@ struct UInt256
|
||||
UInt64 c;
|
||||
UInt64 d;
|
||||
|
||||
bool operator== (const UInt256 rhs) const
|
||||
bool operator== (const DummyUInt256 rhs) const
|
||||
{
|
||||
return a == rhs.a && b == rhs.b && c == rhs.c && d == rhs.d;
|
||||
|
||||
@ -164,7 +164,7 @@ struct UInt256
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(&rhs.c)))));*/
|
||||
}
|
||||
|
||||
bool operator!= (const UInt256 rhs) const { return !operator==(rhs); }
|
||||
bool operator!= (const DummyUInt256 rhs) const { return !operator==(rhs); }
|
||||
|
||||
bool operator== (const UInt64 rhs) const { return a == rhs && b == 0 && c == 0 && d == 0; }
|
||||
bool operator!= (const UInt64 rhs) const { return !operator==(rhs); }
|
||||
@ -173,12 +173,12 @@ struct UInt256
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
|
||||
UInt256 & operator= (const UInt64 rhs) { a = rhs; b = 0; c = 0; d = 0; return *this; }
|
||||
DummyUInt256 & operator = (const UInt64 rhs) { a = rhs; b = 0; c = 0; d = 0; return *this; }
|
||||
};
|
||||
|
||||
struct UInt256Hash
|
||||
{
|
||||
size_t operator()(UInt256 x) const
|
||||
size_t operator()(DummyUInt256 x) const
|
||||
{
|
||||
/// NOTE suboptimal
|
||||
return CityHash_v1_0_2::Hash128to64({CityHash_v1_0_2::Hash128to64({x.a, x.b}), CityHash_v1_0_2::Hash128to64({x.c, x.d})});
|
||||
@ -189,7 +189,7 @@ struct UInt256Hash
|
||||
|
||||
struct UInt256HashCRC32
|
||||
{
|
||||
size_t operator()(UInt256 x) const
|
||||
size_t operator()(DummyUInt256 x) const
|
||||
{
|
||||
UInt64 crc = -1ULL;
|
||||
crc = _mm_crc32_u64(crc, x.a);
|
||||
|
@ -106,6 +106,15 @@ void CompressionCodecMultiple::doDecompressData(const char * source, UInt32 sour
|
||||
memcpy(dest, compressed_buf.data(), decompressed_size);
|
||||
}
|
||||
|
||||
std::vector<uint8_t> CompressionCodecMultiple::getCodecsBytesFromData(const char * source)
|
||||
{
|
||||
std::vector<uint8_t> result;
|
||||
uint8_t compression_methods_size = source[0];
|
||||
for (size_t i = 0; i < compression_methods_size; ++i)
|
||||
result.push_back(source[1 + i]);
|
||||
return result;
|
||||
}
|
||||
|
||||
bool CompressionCodecMultiple::isCompression() const
|
||||
{
|
||||
for (const auto & codec : codecs)
|
||||
|
@ -17,6 +17,8 @@ public:
|
||||
|
||||
UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override;
|
||||
|
||||
static std::vector<uint8_t> getCodecsBytesFromData(const char * source);
|
||||
|
||||
protected:
|
||||
UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;
|
||||
|
||||
|
@ -5,6 +5,7 @@
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <common/unaligned.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Parsers/queryToString.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -20,7 +21,17 @@ ASTPtr ICompressionCodec::getFullCodecDesc() const
|
||||
{
|
||||
std::shared_ptr<ASTFunction> result = std::make_shared<ASTFunction>();
|
||||
result->name = "CODEC";
|
||||
result->arguments = getCodecDesc();
|
||||
ASTPtr codec_desc = getCodecDesc();
|
||||
if (codec_desc->as<ASTExpressionList>())
|
||||
{
|
||||
result->arguments = codec_desc;
|
||||
}
|
||||
else
|
||||
{
|
||||
result->arguments = std::make_shared<ASTExpressionList>();
|
||||
result->arguments->children.push_back(codec_desc);
|
||||
}
|
||||
result->children.push_back(result->arguments);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
43
src/Compression/getCompressionCodecForFile.cpp
Normal file
43
src/Compression/getCompressionCodecForFile.cpp
Normal file
@ -0,0 +1,43 @@
|
||||
#include <Compression/getCompressionCodecForFile.h>
|
||||
#include <Compression/CompressionInfo.h>
|
||||
#include <Compression/CompressionFactory.h>
|
||||
#include <IO/ReadBufferFromFileBase.h>
|
||||
#include <Compression/CompressionCodecMultiple.h>
|
||||
#include <Common/PODArray.h>
|
||||
#include <common/logger_useful.h>
|
||||
|
||||
#include <Common/UInt128.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
|
||||
using Checksum = CityHash_v1_0_2::uint128;
|
||||
|
||||
CompressionCodecPtr getCompressionCodecForFile(const DiskPtr & disk, const String & relative_path)
|
||||
{
|
||||
auto read_buffer = disk->readFile(relative_path);
|
||||
read_buffer->ignore(sizeof(Checksum));
|
||||
|
||||
UInt8 header_size = ICompressionCodec::getHeaderSize();
|
||||
PODArray<char> compressed_buffer;
|
||||
compressed_buffer.resize(header_size);
|
||||
read_buffer->readStrict(compressed_buffer.data(), header_size);
|
||||
uint8_t method = ICompressionCodec::readMethod(compressed_buffer.data());
|
||||
if (method == static_cast<uint8_t>(CompressionMethodByte::Multiple))
|
||||
{
|
||||
compressed_buffer.resize(1);
|
||||
read_buffer->readStrict(compressed_buffer.data(), 1);
|
||||
compressed_buffer.resize(1 + compressed_buffer[0]);
|
||||
read_buffer->readStrict(compressed_buffer.data() + 1, compressed_buffer[0]);
|
||||
auto codecs_bytes = CompressionCodecMultiple::getCodecsBytesFromData(compressed_buffer.data());
|
||||
Codecs codecs;
|
||||
for (auto byte : codecs_bytes)
|
||||
codecs.push_back(CompressionCodecFactory::instance().get(byte));
|
||||
|
||||
return std::make_shared<CompressionCodecMultiple>(codecs);
|
||||
}
|
||||
return CompressionCodecFactory::instance().get(method);
|
||||
}
|
||||
|
||||
}
|
15
src/Compression/getCompressionCodecForFile.h
Normal file
15
src/Compression/getCompressionCodecForFile.h
Normal file
@ -0,0 +1,15 @@
|
||||
#pragma once
|
||||
|
||||
#include <Compression/ICompressionCodec.h>
|
||||
#include <Disks/IDisk.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// Return compression codec with default parameters for file compressed in
|
||||
/// clickhouse fashion (with checksums, headers for each block, etc). This
|
||||
/// method should be used as fallback when we cannot deduce compression codec
|
||||
/// from metadata.
|
||||
CompressionCodecPtr getCompressionCodecForFile(const DiskPtr & disk, const String & relative_path);
|
||||
|
||||
}
|
@ -27,6 +27,7 @@ SRCS(
|
||||
CompressionCodecT64.cpp
|
||||
CompressionCodecZSTD.cpp
|
||||
CompressionFactory.cpp
|
||||
getCompressionCodecForFile.cpp
|
||||
ICompressionCodec.cpp
|
||||
LZ4_decompress_faster.cpp
|
||||
|
||||
|
@ -52,7 +52,7 @@ template <typename TInt, typename TUInt>
|
||||
constexpr bool is_le_int_vs_uint = is_any_int_vs_uint<TInt, TUInt> && (sizeof(TInt) <= sizeof(TUInt));
|
||||
|
||||
static_assert(is_le_int_vs_uint<Int128, DB::UInt128>);
|
||||
static_assert(is_le_int_vs_uint<Int128, bUInt256>);
|
||||
static_assert(is_le_int_vs_uint<Int128, DB::UInt256>);
|
||||
|
||||
template <typename TInt, typename TUInt>
|
||||
using bool_if_le_int_vs_uint_t = std::enable_if_t<is_le_int_vs_uint<TInt, TUInt>, bool>;
|
||||
@ -92,39 +92,27 @@ using bool_if_gt_int_vs_uint = std::enable_if_t<is_gt_int_vs_uint<TInt, TUInt>,
|
||||
template <typename TInt, typename TUInt>
|
||||
inline bool_if_gt_int_vs_uint<TInt, TUInt> greaterOpTmpl(TInt a, TUInt b)
|
||||
{
|
||||
if constexpr (is_big_int_v<TInt> && std::is_same_v<TUInt, UInt8>)
|
||||
return static_cast<TInt>(a) > static_cast<TInt>(static_cast<UInt16>(b));
|
||||
else
|
||||
return static_cast<TInt>(a) > static_cast<TInt>(b);
|
||||
return bigint_cast<TInt>(a) > bigint_cast<TInt>(b);
|
||||
}
|
||||
|
||||
template <typename TInt, typename TUInt>
|
||||
inline bool_if_gt_int_vs_uint<TInt, TUInt> greaterOpTmpl(TUInt a, TInt b)
|
||||
{
|
||||
if constexpr (is_big_int_v<TInt> && std::is_same_v<TUInt, UInt8>)
|
||||
return static_cast<TInt>(static_cast<UInt16>(a)) > static_cast<TInt>(b);
|
||||
else if constexpr (is_big_int_v<TInt> && std::is_same_v<TUInt, DB::UInt128>)
|
||||
return static_cast<bUInt256>(a) > b;
|
||||
else
|
||||
return static_cast<TInt>(a) > b;
|
||||
using CastA = std::conditional_t<is_big_int_v<TInt> && std::is_same_v<TUInt, DB::UInt128>, DB::UInt256, TInt>;
|
||||
|
||||
return bigint_cast<CastA>(a) > b;
|
||||
}
|
||||
|
||||
template <typename TInt, typename TUInt>
|
||||
inline bool_if_gt_int_vs_uint<TInt, TUInt> equalsOpTmpl(TInt a, TUInt b)
|
||||
{
|
||||
if constexpr (is_big_int_v<TInt> && std::is_same_v<TUInt, UInt8>)
|
||||
return static_cast<TInt>(a) == static_cast<TInt>(static_cast<UInt16>(b));
|
||||
else
|
||||
return static_cast<TInt>(a) == static_cast<TInt>(b);
|
||||
return bigint_cast<TInt>(a) == bigint_cast<TInt>(b);
|
||||
}
|
||||
|
||||
template <typename TInt, typename TUInt>
|
||||
inline bool_if_gt_int_vs_uint<TInt, TUInt> equalsOpTmpl(TUInt a, TInt b)
|
||||
{
|
||||
if constexpr (is_big_int_v<TInt> && std::is_same_v<TUInt, UInt8>)
|
||||
return static_cast<TInt>(static_cast<UInt16>(a)) == static_cast<TInt>(b);
|
||||
else
|
||||
return static_cast<TInt>(a) == static_cast<TInt>(b);
|
||||
return bigint_cast<TInt>(a) == bigint_cast<TInt>(b);
|
||||
}
|
||||
|
||||
|
||||
@ -200,16 +188,14 @@ inline bool_if_not_safe_conversion<A, B> greaterOp(A a, B b)
|
||||
template <typename A, typename B>
|
||||
inline bool_if_safe_conversion<A, B> greaterOp(A a, B b)
|
||||
{
|
||||
if constexpr (is_big_int_v<A> && std::is_same_v<B, UInt8>)
|
||||
return a > static_cast<UInt16>(b);
|
||||
else if constexpr (is_big_int_v<B> && std::is_same_v<A, UInt8>)
|
||||
return static_cast<UInt16>(a) > b;
|
||||
else if constexpr (std::is_same_v<A, DB::UInt128> && is_big_int_v<B>)
|
||||
// hack for UInt128 and bUInt128
|
||||
return static_cast<B>(a) > b;
|
||||
else if constexpr (std::is_same_v<B, DB::UInt128> && is_big_int_v<A>)
|
||||
// hack for UInt128 and bUInt128
|
||||
return a > static_cast<A>(b);
|
||||
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
|
||||
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
|
||||
|
||||
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
|
||||
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
|
||||
|
||||
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
|
||||
return bigint_cast<CastA>(a) > bigint_cast<CastB>(b);
|
||||
else
|
||||
return a > b;
|
||||
}
|
||||
@ -317,17 +303,9 @@ inline bool_if_not_safe_conversion<A, B> equalsOp(A a, B b)
|
||||
template <typename A, typename B>
|
||||
inline bool_if_safe_conversion<A, B> equalsOp(A a, B b)
|
||||
{
|
||||
using LargestType = std::conditional_t<sizeof(A) >= sizeof(B), A, B>;
|
||||
using LargestType = std::conditional_t<(sizeof(A) > sizeof(B)) || ((sizeof(A) == sizeof(B)) && !std::is_same_v<A, DB::UInt128>), A, B>;
|
||||
|
||||
if constexpr (is_big_int_v<LargestType> && std::is_same_v<A, UInt8>)
|
||||
return static_cast<LargestType>(static_cast<UInt16>(a)) == static_cast<LargestType>(b);
|
||||
else if constexpr (is_big_int_v<LargestType> && std::is_same_v<B, UInt8>)
|
||||
return static_cast<LargestType>(a) == static_cast<LargestType>(static_cast<UInt16>(b));
|
||||
else if constexpr (std::is_same_v<LargestType, DB::UInt128> && is_big_int_v<B>)
|
||||
// hack for UInt128 and bUInt128
|
||||
return static_cast<B>(a) == static_cast<B>(b);
|
||||
else
|
||||
return static_cast<LargestType>(a) == static_cast<LargestType>(b);
|
||||
return bigint_cast<LargestType>(a) == bigint_cast<LargestType>(b);
|
||||
}
|
||||
|
||||
template <>
|
||||
@ -443,16 +421,14 @@ inline bool_if_not_safe_conversion<A, B> notEqualsOp(A a, B b)
|
||||
template <typename A, typename B>
|
||||
inline bool_if_safe_conversion<A, B> notEqualsOp(A a, B b)
|
||||
{
|
||||
if constexpr (std::is_same_v<A, UInt8> && is_big_int_v<B>)
|
||||
return static_cast<UInt16>(a) != b;
|
||||
else if constexpr (std::is_same_v<B, UInt8> && is_big_int_v<A>)
|
||||
return a != static_cast<UInt16>(b);
|
||||
else if constexpr (std::is_same_v<A, DB::UInt128> && is_big_int_v<B>)
|
||||
// hack for UInt128 and bUInt128
|
||||
return static_cast<B>(a) != b;
|
||||
else if constexpr (std::is_same_v<B, DB::UInt128> && is_big_int_v<A>)
|
||||
// hack for UInt128 and bUInt128
|
||||
return a != static_cast<A>(b);
|
||||
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
|
||||
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
|
||||
|
||||
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
|
||||
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
|
||||
|
||||
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
|
||||
return bigint_cast<CastA>(a) != bigint_cast<CastB>(b);
|
||||
else
|
||||
return a != b;
|
||||
}
|
||||
@ -467,16 +443,14 @@ inline bool_if_not_safe_conversion<A, B> lessOp(A a, B b)
|
||||
template <typename A, typename B>
|
||||
inline bool_if_safe_conversion<A, B> lessOp(A a, B b)
|
||||
{
|
||||
if constexpr (std::is_same_v<A, UInt8> && is_big_int_v<B>)
|
||||
return static_cast<UInt16>(a) < b;
|
||||
else if constexpr (std::is_same_v<B, UInt8> && is_big_int_v<A>)
|
||||
return a < static_cast<UInt16>(b);
|
||||
else if constexpr (std::is_same_v<A, DB::UInt128> && is_big_int_v<B>)
|
||||
// hack for UInt128 and bUInt128
|
||||
return static_cast<B>(a) < b;
|
||||
else if constexpr (std::is_same_v<B, DB::UInt128> && is_big_int_v<A>)
|
||||
// hack for UInt128 and bUInt128
|
||||
return a < static_cast<A>(b);
|
||||
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
|
||||
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
|
||||
|
||||
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
|
||||
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
|
||||
|
||||
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
|
||||
return bigint_cast<CastA>(a) < bigint_cast<CastB>(b);
|
||||
else
|
||||
return a < b;
|
||||
}
|
||||
@ -493,16 +467,14 @@ inline bool_if_not_safe_conversion<A, B> lessOrEqualsOp(A a, B b)
|
||||
template <typename A, typename B>
|
||||
inline bool_if_safe_conversion<A, B> lessOrEqualsOp(A a, B b)
|
||||
{
|
||||
if constexpr (std::is_same_v<A, UInt8> && is_big_int_v<B>)
|
||||
return static_cast<UInt16>(a) <= b;
|
||||
else if constexpr (std::is_same_v<B, UInt8> && is_big_int_v<A>)
|
||||
return a <= static_cast<UInt16>(b);
|
||||
else if constexpr (std::is_same_v<A, DB::UInt128> && is_big_int_v<B>)
|
||||
// hack for UInt128 and bUInt128
|
||||
return static_cast<B>(a) <= b;
|
||||
else if constexpr (std::is_same_v<B, DB::UInt128> && is_big_int_v<A>)
|
||||
// hack for UInt128 and bUInt128
|
||||
return a <= static_cast<A>(b);
|
||||
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
|
||||
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
|
||||
|
||||
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
|
||||
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
|
||||
|
||||
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
|
||||
return bigint_cast<CastA>(a) <= bigint_cast<CastB>(b);
|
||||
else
|
||||
return a <= b;
|
||||
}
|
||||
@ -519,16 +491,14 @@ inline bool_if_not_safe_conversion<A, B> greaterOrEqualsOp(A a, B b)
|
||||
template <typename A, typename B>
|
||||
inline bool_if_safe_conversion<A, B> greaterOrEqualsOp(A a, B b)
|
||||
{
|
||||
if constexpr (std::is_same_v<A, UInt8> && is_big_int_v<B>)
|
||||
return static_cast<UInt16>(a) >= b;
|
||||
else if constexpr (std::is_same_v<B, UInt8> && is_big_int_v<A>)
|
||||
return a >= static_cast<UInt16>(b);
|
||||
else if constexpr (std::is_same_v<A, DB::UInt128> && is_big_int_v<B>)
|
||||
// hack for UInt128 and bUInt128
|
||||
return static_cast<B>(a) >= b;
|
||||
else if constexpr (std::is_same_v<B, DB::UInt128> && is_big_int_v<A>)
|
||||
// hack for UInt128 and bUInt128
|
||||
return a >= static_cast<A>(b);
|
||||
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
|
||||
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
|
||||
|
||||
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
|
||||
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
|
||||
|
||||
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
|
||||
return bigint_cast<CastA>(a) >= bigint_cast<CastB>(b);
|
||||
else
|
||||
return a >= b;
|
||||
}
|
||||
|
@ -37,7 +37,7 @@ inline bool allowDecimalComparison(const DataTypePtr & left_type, const DataType
|
||||
template <size_t > struct ConstructDecInt { using Type = Int32; };
|
||||
template <> struct ConstructDecInt<8> { using Type = Int64; };
|
||||
template <> struct ConstructDecInt<16> { using Type = Int128; };
|
||||
template <> struct ConstructDecInt<48> { using Type = bInt256; };
|
||||
template <> struct ConstructDecInt<48> { using Type = Int256; };
|
||||
|
||||
template <typename T, typename U>
|
||||
struct DecCompareInt
|
||||
@ -228,18 +228,14 @@ private:
|
||||
CompareInt x;
|
||||
if constexpr (is_big_int_v<CompareInt> && IsDecimalNumber<A>)
|
||||
x = a.value;
|
||||
else if constexpr (is_big_int_v<CompareInt> && std::is_same_v<A, UInt8>)
|
||||
x = static_cast<UInt16>(a);
|
||||
else
|
||||
x = static_cast<CompareInt>(a);
|
||||
x = bigint_cast<CompareInt>(a);
|
||||
|
||||
CompareInt y;
|
||||
if constexpr (is_big_int_v<CompareInt> && IsDecimalNumber<B>)
|
||||
y = b.value;
|
||||
else if constexpr (is_big_int_v<CompareInt> && std::is_same_v<B, UInt8>)
|
||||
y = static_cast<UInt16>(b);
|
||||
else
|
||||
y = static_cast<CompareInt>(b);
|
||||
y = bigint_cast<CompareInt>(b);
|
||||
|
||||
if constexpr (_check_overflow)
|
||||
{
|
||||
|
@ -36,7 +36,7 @@ inline auto scaleMultiplier(UInt32 scale)
|
||||
return common::exp10_i64(scale);
|
||||
else if constexpr (std::is_same_v<T, Int128> || std::is_same_v<T, Decimal128>)
|
||||
return common::exp10_i128(scale);
|
||||
else if constexpr (std::is_same_v<T, bInt256> || std::is_same_v<T, Decimal256>)
|
||||
else if constexpr (std::is_same_v<T, Int256> || std::is_same_v<T, Decimal256>)
|
||||
return common::exp10_i256(scale);
|
||||
}
|
||||
|
||||
|
@ -218,16 +218,16 @@ void readBinary(Tuple & x, ReadBuffer & buf)
|
||||
x.push_back(value);
|
||||
break;
|
||||
}
|
||||
case Field::Types::bUInt256:
|
||||
case Field::Types::UInt256:
|
||||
{
|
||||
bUInt256 value;
|
||||
UInt256 value;
|
||||
DB::readBinary(value, buf);
|
||||
x.push_back(value);
|
||||
break;
|
||||
}
|
||||
case Field::Types::bInt256:
|
||||
case Field::Types::Int256:
|
||||
{
|
||||
bInt256 value;
|
||||
Int256 value;
|
||||
DB::readBinary(value, buf);
|
||||
x.push_back(value);
|
||||
break;
|
||||
@ -301,14 +301,14 @@ void writeBinary(const Tuple & x, WriteBuffer & buf)
|
||||
DB::writeStringBinary(get<std::string>(elem), buf);
|
||||
break;
|
||||
}
|
||||
case Field::Types::bUInt256:
|
||||
case Field::Types::UInt256:
|
||||
{
|
||||
DB::writeBinary(get<bUInt256>(elem), buf);
|
||||
DB::writeBinary(get<UInt256>(elem), buf);
|
||||
break;
|
||||
}
|
||||
case Field::Types::bInt256:
|
||||
case Field::Types::Int256:
|
||||
{
|
||||
DB::writeBinary(get<bInt256>(elem), buf);
|
||||
DB::writeBinary(get<Int256>(elem), buf);
|
||||
break;
|
||||
}
|
||||
case Field::Types::Array:
|
||||
@ -411,14 +411,14 @@ Field Field::restoreFromDump(const std::string_view & dump_)
|
||||
prefix = std::string_view{"Int256_"};
|
||||
if (dump.starts_with(prefix))
|
||||
{
|
||||
bInt256 value = parseFromString<bInt256>(dump.substr(prefix.length()));
|
||||
Int256 value = parseFromString<Int256>(dump.substr(prefix.length()));
|
||||
return value;
|
||||
}
|
||||
|
||||
prefix = std::string_view{"UInt256_"};
|
||||
if (dump.starts_with(prefix))
|
||||
{
|
||||
bUInt256 value = parseFromString<bUInt256>(dump.substr(prefix.length()));
|
||||
UInt256 value = parseFromString<UInt256>(dump.substr(prefix.length()));
|
||||
return value;
|
||||
}
|
||||
|
||||
|
@ -185,8 +185,8 @@ template <> struct NearestFieldTypeImpl<long long> { using Type = Int64; };
|
||||
template <> struct NearestFieldTypeImpl<unsigned long> { using Type = UInt64; };
|
||||
template <> struct NearestFieldTypeImpl<unsigned long long> { using Type = UInt64; };
|
||||
|
||||
template <> struct NearestFieldTypeImpl<bUInt256> { using Type = bUInt256; };
|
||||
template <> struct NearestFieldTypeImpl<bInt256> { using Type = bInt256; };
|
||||
template <> struct NearestFieldTypeImpl<UInt256> { using Type = UInt256; };
|
||||
template <> struct NearestFieldTypeImpl<Int256> { using Type = Int256; };
|
||||
|
||||
template <> struct NearestFieldTypeImpl<Int128> { using Type = Int128; };
|
||||
template <> struct NearestFieldTypeImpl<Decimal32> { using Type = DecimalField<Decimal32>; };
|
||||
@ -255,8 +255,8 @@ public:
|
||||
Decimal128 = 21,
|
||||
AggregateFunctionState = 22,
|
||||
Decimal256 = 23,
|
||||
bUInt256 = 24,
|
||||
bInt256 = 25,
|
||||
UInt256 = 24,
|
||||
Int256 = 25,
|
||||
};
|
||||
|
||||
static const int MIN_NON_POD = 16;
|
||||
@ -279,8 +279,8 @@ public:
|
||||
case Decimal128: return "Decimal128";
|
||||
case Decimal256: return "Decimal256";
|
||||
case AggregateFunctionState: return "AggregateFunctionState";
|
||||
case bUInt256: return "UInt256";
|
||||
case bInt256: return "Int256";
|
||||
case UInt256: return "UInt256";
|
||||
case Int256: return "Int256";
|
||||
}
|
||||
|
||||
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
|
||||
@ -467,8 +467,8 @@ public:
|
||||
case Types::Decimal128: return get<DecimalField<Decimal128>>() < rhs.get<DecimalField<Decimal128>>();
|
||||
case Types::Decimal256: return get<DecimalField<Decimal256>>() < rhs.get<DecimalField<Decimal256>>();
|
||||
case Types::AggregateFunctionState: return get<AggregateFunctionStateData>() < rhs.get<AggregateFunctionStateData>();
|
||||
case Types::bUInt256: return get<bUInt256>() < rhs.get<bUInt256>();
|
||||
case Types::bInt256: return get<bInt256>() < rhs.get<bInt256>();
|
||||
case Types::UInt256: return get<UInt256>() < rhs.get<UInt256>();
|
||||
case Types::Int256: return get<Int256>() < rhs.get<Int256>();
|
||||
}
|
||||
|
||||
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
|
||||
@ -502,8 +502,8 @@ public:
|
||||
case Types::Decimal128: return get<DecimalField<Decimal128>>() <= rhs.get<DecimalField<Decimal128>>();
|
||||
case Types::Decimal256: return get<DecimalField<Decimal256>>() <= rhs.get<DecimalField<Decimal256>>();
|
||||
case Types::AggregateFunctionState: return get<AggregateFunctionStateData>() <= rhs.get<AggregateFunctionStateData>();
|
||||
case Types::bUInt256: return get<bUInt256>() <= rhs.get<bUInt256>();
|
||||
case Types::bInt256: return get<bInt256>() <= rhs.get<bInt256>();
|
||||
case Types::UInt256: return get<UInt256>() <= rhs.get<UInt256>();
|
||||
case Types::Int256: return get<Int256>() <= rhs.get<Int256>();
|
||||
}
|
||||
|
||||
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
|
||||
@ -541,8 +541,8 @@ public:
|
||||
case Types::Decimal128: return get<DecimalField<Decimal128>>() == rhs.get<DecimalField<Decimal128>>();
|
||||
case Types::Decimal256: return get<DecimalField<Decimal256>>() == rhs.get<DecimalField<Decimal256>>();
|
||||
case Types::AggregateFunctionState: return get<AggregateFunctionStateData>() == rhs.get<AggregateFunctionStateData>();
|
||||
case Types::bUInt256: return get<bUInt256>() == rhs.get<bUInt256>();
|
||||
case Types::bInt256: return get<bInt256>() == rhs.get<bInt256>();
|
||||
case Types::UInt256: return get<UInt256>() == rhs.get<UInt256>();
|
||||
case Types::Int256: return get<Int256>() == rhs.get<Int256>();
|
||||
}
|
||||
|
||||
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
|
||||
@ -579,8 +579,8 @@ public:
|
||||
case Types::Decimal256: return f(field.template get<DecimalField<Decimal256>>());
|
||||
case Types::AggregateFunctionState: return f(field.template get<AggregateFunctionStateData>());
|
||||
case Types::Int128: return f(field.template get<Int128>());
|
||||
case Types::bUInt256: return f(field.template get<bUInt256>());
|
||||
case Types::bInt256: return f(field.template get<bInt256>());
|
||||
case Types::UInt256: return f(field.template get<UInt256>());
|
||||
case Types::Int256: return f(field.template get<Int256>());
|
||||
#if !__clang__
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
@ -601,7 +601,7 @@ private:
|
||||
Null, UInt64, UInt128, Int64, Int128, Float64, String, Array, Tuple,
|
||||
DecimalField<Decimal32>, DecimalField<Decimal64>, DecimalField<Decimal128>, DecimalField<Decimal256>,
|
||||
AggregateFunctionStateData,
|
||||
bUInt256, bInt256
|
||||
UInt256, Int256
|
||||
> storage;
|
||||
|
||||
Types::Which which;
|
||||
@ -732,8 +732,8 @@ template <> struct Field::TypeToEnum<DecimalField<Decimal64>>{ static const Type
|
||||
template <> struct Field::TypeToEnum<DecimalField<Decimal128>>{ static const Types::Which value = Types::Decimal128; };
|
||||
template <> struct Field::TypeToEnum<DecimalField<Decimal256>>{ static const Types::Which value = Types::Decimal256; };
|
||||
template <> struct Field::TypeToEnum<AggregateFunctionStateData>{ static const Types::Which value = Types::AggregateFunctionState; };
|
||||
template <> struct Field::TypeToEnum<bUInt256> { static const Types::Which value = Types::bUInt256; };
|
||||
template <> struct Field::TypeToEnum<bInt256> { static const Types::Which value = Types::bInt256; };
|
||||
template <> struct Field::TypeToEnum<UInt256> { static const Types::Which value = Types::UInt256; };
|
||||
template <> struct Field::TypeToEnum<Int256> { static const Types::Which value = Types::Int256; };
|
||||
|
||||
template <> struct Field::EnumToType<Field::Types::Null> { using Type = Null; };
|
||||
template <> struct Field::EnumToType<Field::Types::UInt64> { using Type = UInt64; };
|
||||
@ -749,8 +749,8 @@ template <> struct Field::EnumToType<Field::Types::Decimal64> { using Type = Dec
|
||||
template <> struct Field::EnumToType<Field::Types::Decimal128> { using Type = DecimalField<Decimal128>; };
|
||||
template <> struct Field::EnumToType<Field::Types::Decimal256> { using Type = DecimalField<Decimal256>; };
|
||||
template <> struct Field::EnumToType<Field::Types::AggregateFunctionState> { using Type = DecimalField<AggregateFunctionStateData>; };
|
||||
template <> struct Field::EnumToType<Field::Types::bUInt256> { using Type = bUInt256; };
|
||||
template <> struct Field::EnumToType<Field::Types::bInt256> { using Type = bInt256; };
|
||||
template <> struct Field::EnumToType<Field::Types::UInt256> { using Type = UInt256; };
|
||||
template <> struct Field::EnumToType<Field::Types::Int256> { using Type = Int256; };
|
||||
|
||||
inline constexpr bool isInt64FieldType(Field::Types::Which t)
|
||||
{
|
||||
|
@ -8,7 +8,7 @@ namespace DB
|
||||
{
|
||||
|
||||
using TypeListNativeNumbers = TypeList<UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64>;
|
||||
using TypeListExtendedNumbers = TypeList<Int128, bUInt256, bInt256>;
|
||||
using TypeListExtendedNumbers = TypeList<Int128, UInt256, Int256>;
|
||||
using TypeListDecimalNumbers = TypeList<Decimal32, Decimal64, Decimal128, Decimal256>;
|
||||
|
||||
using TypeListGeneralNumbers = typename TypeListConcat<TypeListNativeNumbers, TypeListExtendedNumbers>::Type;
|
||||
|
@ -22,13 +22,13 @@ enum class TypeIndex
|
||||
UInt32,
|
||||
UInt64,
|
||||
UInt128,
|
||||
bUInt256,
|
||||
UInt256,
|
||||
Int8,
|
||||
Int16,
|
||||
Int32,
|
||||
Int64,
|
||||
Int128,
|
||||
bInt256,
|
||||
Int256,
|
||||
Float32,
|
||||
Float64,
|
||||
Date,
|
||||
@ -58,14 +58,14 @@ using UInt8 = ::UInt8;
|
||||
using UInt16 = ::UInt16;
|
||||
using UInt32 = ::UInt32;
|
||||
using UInt64 = ::UInt64;
|
||||
using bUInt256 = ::bUInt256;
|
||||
using UInt256 = ::bUInt256;
|
||||
|
||||
using Int8 = ::Int8;
|
||||
using Int16 = ::Int16;
|
||||
using Int32 = ::Int32;
|
||||
using Int64 = ::Int64;
|
||||
using Int128 = ::Int128;
|
||||
using bInt256 = ::bInt256;
|
||||
using Int256 = ::bInt256;
|
||||
|
||||
using Float32 = float;
|
||||
using Float64 = double;
|
||||
@ -81,13 +81,13 @@ template <> inline constexpr bool IsNumber<UInt8> = true;
|
||||
template <> inline constexpr bool IsNumber<UInt16> = true;
|
||||
template <> inline constexpr bool IsNumber<UInt32> = true;
|
||||
template <> inline constexpr bool IsNumber<UInt64> = true;
|
||||
template <> inline constexpr bool IsNumber<bUInt256> = true;
|
||||
template <> inline constexpr bool IsNumber<UInt256> = true;
|
||||
template <> inline constexpr bool IsNumber<Int8> = true;
|
||||
template <> inline constexpr bool IsNumber<Int16> = true;
|
||||
template <> inline constexpr bool IsNumber<Int32> = true;
|
||||
template <> inline constexpr bool IsNumber<Int64> = true;
|
||||
template <> inline constexpr bool IsNumber<Int128> = true;
|
||||
template <> inline constexpr bool IsNumber<bInt256> = true;
|
||||
template <> inline constexpr bool IsNumber<Int256> = true;
|
||||
template <> inline constexpr bool IsNumber<Float32> = true;
|
||||
template <> inline constexpr bool IsNumber<Float64> = true;
|
||||
|
||||
@ -97,13 +97,13 @@ template <> struct TypeName<UInt8> { static constexpr const char * get() { ret
|
||||
template <> struct TypeName<UInt16> { static constexpr const char * get() { return "UInt16"; } };
|
||||
template <> struct TypeName<UInt32> { static constexpr const char * get() { return "UInt32"; } };
|
||||
template <> struct TypeName<UInt64> { static constexpr const char * get() { return "UInt64"; } };
|
||||
template <> struct TypeName<bUInt256> { static constexpr const char * get() { return "UInt256"; } };
|
||||
template <> struct TypeName<UInt256> { static constexpr const char * get() { return "UInt256"; } };
|
||||
template <> struct TypeName<Int8> { static constexpr const char * get() { return "Int8"; } };
|
||||
template <> struct TypeName<Int16> { static constexpr const char * get() { return "Int16"; } };
|
||||
template <> struct TypeName<Int32> { static constexpr const char * get() { return "Int32"; } };
|
||||
template <> struct TypeName<Int64> { static constexpr const char * get() { return "Int64"; } };
|
||||
template <> struct TypeName<Int128> { static constexpr const char * get() { return "Int128"; } };
|
||||
template <> struct TypeName<bInt256> { static constexpr const char * get() { return "Int256"; } };
|
||||
template <> struct TypeName<Int256> { static constexpr const char * get() { return "Int256"; } };
|
||||
template <> struct TypeName<Float32> { static constexpr const char * get() { return "Float32"; } };
|
||||
template <> struct TypeName<Float64> { static constexpr const char * get() { return "Float64"; } };
|
||||
template <> struct TypeName<String> { static constexpr const char * get() { return "String"; } };
|
||||
@ -113,13 +113,13 @@ template <> struct TypeId<UInt8> { static constexpr const TypeIndex value = T
|
||||
template <> struct TypeId<UInt16> { static constexpr const TypeIndex value = TypeIndex::UInt16; };
|
||||
template <> struct TypeId<UInt32> { static constexpr const TypeIndex value = TypeIndex::UInt32; };
|
||||
template <> struct TypeId<UInt64> { static constexpr const TypeIndex value = TypeIndex::UInt64; };
|
||||
template <> struct TypeId<bUInt256> { static constexpr const TypeIndex value = TypeIndex::bUInt256; };
|
||||
template <> struct TypeId<UInt256> { static constexpr const TypeIndex value = TypeIndex::UInt256; };
|
||||
template <> struct TypeId<Int8> { static constexpr const TypeIndex value = TypeIndex::Int8; };
|
||||
template <> struct TypeId<Int16> { static constexpr const TypeIndex value = TypeIndex::Int16; };
|
||||
template <> struct TypeId<Int32> { static constexpr const TypeIndex value = TypeIndex::Int32; };
|
||||
template <> struct TypeId<Int64> { static constexpr const TypeIndex value = TypeIndex::Int64; };
|
||||
template <> struct TypeId<Int128> { static constexpr const TypeIndex value = TypeIndex::Int128; };
|
||||
template <> struct TypeId<bInt256> { static constexpr const TypeIndex value = TypeIndex::bInt256; };
|
||||
template <> struct TypeId<Int256> { static constexpr const TypeIndex value = TypeIndex::Int256; };
|
||||
template <> struct TypeId<Float32> { static constexpr const TypeIndex value = TypeIndex::Float32; };
|
||||
template <> struct TypeId<Float64> { static constexpr const TypeIndex value = TypeIndex::Float64; };
|
||||
|
||||
@ -158,16 +158,12 @@ struct Decimal
|
||||
if constexpr (std::is_same_v<U, Decimal<Int32>> ||
|
||||
std::is_same_v<U, Decimal<Int64>> ||
|
||||
std::is_same_v<U, Decimal<Int128>> ||
|
||||
std::is_same_v<U, Decimal<bInt256>>)
|
||||
std::is_same_v<U, Decimal<Int256>>)
|
||||
{
|
||||
return convertTo<typename U::NativeType>();
|
||||
}
|
||||
else if constexpr (is_big_int_v<NativeType>)
|
||||
{
|
||||
return value. template convert_to<U>();
|
||||
}
|
||||
else
|
||||
return static_cast<U>(value);
|
||||
return bigint_cast<U>(value);
|
||||
}
|
||||
|
||||
const Decimal<T> & operator += (const T & x) { value += x; return *this; }
|
||||
@ -193,7 +189,7 @@ template <typename T> inline Decimal<T> operator- (const Decimal<T> & x) { retur
|
||||
using Decimal32 = Decimal<Int32>;
|
||||
using Decimal64 = Decimal<Int64>;
|
||||
using Decimal128 = Decimal<Int128>;
|
||||
using Decimal256 = Decimal<bInt256>;
|
||||
using Decimal256 = Decimal<Int256>;
|
||||
|
||||
using DateTime64 = Decimal64;
|
||||
|
||||
@ -217,11 +213,11 @@ template <typename T> struct NativeType { using Type = T; };
|
||||
template <> struct NativeType<Decimal32> { using Type = Int32; };
|
||||
template <> struct NativeType<Decimal64> { using Type = Int64; };
|
||||
template <> struct NativeType<Decimal128> { using Type = Int128; };
|
||||
template <> struct NativeType<Decimal256> { using Type = bInt256; };
|
||||
template <> struct NativeType<Decimal256> { using Type = Int256; };
|
||||
|
||||
template <typename T> constexpr bool OverBigInt = false;
|
||||
template <> inline constexpr bool OverBigInt<bInt256> = true;
|
||||
template <> inline constexpr bool OverBigInt<bUInt256> = true;
|
||||
template <> inline constexpr bool OverBigInt<Int256> = true;
|
||||
template <> inline constexpr bool OverBigInt<UInt256> = true;
|
||||
template <> inline constexpr bool OverBigInt<Decimal256> = true;
|
||||
|
||||
inline constexpr const char * getTypeName(TypeIndex idx)
|
||||
@ -234,13 +230,13 @@ inline constexpr const char * getTypeName(TypeIndex idx)
|
||||
case TypeIndex::UInt32: return TypeName<UInt32>::get();
|
||||
case TypeIndex::UInt64: return TypeName<UInt64>::get();
|
||||
case TypeIndex::UInt128: return "UInt128";
|
||||
case TypeIndex::bUInt256: return TypeName<bUInt256>::get();
|
||||
case TypeIndex::UInt256: return TypeName<UInt256>::get();
|
||||
case TypeIndex::Int8: return TypeName<Int8>::get();
|
||||
case TypeIndex::Int16: return TypeName<Int16>::get();
|
||||
case TypeIndex::Int32: return TypeName<Int32>::get();
|
||||
case TypeIndex::Int64: return TypeName<Int64>::get();
|
||||
case TypeIndex::Int128: return TypeName<Int128>::get();
|
||||
case TypeIndex::bInt256: return TypeName<bInt256>::get();
|
||||
case TypeIndex::Int256: return TypeName<Int256>::get();
|
||||
case TypeIndex::Float32: return TypeName<Float32>::get();
|
||||
case TypeIndex::Float64: return TypeName<Float64>::get();
|
||||
case TypeIndex::Date: return "Date";
|
||||
|
@ -26,14 +26,14 @@ bool callOnBasicType(TypeIndex number, F && f)
|
||||
case TypeIndex::UInt16: return f(TypePair<T, UInt16>());
|
||||
case TypeIndex::UInt32: return f(TypePair<T, UInt32>());
|
||||
case TypeIndex::UInt64: return f(TypePair<T, UInt64>());
|
||||
case TypeIndex::bUInt256: return f(TypePair<T, bUInt256>());
|
||||
case TypeIndex::UInt256: return f(TypePair<T, UInt256>());
|
||||
|
||||
case TypeIndex::Int8: return f(TypePair<T, Int8>());
|
||||
case TypeIndex::Int16: return f(TypePair<T, Int16>());
|
||||
case TypeIndex::Int32: return f(TypePair<T, Int32>());
|
||||
case TypeIndex::Int64: return f(TypePair<T, Int64>());
|
||||
case TypeIndex::Int128: return f(TypePair<T, Int128>());
|
||||
case TypeIndex::bInt256: return f(TypePair<T, bInt256>());
|
||||
case TypeIndex::Int256: return f(TypePair<T, Int256>());
|
||||
|
||||
case TypeIndex::Enum8: return f(TypePair<T, Int8>());
|
||||
case TypeIndex::Enum16: return f(TypePair<T, Int16>());
|
||||
@ -94,14 +94,14 @@ inline bool callOnBasicTypes(TypeIndex type_num1, TypeIndex type_num2, F && f)
|
||||
case TypeIndex::UInt16: return callOnBasicType<UInt16, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
|
||||
case TypeIndex::UInt32: return callOnBasicType<UInt32, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
|
||||
case TypeIndex::UInt64: return callOnBasicType<UInt64, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
|
||||
case TypeIndex::bUInt256: return callOnBasicType<bUInt256, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
|
||||
case TypeIndex::UInt256: return callOnBasicType<UInt256, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
|
||||
|
||||
case TypeIndex::Int8: return callOnBasicType<Int8, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
|
||||
case TypeIndex::Int16: return callOnBasicType<Int16, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
|
||||
case TypeIndex::Int32: return callOnBasicType<Int32, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
|
||||
case TypeIndex::Int64: return callOnBasicType<Int64, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
|
||||
case TypeIndex::Int128: return callOnBasicType<Int128, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
|
||||
case TypeIndex::bInt256: return callOnBasicType<bInt256, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
|
||||
case TypeIndex::Int256: return callOnBasicType<Int256, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
|
||||
|
||||
case TypeIndex::Enum8: return callOnBasicType<Int8, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
|
||||
case TypeIndex::Enum16: return callOnBasicType<Int16, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
|
||||
@ -171,14 +171,14 @@ bool callOnIndexAndDataType(TypeIndex number, F && f)
|
||||
case TypeIndex::UInt16: return f(TypePair<DataTypeNumber<UInt16>, T>());
|
||||
case TypeIndex::UInt32: return f(TypePair<DataTypeNumber<UInt32>, T>());
|
||||
case TypeIndex::UInt64: return f(TypePair<DataTypeNumber<UInt64>, T>());
|
||||
case TypeIndex::bUInt256: return f(TypePair<DataTypeNumber<bUInt256>, T>());
|
||||
case TypeIndex::UInt256: return f(TypePair<DataTypeNumber<UInt256>, T>());
|
||||
|
||||
case TypeIndex::Int8: return f(TypePair<DataTypeNumber<Int8>, T>());
|
||||
case TypeIndex::Int16: return f(TypePair<DataTypeNumber<Int16>, T>());
|
||||
case TypeIndex::Int32: return f(TypePair<DataTypeNumber<Int32>, T>());
|
||||
case TypeIndex::Int64: return f(TypePair<DataTypeNumber<Int64>, T>());
|
||||
case TypeIndex::Int128: return f(TypePair<DataTypeNumber<Int128>, T>());
|
||||
case TypeIndex::bInt256: return f(TypePair<DataTypeNumber<bInt256>, T>());
|
||||
case TypeIndex::Int256: return f(TypePair<DataTypeNumber<Int256>, T>());
|
||||
|
||||
case TypeIndex::Float32: return f(TypePair<DataTypeNumber<Float32>, T>());
|
||||
case TypeIndex::Float64: return f(TypePair<DataTypeNumber<Float64>, T>());
|
||||
|
@ -257,14 +257,14 @@ template class DataTypeNumberBase<UInt8>;
|
||||
template class DataTypeNumberBase<UInt16>;
|
||||
template class DataTypeNumberBase<UInt32>;
|
||||
template class DataTypeNumberBase<UInt64>;
|
||||
template class DataTypeNumberBase<UInt128>; // used only in UUID
|
||||
template class DataTypeNumberBase<bUInt256>;
|
||||
template class DataTypeNumberBase<UInt128>; // base for UUID
|
||||
template class DataTypeNumberBase<UInt256>;
|
||||
template class DataTypeNumberBase<Int8>;
|
||||
template class DataTypeNumberBase<Int16>;
|
||||
template class DataTypeNumberBase<Int32>;
|
||||
template class DataTypeNumberBase<Int64>;
|
||||
template class DataTypeNumberBase<Int128>;
|
||||
template class DataTypeNumberBase<bInt256>;
|
||||
template class DataTypeNumberBase<Int256>;
|
||||
template class DataTypeNumberBase<Float32>;
|
||||
template class DataTypeNumberBase<Float64>;
|
||||
|
||||
|
@ -171,7 +171,7 @@ convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
|
||||
else
|
||||
{
|
||||
if constexpr (is_big_int_v<FromFieldType>)
|
||||
return convertDecimals<DataTypeDecimal<Decimal256>, ToDataType>(static_cast<bInt256>(value), 0, scale);
|
||||
return convertDecimals<DataTypeDecimal<Decimal256>, ToDataType>(static_cast<Int256>(value), 0, scale);
|
||||
else if constexpr (std::is_same_v<FromFieldType, UInt64>)
|
||||
return convertDecimals<DataTypeDecimal<Decimal128>, ToDataType>(value, 0, scale);
|
||||
else
|
||||
|
@ -39,7 +39,7 @@ using DataTypeFloat32 = DataTypeNumber<Float32>;
|
||||
using DataTypeFloat64 = DataTypeNumber<Float64>;
|
||||
|
||||
using DataTypeInt128 = DataTypeNumber<Int128>;
|
||||
using DataTypeUInt256 = DataTypeNumber<bUInt256>;
|
||||
using DataTypeInt256 = DataTypeNumber<bInt256>;
|
||||
using DataTypeUInt256 = DataTypeNumber<UInt256>;
|
||||
using DataTypeInt256 = DataTypeNumber<Int256>;
|
||||
|
||||
}
|
||||
|
@ -124,12 +124,12 @@ DataTypePtr FieldToDataType::operator() (const AggregateFunctionStateData & x) c
|
||||
return DataTypeFactory::instance().get(name);
|
||||
}
|
||||
|
||||
DataTypePtr FieldToDataType::operator() (const bUInt256 &) const
|
||||
DataTypePtr FieldToDataType::operator() (const UInt256 &) const
|
||||
{
|
||||
throw Exception("There are no UInt256 literals in SQL", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
DataTypePtr FieldToDataType::operator() (const bInt256 &) const
|
||||
DataTypePtr FieldToDataType::operator() (const Int256 &) const
|
||||
{
|
||||
throw Exception("There are no Int256 literals in SQL", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
@ -31,8 +31,8 @@ public:
|
||||
DataTypePtr operator() (const DecimalField<Decimal128> & x) const;
|
||||
DataTypePtr operator() (const DecimalField<Decimal256> & x) const;
|
||||
DataTypePtr operator() (const AggregateFunctionStateData & x) const;
|
||||
DataTypePtr operator() (const bUInt256 & x) const;
|
||||
DataTypePtr operator() (const bInt256 & x) const;
|
||||
DataTypePtr operator() (const UInt256 & x) const;
|
||||
DataTypePtr operator() (const Int256 & x) const;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -482,8 +482,8 @@ struct WhichDataType
|
||||
bool isUInt32() const { return idx == TypeIndex::UInt32; }
|
||||
bool isUInt64() const { return idx == TypeIndex::UInt64; }
|
||||
bool isUInt128() const { return idx == TypeIndex::UInt128; }
|
||||
bool isbUInt256() const { return idx == TypeIndex::bUInt256; }
|
||||
bool isUInt() const { return isUInt8() || isUInt16() || isUInt32() || isUInt64() || isUInt128() || isbUInt256(); }
|
||||
bool isUInt256() const { return idx == TypeIndex::UInt256; }
|
||||
bool isUInt() const { return isUInt8() || isUInt16() || isUInt32() || isUInt64() || isUInt128() || isUInt256(); }
|
||||
bool isNativeUInt() const { return isUInt8() || isUInt16() || isUInt32() || isUInt64(); }
|
||||
|
||||
bool isInt8() const { return idx == TypeIndex::Int8; }
|
||||
@ -491,8 +491,8 @@ struct WhichDataType
|
||||
bool isInt32() const { return idx == TypeIndex::Int32; }
|
||||
bool isInt64() const { return idx == TypeIndex::Int64; }
|
||||
bool isInt128() const { return idx == TypeIndex::Int128; }
|
||||
bool isbInt256() const { return idx == TypeIndex::bInt256; }
|
||||
bool isInt() const { return isInt8() || isInt16() || isInt32() || isInt64() || isInt128() || isbInt256(); }
|
||||
bool isInt256() const { return idx == TypeIndex::Int256; }
|
||||
bool isInt() const { return isInt8() || isInt16() || isInt32() || isInt64() || isInt128() || isInt256(); }
|
||||
bool isNativeInt() const { return isInt8() || isInt16() || isInt32() || isInt64(); }
|
||||
|
||||
bool isDecimal32() const { return idx == TypeIndex::Decimal32; }
|
||||
@ -529,7 +529,7 @@ struct WhichDataType
|
||||
bool isFunction() const { return idx == TypeIndex::Function; }
|
||||
bool isAggregateFunction() const { return idx == TypeIndex::AggregateFunction; }
|
||||
|
||||
bool IsBigIntOrDeimal() const { return isInt128() || isbInt256() || isbUInt256() || isDecimal256(); }
|
||||
bool IsBigIntOrDeimal() const { return isInt128() || isInt256() || isUInt256() || isDecimal256(); }
|
||||
};
|
||||
|
||||
/// IDataType helpers (alternative for IDataType virtual methods with single point of truth)
|
||||
|
@ -55,9 +55,9 @@ template <> struct Construct<false, false, 1> { using Type = UInt8; };
|
||||
template <> struct Construct<false, false, 2> { using Type = UInt16; };
|
||||
template <> struct Construct<false, false, 4> { using Type = UInt32; };
|
||||
template <> struct Construct<false, false, 8> { using Type = UInt64; };
|
||||
template <> struct Construct<false, false, 16> { using Type = bUInt256; };
|
||||
template <> struct Construct<false, false, 32> { using Type = bUInt256; };
|
||||
template <> struct Construct<false, false, 48> { using Type = bUInt256; };
|
||||
template <> struct Construct<false, false, 16> { using Type = UInt256; };
|
||||
template <> struct Construct<false, false, 32> { using Type = UInt256; };
|
||||
template <> struct Construct<false, false, 48> { using Type = UInt256; };
|
||||
template <> struct Construct<false, true, 1> { using Type = Float32; };
|
||||
template <> struct Construct<false, true, 2> { using Type = Float32; };
|
||||
template <> struct Construct<false, true, 4> { using Type = Float32; };
|
||||
@ -68,7 +68,7 @@ template <> struct Construct<true, false, 4> { using Type = Int32; };
|
||||
template <> struct Construct<true, false, 8> { using Type = Int64; };
|
||||
template <> struct Construct<true, false, 16> { using Type = Int128; };
|
||||
template <> struct Construct<true, false, 32> { using Type = Int128; };
|
||||
template <> struct Construct<true, false, 48> { using Type = bInt256; };
|
||||
template <> struct Construct<true, false, 48> { using Type = Int256; };
|
||||
template <> struct Construct<true, true, 1> { using Type = Float32; };
|
||||
template <> struct Construct<true, true, 2> { using Type = Float32; };
|
||||
template <> struct Construct<true, true, 4> { using Type = Float32; };
|
||||
@ -228,10 +228,7 @@ using ResultOfGreatest = std::conditional_t<LeastGreatestSpecialCase<A, B>,
|
||||
template <typename T>
|
||||
static inline auto littleBits(const T & x)
|
||||
{
|
||||
if constexpr (is_big_int_v<T>)
|
||||
return x. template convert_to<UInt32>();
|
||||
else
|
||||
return UInt8(x);
|
||||
return bigint_cast<UInt8>(x);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -33,6 +33,11 @@ static constexpr size_t PRINT_MESSAGE_EACH_N_OBJECTS = 256;
|
||||
static constexpr size_t PRINT_MESSAGE_EACH_N_SECONDS = 5;
|
||||
static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
void tryAttachTable(
|
||||
@ -249,6 +254,9 @@ void DatabaseOrdinary::alterTable(const Context & context, const StorageID & tab
|
||||
|
||||
auto & ast_create_query = ast->as<ASTCreateQuery &>();
|
||||
|
||||
if (ast_create_query.as_table_function)
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot alter table {} because it was created AS table function", backQuote(table_name));
|
||||
|
||||
ASTPtr new_columns = InterpreterCreateQuery::formatColumns(metadata.columns);
|
||||
ASTPtr new_indices = InterpreterCreateQuery::formatIndices(metadata.secondary_indices);
|
||||
ASTPtr new_constraints = InterpreterCreateQuery::formatConstraints(metadata.constraints);
|
||||
|
@ -52,12 +52,14 @@ namespace DB
|
||||
const String & host_,
|
||||
UInt16 port_,
|
||||
UInt8 db_index_,
|
||||
const String & password_,
|
||||
RedisStorageType storage_type_,
|
||||
const Block & sample_block_)
|
||||
: dict_struct{dict_struct_}
|
||||
, host{host_}
|
||||
, port{port_}
|
||||
, db_index{db_index_}
|
||||
, password{password_}
|
||||
, storage_type{storage_type_}
|
||||
, sample_block{sample_block_}
|
||||
, client{std::make_shared<Poco::Redis::Client>(host, port)}
|
||||
@ -78,16 +80,22 @@ namespace DB
|
||||
ErrorCodes::INVALID_CONFIG_PARAMETER};
|
||||
// suppose key[0] is primary key, key[1] is secondary key
|
||||
}
|
||||
if (!password.empty())
|
||||
{
|
||||
RedisCommand command("AUTH");
|
||||
command << password;
|
||||
String reply = client->execute<String>(command);
|
||||
if (reply != "OK")
|
||||
throw Exception{"Authentication failed with reason "
|
||||
+ reply, ErrorCodes::INTERNAL_REDIS_ERROR};
|
||||
}
|
||||
|
||||
if (db_index != 0)
|
||||
{
|
||||
RedisCommand command("SELECT");
|
||||
// Use poco's Int64, because it is defined as long long, and on
|
||||
// MacOS, for the purposes of template instantiation, this type is
|
||||
// distinct from int64_t, which is our Int64.
|
||||
command << static_cast<Poco::Int64>(db_index);
|
||||
command << std::to_string(db_index);
|
||||
String reply = client->execute<String>(command);
|
||||
if (reply != "+OK\r\n")
|
||||
if (reply != "OK")
|
||||
throw Exception{"Selecting database with index " + DB::toString(db_index)
|
||||
+ " failed with reason " + reply, ErrorCodes::INTERNAL_REDIS_ERROR};
|
||||
}
|
||||
@ -104,6 +112,7 @@ namespace DB
|
||||
config_.getString(config_prefix_ + ".host"),
|
||||
config_.getUInt(config_prefix_ + ".port"),
|
||||
config_.getUInt(config_prefix_ + ".db_index", 0),
|
||||
config_.getString(config_prefix_ + ".password",""),
|
||||
parseStorageType(config_.getString(config_prefix_ + ".storage_type", "")),
|
||||
sample_block_)
|
||||
{
|
||||
@ -115,6 +124,7 @@ namespace DB
|
||||
other.host,
|
||||
other.port,
|
||||
other.db_index,
|
||||
other.password,
|
||||
other.storage_type,
|
||||
other.sample_block}
|
||||
{
|
||||
|
@ -41,6 +41,7 @@ namespace ErrorCodes
|
||||
const std::string & host,
|
||||
UInt16 port,
|
||||
UInt8 db_index,
|
||||
const std::string & password,
|
||||
RedisStorageType storage_type,
|
||||
const Block & sample_block);
|
||||
|
||||
@ -91,6 +92,7 @@ namespace ErrorCodes
|
||||
const std::string host;
|
||||
const UInt16 port;
|
||||
const UInt8 db_index;
|
||||
const std::string password;
|
||||
const RedisStorageType storage_type;
|
||||
Block sample_block;
|
||||
|
||||
|
@ -456,8 +456,8 @@ public:
|
||||
}
|
||||
|
||||
bool readInt128(Int128 &) override { cannotConvertType("Int128"); }
|
||||
bool readbInt256(bInt256 &) override { cannotConvertType("Int256"); }
|
||||
bool readbUInt256(bUInt256 &) override { cannotConvertType("UInt256"); }
|
||||
bool readInt256(Int256 &) override { cannotConvertType("Int256"); }
|
||||
bool readUInt256(UInt256 &) override { cannotConvertType("UInt256"); }
|
||||
|
||||
bool readFloat32(Float32 &) override
|
||||
{
|
||||
|
@ -62,8 +62,8 @@ public:
|
||||
bool readNumber(UInt64 & value) { return current_converter->readUInt64(value); }
|
||||
bool readNumber(Int128 & value) { return current_converter->readInt128(value); }
|
||||
bool readNumber(UInt128 & value) { return current_converter->readUInt128(value); }
|
||||
bool readNumber(bInt256 & value) { return current_converter->readbInt256(value); }
|
||||
bool readNumber(bUInt256 & value) { return current_converter->readbUInt256(value); }
|
||||
bool readNumber(Int256 & value) { return current_converter->readInt256(value); }
|
||||
bool readNumber(UInt256 & value) { return current_converter->readUInt256(value); }
|
||||
bool readNumber(Float32 & value) { return current_converter->readFloat32(value); }
|
||||
bool readNumber(Float64 & value) { return current_converter->readFloat64(value); }
|
||||
|
||||
@ -152,8 +152,8 @@ private:
|
||||
virtual bool readInt128(Int128 &) = 0;
|
||||
virtual bool readUInt128(UInt128 &) = 0;
|
||||
|
||||
virtual bool readbInt256(bInt256 &) = 0;
|
||||
virtual bool readbUInt256(bUInt256 &) = 0;
|
||||
virtual bool readInt256(Int256 &) = 0;
|
||||
virtual bool readUInt256(UInt256 &) = 0;
|
||||
|
||||
virtual bool readFloat32(Float32 &) = 0;
|
||||
virtual bool readFloat64(Float64 &) = 0;
|
||||
@ -232,8 +232,8 @@ public:
|
||||
bool readNumber(UInt64 &) { return false; }
|
||||
bool readNumber(Int128 &) { return false; }
|
||||
bool readNumber(UInt128 &) { return false; }
|
||||
bool readNumber(bInt256 &) { return false; }
|
||||
bool readNumber(bUInt256 &) { return false; }
|
||||
bool readNumber(Int256 &) { return false; }
|
||||
bool readNumber(UInt256 &) { return false; }
|
||||
bool readNumber(Float32 &) { return false; }
|
||||
bool readNumber(Float64 &) { return false; }
|
||||
bool readStringInto(PaddedPODArray<UInt8> &) { return false; }
|
||||
|
@ -319,8 +319,8 @@ public:
|
||||
virtual void writeUInt64(UInt64) override { cannotConvertType("UInt64"); }
|
||||
virtual void writeInt128(Int128) override { cannotConvertType("Int128"); }
|
||||
virtual void writeUInt128(const UInt128 &) override { cannotConvertType("UInt128"); }
|
||||
virtual void writebInt256(const bInt256 &) override { cannotConvertType("Int256"); }
|
||||
virtual void writebUInt256(const bUInt256 &) override { cannotConvertType("UInt256"); }
|
||||
virtual void writeInt256(const Int256 &) override { cannotConvertType("Int256"); }
|
||||
virtual void writeUInt256(const UInt256 &) override { cannotConvertType("UInt256"); }
|
||||
virtual void writeFloat32(Float32) override { cannotConvertType("Float32"); }
|
||||
virtual void writeFloat64(Float64) override { cannotConvertType("Float64"); }
|
||||
virtual void prepareEnumMapping8(const std::vector<std::pair<std::string, Int8>> &) override {}
|
||||
|
@ -65,8 +65,8 @@ public:
|
||||
bool writeNumber(Int128 value) { return writeValueIfPossible(&IConverter::writeInt128, value); }
|
||||
bool writeNumber(UInt128 value) { return writeValueIfPossible(&IConverter::writeUInt128, value); }
|
||||
|
||||
bool writeNumber(bInt256 value) { return writeValueIfPossible(&IConverter::writebInt256, value); }
|
||||
bool writeNumber(bUInt256 value) { return writeValueIfPossible(&IConverter::writebUInt256, value); }
|
||||
bool writeNumber(Int256 value) { return writeValueIfPossible(&IConverter::writeInt256, value); }
|
||||
bool writeNumber(UInt256 value) { return writeValueIfPossible(&IConverter::writeUInt256, value); }
|
||||
|
||||
bool writeNumber(Float32 value) { return writeValueIfPossible(&IConverter::writeFloat32, value); }
|
||||
bool writeNumber(Float64 value) { return writeValueIfPossible(&IConverter::writeFloat64, value); }
|
||||
@ -156,8 +156,8 @@ private:
|
||||
virtual void writeInt128(Int128) = 0;
|
||||
virtual void writeUInt128(const UInt128 &) = 0;
|
||||
|
||||
virtual void writebInt256(const bInt256 &) = 0;
|
||||
virtual void writebUInt256(const bUInt256 &) = 0;
|
||||
virtual void writeInt256(const Int256 &) = 0;
|
||||
virtual void writeUInt256(const UInt256 &) = 0;
|
||||
|
||||
virtual void writeFloat32(Float32) = 0;
|
||||
virtual void writeFloat64(Float64) = 0;
|
||||
@ -267,8 +267,8 @@ public:
|
||||
bool writeNumber(UInt64 /* value */) { return false; }
|
||||
bool writeNumber(Int128 /* value */) { return false; }
|
||||
bool writeNumber(UInt128 /* value */) { return false; }
|
||||
bool writeNumber(bInt256 /* value */) { return false; }
|
||||
bool writeNumber(bUInt256 /* value */) { return false; }
|
||||
bool writeNumber(Int256 /* value */) { return false; }
|
||||
bool writeNumber(UInt256 /* value */) { return false; }
|
||||
bool writeNumber(Float32 /* value */) { return false; }
|
||||
bool writeNumber(Float64 /* value */) { return false; }
|
||||
bool writeString(const StringRef & /* value */) { return false; }
|
||||
|
@ -50,7 +50,11 @@ inline auto checkedDivision(A a, B b)
|
||||
{
|
||||
throwIfDivisionLeadsToFPE(a, b);
|
||||
|
||||
if constexpr (is_big_int_v<A> && is_big_int_v<B>)
|
||||
if constexpr (is_big_int_v<A> && std::is_floating_point_v<B>)
|
||||
return bigint_cast<B>(a) / b;
|
||||
else if constexpr (is_big_int_v<B> && std::is_floating_point_v<A>)
|
||||
return a / bigint_cast<A>(b);
|
||||
else if constexpr (is_big_int_v<A> && is_big_int_v<B>)
|
||||
return static_cast<A>(a / b);
|
||||
else if constexpr (!is_big_int_v<A> && is_big_int_v<B>)
|
||||
return static_cast<A>(B(a) / b);
|
||||
@ -70,28 +74,18 @@ struct DivideIntegralImpl
|
||||
template <typename Result = ResultType>
|
||||
static inline Result apply(A a, B b)
|
||||
{
|
||||
if constexpr (is_big_int_v<A> && std::is_floating_point_v<B>)
|
||||
return Result(static_cast<B>(a) / b);
|
||||
else if constexpr (is_big_int_v<B> && std::is_floating_point_v<A>)
|
||||
return a / static_cast<Result>(b);
|
||||
else if constexpr (is_big_int_v<A> && std::is_same_v<B, UInt8>)
|
||||
return static_cast<Result>(checkedDivision(make_signed_t<A>(a), Int16(b)));
|
||||
else if constexpr (is_big_int_v<B> && std::is_same_v<A, UInt8>)
|
||||
return checkedDivision(Int16(a), make_signed_t<B>(b));
|
||||
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
|
||||
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
|
||||
|
||||
/// Otherwise overflow may occur due to integer promotion. Example: int8_t(-1) / uint64_t(2).
|
||||
/// NOTE: overflow is still possible when dividing large signed number to large unsigned number or vice-versa. But it's less harmful.
|
||||
else if constexpr (is_signed_v<A> || is_signed_v<B>)
|
||||
if constexpr (is_integer_v<A> && is_integer_v<B> && (is_signed_v<A> || is_signed_v<B>))
|
||||
{
|
||||
if constexpr (is_integer_v<A> && is_integer_v<B>)
|
||||
{
|
||||
return checkedDivision(make_signed_t<A>(a),
|
||||
sizeof(A) > sizeof(B) ? make_signed_t<A>(b) : make_signed_t<B>(b));
|
||||
}
|
||||
else
|
||||
return checkedDivision(a, b);
|
||||
return checkedDivision(make_signed_t<CastA>(a),
|
||||
sizeof(A) > sizeof(B) ? make_signed_t<A>(CastB(b)) : make_signed_t<CastB>(b));
|
||||
}
|
||||
else
|
||||
return checkedDivision(a, b);
|
||||
return bigint_cast<Result>(checkedDivision(CastA(a), CastB(b)));
|
||||
}
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
@ -99,19 +93,6 @@ struct DivideIntegralImpl
|
||||
#endif
|
||||
};
|
||||
|
||||
template <typename Result, typename A, typename B>
|
||||
inline Result applyBigIntModulo(A a, B b)
|
||||
{
|
||||
if constexpr (std::is_same_v<A, UInt8>)
|
||||
return UInt16(a) % b;
|
||||
else if constexpr (std::is_same_v<B, UInt8>)
|
||||
return static_cast<UInt16>(a % UInt16(b));
|
||||
else if constexpr (sizeof(A) > sizeof(B))
|
||||
return static_cast<Result>(a % A(b));
|
||||
else
|
||||
return static_cast<Result>(B(a) % b);
|
||||
}
|
||||
|
||||
template <typename A, typename B>
|
||||
struct ModuloImpl
|
||||
{
|
||||
@ -120,7 +101,6 @@ struct ModuloImpl
|
||||
using IntegerBType = typename NumberTraits::ToInteger<B>::Type;
|
||||
|
||||
static const constexpr bool allow_fixed_string = false;
|
||||
static const constexpr bool is_special = is_big_int_v<IntegerAType> || is_big_int_v<IntegerBType>;
|
||||
|
||||
template <typename Result = ResultType>
|
||||
static inline Result apply(A a, B b)
|
||||
@ -134,8 +114,19 @@ struct ModuloImpl
|
||||
{
|
||||
throwIfDivisionLeadsToFPE(IntegerAType(a), IntegerBType(b));
|
||||
|
||||
if constexpr (is_special)
|
||||
return applyBigIntModulo<Result>(IntegerAType(a), IntegerBType(b));
|
||||
if constexpr (is_big_int_v<IntegerAType> || is_big_int_v<IntegerBType>)
|
||||
{
|
||||
using CastA = std::conditional_t<std::is_same_v<IntegerAType, UInt8>, uint8_t, IntegerAType>;
|
||||
using CastB = std::conditional_t<std::is_same_v<IntegerBType, UInt8>, uint8_t, IntegerBType>;
|
||||
|
||||
CastA int_a(a);
|
||||
CastB int_b(b);
|
||||
|
||||
if constexpr (is_big_int_v<IntegerBType> && sizeof(IntegerAType) <= sizeof(IntegerBType))
|
||||
return bigint_cast<Result>(bigint_cast<CastB>(int_a) % int_b);
|
||||
else
|
||||
return bigint_cast<Result>(int_a % int_b);
|
||||
}
|
||||
else
|
||||
return IntegerAType(a) % IntegerBType(b);
|
||||
}
|
||||
|
@ -634,13 +634,13 @@ private:
|
||||
|| executeNumRightType<T0, UInt32>(block, result, col_left, col_right_untyped)
|
||||
|| executeNumRightType<T0, UInt64>(block, result, col_left, col_right_untyped)
|
||||
|| executeNumRightType<T0, UInt128>(block, result, col_left, col_right_untyped)
|
||||
|| executeNumRightType<T0, bUInt256>(block, result, col_left, col_right_untyped)
|
||||
|| executeNumRightType<T0, UInt256>(block, result, col_left, col_right_untyped)
|
||||
|| executeNumRightType<T0, Int8>(block, result, col_left, col_right_untyped)
|
||||
|| executeNumRightType<T0, Int16>(block, result, col_left, col_right_untyped)
|
||||
|| executeNumRightType<T0, Int32>(block, result, col_left, col_right_untyped)
|
||||
|| executeNumRightType<T0, Int64>(block, result, col_left, col_right_untyped)
|
||||
|| executeNumRightType<T0, Int128>(block, result, col_left, col_right_untyped)
|
||||
|| executeNumRightType<T0, bInt256>(block, result, col_left, col_right_untyped)
|
||||
|| executeNumRightType<T0, Int256>(block, result, col_left, col_right_untyped)
|
||||
|| executeNumRightType<T0, Float32>(block, result, col_left, col_right_untyped)
|
||||
|| executeNumRightType<T0, Float64>(block, result, col_left, col_right_untyped))
|
||||
return true;
|
||||
@ -656,13 +656,13 @@ private:
|
||||
|| executeNumConstRightType<T0, UInt32>(block, result, col_left_const, col_right_untyped)
|
||||
|| executeNumConstRightType<T0, UInt64>(block, result, col_left_const, col_right_untyped)
|
||||
|| executeNumConstRightType<T0, UInt128>(block, result, col_left_const, col_right_untyped)
|
||||
|| executeNumConstRightType<T0, bUInt256>(block, result, col_left_const, col_right_untyped)
|
||||
|| executeNumConstRightType<T0, UInt256>(block, result, col_left_const, col_right_untyped)
|
||||
|| executeNumConstRightType<T0, Int8>(block, result, col_left_const, col_right_untyped)
|
||||
|| executeNumConstRightType<T0, Int16>(block, result, col_left_const, col_right_untyped)
|
||||
|| executeNumConstRightType<T0, Int32>(block, result, col_left_const, col_right_untyped)
|
||||
|| executeNumConstRightType<T0, Int64>(block, result, col_left_const, col_right_untyped)
|
||||
|| executeNumConstRightType<T0, Int128>(block, result, col_left_const, col_right_untyped)
|
||||
|| executeNumConstRightType<T0, bInt256>(block, result, col_left_const, col_right_untyped)
|
||||
|| executeNumConstRightType<T0, Int256>(block, result, col_left_const, col_right_untyped)
|
||||
|| executeNumConstRightType<T0, Float32>(block, result, col_left_const, col_right_untyped)
|
||||
|| executeNumConstRightType<T0, Float64>(block, result, col_left_const, col_right_untyped))
|
||||
return true;
|
||||
@ -1223,13 +1223,13 @@ public:
|
||||
|| executeNumLeftType<UInt32>(block, result, col_left_untyped, col_right_untyped)
|
||||
|| executeNumLeftType<UInt64>(block, result, col_left_untyped, col_right_untyped)
|
||||
|| executeNumLeftType<UInt128>(block, result, col_left_untyped, col_right_untyped)
|
||||
|| executeNumLeftType<bUInt256>(block, result, col_left_untyped, col_right_untyped)
|
||||
|| executeNumLeftType<UInt256>(block, result, col_left_untyped, col_right_untyped)
|
||||
|| executeNumLeftType<Int8>(block, result, col_left_untyped, col_right_untyped)
|
||||
|| executeNumLeftType<Int16>(block, result, col_left_untyped, col_right_untyped)
|
||||
|| executeNumLeftType<Int32>(block, result, col_left_untyped, col_right_untyped)
|
||||
|| executeNumLeftType<Int64>(block, result, col_left_untyped, col_right_untyped)
|
||||
|| executeNumLeftType<Int128>(block, result, col_left_untyped, col_right_untyped)
|
||||
|| executeNumLeftType<bInt256>(block, result, col_left_untyped, col_right_untyped)
|
||||
|| executeNumLeftType<Int256>(block, result, col_left_untyped, col_right_untyped)
|
||||
|| executeNumLeftType<Float32>(block, result, col_left_untyped, col_right_untyped)
|
||||
|| executeNumLeftType<Float64>(block, result, col_left_untyped, col_right_untyped)))
|
||||
throw Exception("Illegal column " + col_left_untyped->getName()
|
||||
|
@ -150,13 +150,10 @@ struct ConvertImpl
|
||||
}
|
||||
else if constexpr (is_big_int_v<FromFieldType> || is_big_int_v<ToFieldType>)
|
||||
{
|
||||
using CastFrom = std::conditional_t<std::is_same_v<FromFieldType, UInt8>, uint8_t, FromFieldType>;
|
||||
using CastTo = std::conditional_t<std::is_same_v<ToFieldType, UInt8>, uint8_t, ToFieldType>;
|
||||
|
||||
if constexpr (std::is_same_v<FromFieldType, UInt128> || std::is_same_v<ToFieldType, UInt128>)
|
||||
throw Exception("Unexpected UInt128 to big int conversion", ErrorCodes::NOT_IMPLEMENTED);
|
||||
else
|
||||
vec_to[i] = static_cast<CastTo>(static_cast<CastFrom>(vec_from[i]));
|
||||
vec_to[i] = bigint_cast<ToFieldType>(vec_from[i]);
|
||||
}
|
||||
else if constexpr (std::is_same_v<ToFieldType, UInt128> && sizeof(FromFieldType) <= sizeof(UInt64))
|
||||
vec_to[i] = static_cast<ToFieldType>(static_cast<UInt64>(vec_from[i]));
|
||||
@ -1630,13 +1627,13 @@ using FunctionToUInt8 = FunctionConvert<DataTypeUInt8, NameToUInt8, ToNumberMono
|
||||
using FunctionToUInt16 = FunctionConvert<DataTypeUInt16, NameToUInt16, ToNumberMonotonicity<UInt16>>;
|
||||
using FunctionToUInt32 = FunctionConvert<DataTypeUInt32, NameToUInt32, ToNumberMonotonicity<UInt32>>;
|
||||
using FunctionToUInt64 = FunctionConvert<DataTypeUInt64, NameToUInt64, ToNumberMonotonicity<UInt64>>;
|
||||
using FunctionToUInt256 = FunctionConvert<DataTypeUInt256, NameToUInt256, ToNumberMonotonicity<bUInt256>>;
|
||||
using FunctionToUInt256 = FunctionConvert<DataTypeUInt256, NameToUInt256, ToNumberMonotonicity<UInt256>>;
|
||||
using FunctionToInt8 = FunctionConvert<DataTypeInt8, NameToInt8, ToNumberMonotonicity<Int8>>;
|
||||
using FunctionToInt16 = FunctionConvert<DataTypeInt16, NameToInt16, ToNumberMonotonicity<Int16>>;
|
||||
using FunctionToInt32 = FunctionConvert<DataTypeInt32, NameToInt32, ToNumberMonotonicity<Int32>>;
|
||||
using FunctionToInt64 = FunctionConvert<DataTypeInt64, NameToInt64, ToNumberMonotonicity<Int64>>;
|
||||
using FunctionToInt128 = FunctionConvert<DataTypeInt128, NameToInt128, ToNumberMonotonicity<Int128>>;
|
||||
using FunctionToInt256 = FunctionConvert<DataTypeInt256, NameToInt256, ToNumberMonotonicity<bInt256>>;
|
||||
using FunctionToInt256 = FunctionConvert<DataTypeInt256, NameToInt256, ToNumberMonotonicity<Int256>>;
|
||||
using FunctionToFloat32 = FunctionConvert<DataTypeFloat32, NameToFloat32, ToNumberMonotonicity<Float32>>;
|
||||
using FunctionToFloat64 = FunctionConvert<DataTypeFloat64, NameToFloat64, ToNumberMonotonicity<Float64>>;
|
||||
using FunctionToDate = FunctionConvert<DataTypeDate, NameToDate, ToDateMonotonicity>;
|
||||
|
@ -968,13 +968,13 @@ private:
|
||||
else if (which.isUInt32()) executeIntType<UInt32, first>(icolumn, vec_to);
|
||||
else if (which.isUInt64()) executeIntType<UInt64, first>(icolumn, vec_to);
|
||||
else if (which.isUInt128() || which.isUUID()) executeBigIntType<UInt128, first>(icolumn, vec_to);
|
||||
else if (which.isbUInt256()) executeBigIntType<bUInt256, first>(icolumn, vec_to);
|
||||
else if (which.isUInt256()) executeBigIntType<UInt256, first>(icolumn, vec_to);
|
||||
else if (which.isInt8()) executeIntType<Int8, first>(icolumn, vec_to);
|
||||
else if (which.isInt16()) executeIntType<Int16, first>(icolumn, vec_to);
|
||||
else if (which.isInt32()) executeIntType<Int32, first>(icolumn, vec_to);
|
||||
else if (which.isInt64()) executeIntType<Int64, first>(icolumn, vec_to);
|
||||
else if (which.isInt128()) executeBigIntType<Int128, first>(icolumn, vec_to);
|
||||
else if (which.isbInt256()) executeBigIntType<bInt256, first>(icolumn, vec_to);
|
||||
else if (which.isInt256()) executeBigIntType<Int256, first>(icolumn, vec_to);
|
||||
else if (which.isEnum8()) executeIntType<Int8, first>(icolumn, vec_to);
|
||||
else if (which.isEnum16()) executeIntType<Int16, first>(icolumn, vec_to);
|
||||
else if (which.isDate()) executeIntType<UInt16, first>(icolumn, vec_to);
|
||||
|
@ -45,21 +45,14 @@ void writeSlice(const NumericArraySlice<T> & slice, NumericArraySink<U> & sink)
|
||||
|
||||
if constexpr (OverBigInt<T> || OverBigInt<U>)
|
||||
{
|
||||
if constexpr (std::is_same_v<T, UInt8> || std::is_same_v<U, UInt8>)
|
||||
{
|
||||
if constexpr (IsDecimalNumber<T>)
|
||||
dst = static_cast<NativeU>(static_cast<UInt16>(src.value));
|
||||
else
|
||||
dst = static_cast<NativeU>(static_cast<UInt16>(src));
|
||||
}
|
||||
else if constexpr (std::is_same_v<U, UInt128>)
|
||||
if constexpr (std::is_same_v<U, UInt128>)
|
||||
{
|
||||
throw Exception("No conversion between UInt128 and " + demangle(typeid(T).name()), ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
else if constexpr (IsDecimalNumber<T>)
|
||||
dst = static_cast<NativeU>(src.value);
|
||||
dst = bigint_cast<NativeU>(src.value);
|
||||
else
|
||||
dst = static_cast<NativeU>(src);
|
||||
dst = bigint_cast<NativeU>(src);
|
||||
}
|
||||
else
|
||||
dst = static_cast<NativeU>(src);
|
||||
|
@ -14,20 +14,11 @@ struct BitAndImpl
|
||||
{
|
||||
using ResultType = typename NumberTraits::ResultOfBit<A, B>::Type;
|
||||
static constexpr const bool allow_fixed_string = true;
|
||||
static constexpr bool need_uint8_cast = is_big_int_v<ResultType> && (std::is_same_v<A, UInt8> || std::is_same_v<B, UInt8>);
|
||||
|
||||
template <typename Result = ResultType>
|
||||
static inline Result apply(A a, B b)
|
||||
{
|
||||
if constexpr (need_uint8_cast)
|
||||
{
|
||||
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, A>;
|
||||
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
|
||||
|
||||
return static_cast<Result>(static_cast<CastA>(a)) & static_cast<Result>(static_cast<CastB>(b));
|
||||
}
|
||||
else
|
||||
return static_cast<Result>(a) & static_cast<Result>(b);
|
||||
return bigint_cast<Result>(a) & bigint_cast<Result>(b);
|
||||
}
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
|
@ -13,20 +13,11 @@ struct BitOrImpl
|
||||
{
|
||||
using ResultType = typename NumberTraits::ResultOfBit<A, B>::Type;
|
||||
static constexpr const bool allow_fixed_string = true;
|
||||
static constexpr bool need_uint8_cast = is_big_int_v<ResultType> && (std::is_same_v<A, UInt8> || std::is_same_v<B, UInt8>);
|
||||
|
||||
template <typename Result = ResultType>
|
||||
static inline Result apply(A a, B b)
|
||||
{
|
||||
if constexpr (need_uint8_cast)
|
||||
{
|
||||
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, A>;
|
||||
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
|
||||
|
||||
return static_cast<Result>(static_cast<CastA>(a)) | static_cast<Result>(static_cast<CastB>(b));
|
||||
}
|
||||
else
|
||||
return static_cast<Result>(a) | static_cast<Result>(b);
|
||||
return bigint_cast<Result>(a) | bigint_cast<Result>(b);
|
||||
}
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
|
@ -18,12 +18,10 @@ struct BitShiftLeftImpl
|
||||
template <typename Result = ResultType>
|
||||
static inline NO_SANITIZE_UNDEFINED Result apply(A a [[maybe_unused]], B b [[maybe_unused]])
|
||||
{
|
||||
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
|
||||
|
||||
if constexpr (is_big_int_v<B>)
|
||||
throw Exception("BitShiftLeftImpl is not implemented for big integers as second argument", ErrorCodes::NOT_IMPLEMENTED);
|
||||
else if constexpr (is_big_int_v<ResultType>)
|
||||
return static_cast<Result>(a) << static_cast<CastB>(b);
|
||||
else if constexpr (is_big_int_v<A>)
|
||||
return static_cast<Result>(a) << bigint_cast<UInt32>(b);
|
||||
else
|
||||
return static_cast<Result>(a) << static_cast<Result>(b);
|
||||
}
|
||||
|
@ -18,12 +18,10 @@ struct BitShiftRightImpl
|
||||
template <typename Result = ResultType>
|
||||
static inline NO_SANITIZE_UNDEFINED Result apply(A a [[maybe_unused]], B b [[maybe_unused]])
|
||||
{
|
||||
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
|
||||
|
||||
if constexpr (is_big_int_v<B>)
|
||||
throw Exception("BitRotate is not implemented for big integers as second argument", ErrorCodes::NOT_IMPLEMENTED);
|
||||
else if constexpr (is_big_int_v<ResultType>)
|
||||
return static_cast<Result>(a) >> static_cast<CastB>(b);
|
||||
else if constexpr (is_big_int_v<A>)
|
||||
return static_cast<Result>(a) >> bigint_cast<UInt32>(b);
|
||||
else
|
||||
return static_cast<Result>(a) >> static_cast<Result>(b);
|
||||
}
|
||||
|
@ -13,20 +13,11 @@ struct BitXorImpl
|
||||
{
|
||||
using ResultType = typename NumberTraits::ResultOfBit<A, B>::Type;
|
||||
static constexpr bool allow_fixed_string = true;
|
||||
static constexpr bool need_uint8_cast = is_big_int_v<ResultType> && (std::is_same_v<A, UInt8> || std::is_same_v<B, UInt8>);
|
||||
|
||||
template <typename Result = ResultType>
|
||||
static inline Result apply(A a, B b)
|
||||
{
|
||||
if constexpr (need_uint8_cast)
|
||||
{
|
||||
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, A>;
|
||||
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
|
||||
|
||||
return static_cast<Result>(static_cast<CastA>(a)) ^ static_cast<Result>(static_cast<CastB>(b));
|
||||
}
|
||||
else
|
||||
return static_cast<Result>(a) ^ static_cast<Result>(b);
|
||||
return bigint_cast<Result>(a) ^ bigint_cast<Result>(b);
|
||||
}
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
|
@ -12,22 +12,12 @@ struct GreatestBaseImpl
|
||||
{
|
||||
using ResultType = NumberTraits::ResultOfGreatest<A, B>;
|
||||
static const constexpr bool allow_fixed_string = false;
|
||||
static constexpr bool need_uint8_cast = is_big_int_v<ResultType> && (std::is_same_v<A, UInt8> || std::is_same_v<B, UInt8>);
|
||||
|
||||
template <typename Result = ResultType>
|
||||
static inline Result apply(A a, B b)
|
||||
{
|
||||
if constexpr (need_uint8_cast)
|
||||
{
|
||||
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, A>;
|
||||
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
|
||||
|
||||
return static_cast<Result>(static_cast<CastA>(a)) > static_cast<Result>(static_cast<CastB>(b)) ?
|
||||
static_cast<Result>(static_cast<CastA>(a)) : static_cast<Result>(static_cast<CastB>(b));
|
||||
}
|
||||
else
|
||||
return static_cast<Result>(a) > static_cast<Result>(b) ?
|
||||
static_cast<Result>(a) : static_cast<Result>(b);
|
||||
return bigint_cast<Result>(a) > bigint_cast<Result>(b) ?
|
||||
bigint_cast<Result>(a) : bigint_cast<Result>(b);
|
||||
}
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
|
@ -44,15 +44,6 @@ using namespace GatherUtils;
|
||||
* then, else - numeric types for which there is a general type, or dates, datetimes, or strings, or arrays of these types.
|
||||
*/
|
||||
|
||||
template <typename From, typename To>
|
||||
inline To special_cast(From x)
|
||||
{
|
||||
if constexpr (is_big_int_v<To> && std::is_same_v<From, UInt8>)
|
||||
return static_cast<To>(static_cast<UInt16>(x));
|
||||
else
|
||||
return static_cast<To>(x);
|
||||
}
|
||||
|
||||
template <typename A, typename B, typename ResultType>
|
||||
struct NumIfImpl
|
||||
{
|
||||
@ -68,7 +59,7 @@ struct NumIfImpl
|
||||
typename ColVecResult::Container & res = col_res->getData();
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
res[i] = cond[i] ? special_cast<A, ResultType>(a[i]) : special_cast<B, ResultType>(b[i]);
|
||||
res[i] = cond[i] ? bigint_cast<ResultType>(a[i]) : bigint_cast<ResultType>(b[i]);
|
||||
block.getByPosition(result).column = std::move(col_res);
|
||||
}
|
||||
|
||||
@ -79,7 +70,7 @@ struct NumIfImpl
|
||||
typename ColVecResult::Container & res = col_res->getData();
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
res[i] = cond[i] ? special_cast<A, ResultType>(a[i]) : special_cast<B, ResultType>(b);
|
||||
res[i] = cond[i] ? bigint_cast<ResultType>(a[i]) : bigint_cast<ResultType>(b);
|
||||
block.getByPosition(result).column = std::move(col_res);
|
||||
}
|
||||
|
||||
@ -90,7 +81,7 @@ struct NumIfImpl
|
||||
typename ColVecResult::Container & res = col_res->getData();
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
res[i] = cond[i] ? special_cast<A, ResultType>(a) : special_cast<B, ResultType>(b[i]);
|
||||
res[i] = cond[i] ? bigint_cast<ResultType>(a) : bigint_cast<ResultType>(b[i]);
|
||||
block.getByPosition(result).column = std::move(col_res);
|
||||
}
|
||||
|
||||
@ -101,7 +92,7 @@ struct NumIfImpl
|
||||
typename ColVecResult::Container & res = col_res->getData();
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
res[i] = cond[i] ? special_cast<A, ResultType>(a) : special_cast<B, ResultType>(b);
|
||||
res[i] = cond[i] ? bigint_cast<ResultType>(a) : bigint_cast<ResultType>(b);
|
||||
block.getByPosition(result).column = std::move(col_res);
|
||||
}
|
||||
};
|
||||
|
@ -12,22 +12,12 @@ struct LeastBaseImpl
|
||||
{
|
||||
using ResultType = NumberTraits::ResultOfLeast<A, B>;
|
||||
static const constexpr bool allow_fixed_string = false;
|
||||
static constexpr bool need_uint8_cast = is_big_int_v<ResultType> && (std::is_same_v<A, UInt8> || std::is_same_v<B, UInt8>);
|
||||
|
||||
template <typename Result = ResultType>
|
||||
static inline Result apply(A a, B b)
|
||||
{
|
||||
if constexpr (need_uint8_cast)
|
||||
{
|
||||
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, A>;
|
||||
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
|
||||
|
||||
return static_cast<Result>(static_cast<CastA>(a)) < static_cast<Result>(static_cast<CastB>(b)) ?
|
||||
static_cast<Result>(static_cast<CastA>(a)) : static_cast<Result>(static_cast<CastB>(b));
|
||||
}
|
||||
else
|
||||
/** gcc 4.9.2 successfully vectorizes a loop from this function. */
|
||||
return static_cast<Result>(a) < static_cast<Result>(b) ? static_cast<Result>(a) : static_cast<Result>(b);
|
||||
/** gcc 4.9.2 successfully vectorizes a loop from this function. */
|
||||
return bigint_cast<Result>(a) < bigint_cast<Result>(b) ? bigint_cast<Result>(a) : bigint_cast<Result>(b);
|
||||
}
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
|
@ -17,10 +17,10 @@ struct MinusImpl
|
||||
{
|
||||
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
|
||||
{
|
||||
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, std::conditional_t<std::is_floating_point_v<B>, B, A>>;
|
||||
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, std::conditional_t<std::is_floating_point_v<A>, A, B>>;
|
||||
using CastA = std::conditional_t<std::is_floating_point_v<B>, B, A>;
|
||||
using CastB = std::conditional_t<std::is_floating_point_v<A>, A, B>;
|
||||
|
||||
return static_cast<Result>(static_cast<CastA>(a)) - static_cast<Result>(static_cast<CastB>(b));
|
||||
return bigint_cast<Result>(bigint_cast<CastA>(a)) - bigint_cast<Result>(bigint_cast<CastB>(b));
|
||||
}
|
||||
else
|
||||
return static_cast<Result>(a) - b;
|
||||
|
@ -17,10 +17,10 @@ struct MultiplyImpl
|
||||
{
|
||||
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
|
||||
{
|
||||
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, std::conditional_t<std::is_floating_point_v<B>, B, A>>;
|
||||
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, std::conditional_t<std::is_floating_point_v<A>, A, B>>;
|
||||
using CastA = std::conditional_t<std::is_floating_point_v<B>, B, A>;
|
||||
using CastB = std::conditional_t<std::is_floating_point_v<A>, A, B>;
|
||||
|
||||
return static_cast<Result>(static_cast<CastA>(a)) * static_cast<Result>(static_cast<CastB>(b));
|
||||
return bigint_cast<Result>(bigint_cast<CastA>(a)) * bigint_cast<Result>(bigint_cast<CastB>(b));
|
||||
}
|
||||
else
|
||||
return static_cast<Result>(a) * b;
|
||||
|
@ -18,10 +18,10 @@ struct PlusImpl
|
||||
/// Next everywhere, static_cast - so that there is no wrong result in expressions of the form Int64 c = UInt32(a) * Int32(-1).
|
||||
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
|
||||
{
|
||||
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, std::conditional_t<std::is_floating_point_v<B>, B, A>>;
|
||||
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, std::conditional_t<std::is_floating_point_v<A>, A, B>>;
|
||||
using CastA = std::conditional_t<std::is_floating_point_v<B>, B, A>;
|
||||
using CastB = std::conditional_t<std::is_floating_point_v<A>, A, B>;
|
||||
|
||||
return static_cast<Result>(static_cast<CastA>(a)) + static_cast<Result>(static_cast<CastB>(b));
|
||||
return bigint_cast<Result>(bigint_cast<CastA>(a)) + bigint_cast<Result>(bigint_cast<CastB>(b));
|
||||
}
|
||||
else
|
||||
return static_cast<Result>(a) + b;
|
||||
|
@ -13,169 +13,169 @@ static const std::map<std::pair<std::string, std::string>, std::string> answer =
|
||||
{{"UInt8", "UInt16"}, "UInt16"},
|
||||
{{"UInt8", "UInt32"}, "UInt32"},
|
||||
{{"UInt8", "UInt64"}, "UInt64"},
|
||||
{{"UInt8", "bUInt256"}, "bUInt256"},
|
||||
{{"UInt8", "UInt256"}, "UInt256"},
|
||||
{{"UInt8", "Int8"}, "Int16"},
|
||||
{{"UInt8", "Int16"}, "Int16"},
|
||||
{{"UInt8", "Int32"}, "Int32"},
|
||||
{{"UInt8", "Int64"}, "Int64"},
|
||||
{{"UInt8", "Int128"}, "Int128"},
|
||||
{{"UInt8", "bInt256"}, "bInt256"},
|
||||
{{"UInt8", "Int256"}, "Int256"},
|
||||
{{"UInt8", "Float32"}, "Float32"},
|
||||
{{"UInt8", "Float64"}, "Float64"},
|
||||
{{"UInt16", "UInt8"}, "UInt16"},
|
||||
{{"UInt16", "UInt16"}, "UInt16"},
|
||||
{{"UInt16", "UInt32"}, "UInt32"},
|
||||
{{"UInt16", "UInt64"}, "UInt64"},
|
||||
{{"UInt16", "bUInt256"}, "bUInt256"},
|
||||
{{"UInt16", "UInt256"}, "UInt256"},
|
||||
{{"UInt16", "Int8"}, "Int32"},
|
||||
{{"UInt16", "Int16"}, "Int32"},
|
||||
{{"UInt16", "Int32"}, "Int32"},
|
||||
{{"UInt16", "Int64"}, "Int64"},
|
||||
{{"UInt16", "Int128"}, "Int128"},
|
||||
{{"UInt16", "bInt256"}, "bInt256"},
|
||||
{{"UInt16", "Int256"}, "Int256"},
|
||||
{{"UInt16", "Float32"}, "Float32"},
|
||||
{{"UInt16", "Float64"}, "Float64"},
|
||||
{{"UInt32", "UInt8"}, "UInt32"},
|
||||
{{"UInt32", "UInt16"}, "UInt32"},
|
||||
{{"UInt32", "UInt32"}, "UInt32"},
|
||||
{{"UInt32", "UInt64"}, "UInt64"},
|
||||
{{"UInt32", "bUInt256"}, "bUInt256"},
|
||||
{{"UInt32", "UInt256"}, "UInt256"},
|
||||
{{"UInt32", "Int8"}, "Int64"},
|
||||
{{"UInt32", "Int16"}, "Int64"},
|
||||
{{"UInt32", "Int32"}, "Int64"},
|
||||
{{"UInt32", "Int64"}, "Int64"},
|
||||
{{"UInt32", "Int128"}, "Int128"},
|
||||
{{"UInt32", "bInt256"}, "bInt256"},
|
||||
{{"UInt32", "Int256"}, "Int256"},
|
||||
{{"UInt32", "Float32"}, "Float64"},
|
||||
{{"UInt32", "Float64"}, "Float64"},
|
||||
{{"UInt64", "UInt8"}, "UInt64"},
|
||||
{{"UInt64", "UInt16"}, "UInt64"},
|
||||
{{"UInt64", "UInt32"}, "UInt64"},
|
||||
{{"UInt64", "UInt64"}, "UInt64"},
|
||||
{{"UInt64", "bUInt256"}, "bUInt256"},
|
||||
{{"UInt64", "UInt256"}, "UInt256"},
|
||||
{{"UInt64", "Int8"}, "Int128"},
|
||||
{{"UInt64", "Int16"}, "Int128"},
|
||||
{{"UInt64", "Int32"}, "Int128"},
|
||||
{{"UInt64", "Int64"}, "Int128"},
|
||||
{{"UInt64", "Int128"}, "Int128"},
|
||||
{{"UInt64", "bInt256"}, "bInt256"},
|
||||
{{"UInt64", "Int256"}, "Int256"},
|
||||
{{"UInt64", "Float32"}, "Error"},
|
||||
{{"UInt64", "Float64"}, "Error"},
|
||||
{{"bUInt256", "UInt8"}, "bUInt256"},
|
||||
{{"bUInt256", "UInt16"}, "bUInt256"},
|
||||
{{"bUInt256", "UInt32"}, "bUInt256"},
|
||||
{{"bUInt256", "UInt64"}, "bUInt256"},
|
||||
{{"bUInt256", "bUInt256"}, "bUInt256"},
|
||||
{{"bUInt256", "Int8"}, "Error"},
|
||||
{{"bUInt256", "Int16"}, "Error"},
|
||||
{{"bUInt256", "Int32"}, "Error"},
|
||||
{{"bUInt256", "Int64"}, "Error"},
|
||||
{{"bUInt256", "Int128"}, "Error"},
|
||||
{{"bUInt256", "bInt256"}, "Error"},
|
||||
{{"bUInt256", "Float32"}, "Error"},
|
||||
{{"bUInt256", "Float64"}, "Error"},
|
||||
{{"UInt256", "UInt8"}, "UInt256"},
|
||||
{{"UInt256", "UInt16"}, "UInt256"},
|
||||
{{"UInt256", "UInt32"}, "UInt256"},
|
||||
{{"UInt256", "UInt64"}, "UInt256"},
|
||||
{{"UInt256", "UInt256"}, "UInt256"},
|
||||
{{"UInt256", "Int8"}, "Error"},
|
||||
{{"UInt256", "Int16"}, "Error"},
|
||||
{{"UInt256", "Int32"}, "Error"},
|
||||
{{"UInt256", "Int64"}, "Error"},
|
||||
{{"UInt256", "Int128"}, "Error"},
|
||||
{{"UInt256", "Int256"}, "Error"},
|
||||
{{"UInt256", "Float32"}, "Error"},
|
||||
{{"UInt256", "Float64"}, "Error"},
|
||||
{{"Int8", "UInt8"}, "Int16"},
|
||||
{{"Int8", "UInt16"}, "Int32"},
|
||||
{{"Int8", "UInt32"}, "Int64"},
|
||||
{{"Int8", "UInt64"}, "Int128"},
|
||||
{{"Int8", "bUInt256"}, "Error"},
|
||||
{{"Int8", "UInt256"}, "Error"},
|
||||
{{"Int8", "Int8"}, "Int8"},
|
||||
{{"Int8", "Int16"}, "Int16"},
|
||||
{{"Int8", "Int32"}, "Int32"},
|
||||
{{"Int8", "Int64"}, "Int64"},
|
||||
{{"Int8", "Int128"}, "Int128"},
|
||||
{{"Int8", "bInt256"}, "bInt256"},
|
||||
{{"Int8", "Int256"}, "Int256"},
|
||||
{{"Int8", "Float32"}, "Float32"},
|
||||
{{"Int8", "Float64"}, "Float64"},
|
||||
{{"Int16", "UInt8"}, "Int16"},
|
||||
{{"Int16", "UInt16"}, "Int32"},
|
||||
{{"Int16", "UInt32"}, "Int64"},
|
||||
{{"Int16", "UInt64"}, "Int128"},
|
||||
{{"Int16", "bUInt256"}, "Error"},
|
||||
{{"Int16", "UInt256"}, "Error"},
|
||||
{{"Int16", "Int8"}, "Int16"},
|
||||
{{"Int16", "Int16"}, "Int16"},
|
||||
{{"Int16", "Int32"}, "Int32"},
|
||||
{{"Int16", "Int64"}, "Int64"},
|
||||
{{"Int16", "Int128"}, "Int128"},
|
||||
{{"Int16", "bInt256"}, "bInt256"},
|
||||
{{"Int16", "Int256"}, "Int256"},
|
||||
{{"Int16", "Float32"}, "Float32"},
|
||||
{{"Int16", "Float64"}, "Float64"},
|
||||
{{"Int32", "UInt8"}, "Int32"},
|
||||
{{"Int32", "UInt16"}, "Int32"},
|
||||
{{"Int32", "UInt32"}, "Int64"},
|
||||
{{"Int32", "UInt64"}, "Int128"},
|
||||
{{"Int32", "bUInt256"}, "Error"},
|
||||
{{"Int32", "UInt256"}, "Error"},
|
||||
{{"Int32", "Int8"}, "Int32"},
|
||||
{{"Int32", "Int16"}, "Int32"},
|
||||
{{"Int32", "Int32"}, "Int32"},
|
||||
{{"Int32", "Int64"}, "Int64"},
|
||||
{{"Int32", "Int128"}, "Int128"},
|
||||
{{"Int32", "bInt256"}, "bInt256"},
|
||||
{{"Int32", "Int256"}, "Int256"},
|
||||
{{"Int32", "Float32"}, "Float64"},
|
||||
{{"Int32", "Float64"}, "Float64"},
|
||||
{{"Int64", "UInt8"}, "Int64"},
|
||||
{{"Int64", "UInt16"}, "Int64"},
|
||||
{{"Int64", "UInt32"}, "Int64"},
|
||||
{{"Int64", "UInt64"}, "Int128"},
|
||||
{{"Int64", "bUInt256"}, "Error"},
|
||||
{{"Int64", "UInt256"}, "Error"},
|
||||
{{"Int64", "Int8"}, "Int64"},
|
||||
{{"Int64", "Int16"}, "Int64"},
|
||||
{{"Int64", "Int32"}, "Int64"},
|
||||
{{"Int64", "Int64"}, "Int64"},
|
||||
{{"Int64", "Int128"}, "Int128"},
|
||||
{{"Int64", "bInt256"}, "bInt256"},
|
||||
{{"Int64", "Int256"}, "Int256"},
|
||||
{{"Int64", "Float32"}, "Error"},
|
||||
{{"Int64", "Float64"}, "Error"},
|
||||
{{"Int128", "UInt8"}, "Int128"},
|
||||
{{"Int128", "UInt16"}, "Int128"},
|
||||
{{"Int128", "UInt32"}, "Int128"},
|
||||
{{"Int128", "UInt64"}, "Int128"},
|
||||
{{"Int128", "bUInt256"}, "Error"},
|
||||
{{"Int128", "UInt256"}, "Error"},
|
||||
{{"Int128", "Int8"}, "Int128"},
|
||||
{{"Int128", "Int16"}, "Int128"},
|
||||
{{"Int128", "Int32"}, "Int128"},
|
||||
{{"Int128", "Int64"}, "Int128"},
|
||||
{{"Int128", "Int128"}, "Int128"},
|
||||
{{"Int128", "bInt256"}, "bInt256"},
|
||||
{{"Int128", "Int256"}, "Int256"},
|
||||
{{"Int128", "Float32"}, "Error"},
|
||||
{{"Int128", "Float64"}, "Error"},
|
||||
{{"bInt256", "UInt8"}, "bInt256"},
|
||||
{{"bInt256", "UInt16"}, "bInt256"},
|
||||
{{"bInt256", "UInt32"}, "bInt256"},
|
||||
{{"bInt256", "UInt64"}, "bInt256"},
|
||||
{{"bInt256", "bUInt256"}, "Error"},
|
||||
{{"bInt256", "Int8"}, "bInt256"},
|
||||
{{"bInt256", "Int16"}, "bInt256"},
|
||||
{{"bInt256", "Int32"}, "bInt256"},
|
||||
{{"bInt256", "Int64"}, "bInt256"},
|
||||
{{"bInt256", "Int128"}, "bInt256"},
|
||||
{{"bInt256", "bInt256"}, "bInt256"},
|
||||
{{"bInt256", "Float32"}, "Error"},
|
||||
{{"bInt256", "Float64"}, "Error"},
|
||||
{{"Int256", "UInt8"}, "Int256"},
|
||||
{{"Int256", "UInt16"}, "Int256"},
|
||||
{{"Int256", "UInt32"}, "Int256"},
|
||||
{{"Int256", "UInt64"}, "Int256"},
|
||||
{{"Int256", "UInt256"}, "Error"},
|
||||
{{"Int256", "Int8"}, "Int256"},
|
||||
{{"Int256", "Int16"}, "Int256"},
|
||||
{{"Int256", "Int32"}, "Int256"},
|
||||
{{"Int256", "Int64"}, "Int256"},
|
||||
{{"Int256", "Int128"}, "Int256"},
|
||||
{{"Int256", "Int256"}, "Int256"},
|
||||
{{"Int256", "Float32"}, "Error"},
|
||||
{{"Int256", "Float64"}, "Error"},
|
||||
{{"Float32", "UInt8"}, "Float32"},
|
||||
{{"Float32", "UInt16"}, "Float32"},
|
||||
{{"Float32", "UInt32"}, "Float64"},
|
||||
{{"Float32", "UInt64"}, "Error"},
|
||||
{{"Float32", "bUInt256"}, "Error"},
|
||||
{{"Float32", "UInt256"}, "Error"},
|
||||
{{"Float32", "Int8"}, "Float32"},
|
||||
{{"Float32", "Int16"}, "Float32"},
|
||||
{{"Float32", "Int32"}, "Float64"},
|
||||
{{"Float32", "Int64"}, "Error"},
|
||||
{{"Float32", "Int128"}, "Error"},
|
||||
{{"Float32", "bInt256"}, "Error"},
|
||||
{{"Float32", "Int256"}, "Error"},
|
||||
{{"Float32", "Float32"}, "Float32"},
|
||||
{{"Float32", "Float64"}, "Float64"},
|
||||
{{"Float64", "UInt8"}, "Float64"},
|
||||
{{"Float64", "UInt16"}, "Float64"},
|
||||
{{"Float64", "UInt32"}, "Float64"},
|
||||
{{"Float64", "UInt64"}, "Error"},
|
||||
{{"Float64", "bUInt256"}, "Error"},
|
||||
{{"Float64", "UInt256"}, "Error"},
|
||||
{{"Float64", "Int8"}, "Float64"},
|
||||
{{"Float64", "Int16"}, "Float64"},
|
||||
{{"Float64", "Int32"}, "Float64"},
|
||||
{{"Float64", "Int64"}, "Error"},
|
||||
{{"Float64", "Int128"}, "Error"},
|
||||
{{"Float64", "bInt256"}, "Error"},
|
||||
{{"Float64", "Int256"}, "Error"},
|
||||
{{"Float64", "Float32"}, "Float64"},
|
||||
{{"Float64", "Float64"}, "Float64"}
|
||||
};
|
||||
@ -184,13 +184,13 @@ static std::string getTypeString(DB::UInt8) { return "UInt8"; }
|
||||
static std::string getTypeString(DB::UInt16) { return "UInt16"; }
|
||||
static std::string getTypeString(DB::UInt32) { return "UInt32"; }
|
||||
static std::string getTypeString(DB::UInt64) { return "UInt64"; }
|
||||
static std::string getTypeString(DB::bUInt256) { return "bUInt256"; }
|
||||
static std::string getTypeString(DB::UInt256) { return "UInt256"; }
|
||||
static std::string getTypeString(DB::Int8) { return "Int8"; }
|
||||
static std::string getTypeString(DB::Int16) { return "Int16"; }
|
||||
static std::string getTypeString(DB::Int32) { return "Int32"; }
|
||||
static std::string getTypeString(DB::Int64) { return "Int64"; }
|
||||
static std::string getTypeString(DB::Int128) { return "Int128"; }
|
||||
static std::string getTypeString(DB::bInt256) { return "bInt256"; }
|
||||
static std::string getTypeString(DB::Int256) { return "Int256"; }
|
||||
static std::string getTypeString(DB::Float32) { return "Float32"; }
|
||||
static std::string getTypeString(DB::Float64) { return "Float64"; }
|
||||
static std::string getTypeString(DB::NumberTraits::Error) { return "Error"; }
|
||||
@ -225,13 +225,13 @@ void ifLeftType()
|
||||
ifRightType<T0, DB::UInt16>();
|
||||
ifRightType<T0, DB::UInt32>();
|
||||
ifRightType<T0, DB::UInt64>();
|
||||
ifRightType<T0, DB::bUInt256>();
|
||||
ifRightType<T0, DB::UInt256>();
|
||||
ifRightType<T0, DB::Int8>();
|
||||
ifRightType<T0, DB::Int16>();
|
||||
ifRightType<T0, DB::Int32>();
|
||||
ifRightType<T0, DB::Int64>();
|
||||
ifRightType<T0, DB::Int128>();
|
||||
ifRightType<T0, DB::bInt256>();
|
||||
ifRightType<T0, DB::Int256>();
|
||||
ifRightType<T0, DB::Float32>();
|
||||
ifRightType<T0, DB::Float64>();
|
||||
}
|
||||
@ -268,13 +268,13 @@ TEST(NumberTraits, FunctionIf)
|
||||
ifLeftType<DB::UInt16>();
|
||||
ifLeftType<DB::UInt32>();
|
||||
ifLeftType<DB::UInt64>();
|
||||
ifLeftType<DB::bUInt256>();
|
||||
ifLeftType<DB::UInt256>();
|
||||
ifLeftType<DB::Int8>();
|
||||
ifLeftType<DB::Int16>();
|
||||
ifLeftType<DB::Int32>();
|
||||
ifLeftType<DB::Int64>();
|
||||
ifLeftType<DB::Int128>();
|
||||
ifLeftType<DB::bInt256>();
|
||||
ifLeftType<DB::Int256>();
|
||||
ifLeftType<DB::Float32>();
|
||||
ifLeftType<DB::Float64>();
|
||||
}
|
||||
|
@ -801,15 +801,15 @@ readBinary(T & x, ReadBuffer & buf) { readPODBinary(x, buf); }
|
||||
inline void readBinary(String & x, ReadBuffer & buf) { readStringBinary(x, buf); }
|
||||
inline void readBinary(Int128 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
|
||||
inline void readBinary(UInt128 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
|
||||
inline void readBinary(UInt256 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
|
||||
inline void readBinary(DummyUInt256 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
|
||||
inline void readBinary(Decimal32 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
|
||||
inline void readBinary(Decimal64 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
|
||||
inline void readBinary(Decimal128 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
|
||||
inline void readBinary(Decimal256 & x, ReadBuffer & buf) { readBigIntBinary(x.value, buf); }
|
||||
inline void readBinary(LocalDate & x, ReadBuffer & buf) { readPODBinary(x, buf); }
|
||||
|
||||
inline void readBinary(bUInt256 & x, ReadBuffer & buf) { readBigIntBinary(x, buf); }
|
||||
inline void readBinary(bInt256 & x, ReadBuffer & buf) { readBigIntBinary(x, buf); }
|
||||
inline void readBinary(UInt256 & x, ReadBuffer & buf) { readBigIntBinary(x, buf); }
|
||||
inline void readBinary(Int256 & x, ReadBuffer & buf) { readBigIntBinary(x, buf); }
|
||||
|
||||
template <typename T>
|
||||
inline std::enable_if_t<is_arithmetic_v<T> && (sizeof(T) <= 8), void>
|
||||
@ -943,8 +943,8 @@ inline void readCSV(UUID & x, ReadBuffer & buf) { readCSVSimple(x, buf); }
|
||||
*/
|
||||
throw Exception("UInt128 cannot be read as a text", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
inline void readCSV(bUInt256 & x, ReadBuffer & buf) { readCSVSimple(x, buf); }
|
||||
inline void readCSV(bInt256 & x, ReadBuffer & buf) { readCSVSimple(x, buf); }
|
||||
inline void readCSV(UInt256 & x, ReadBuffer & buf) { readCSVSimple(x, buf); }
|
||||
inline void readCSV(Int256 & x, ReadBuffer & buf) { readCSVSimple(x, buf); }
|
||||
|
||||
template <typename T>
|
||||
void readBinary(std::vector<T> & x, ReadBuffer & buf)
|
||||
|
@ -801,7 +801,7 @@ inline void writeBinary(const String & x, WriteBuffer & buf) { writeStringBinary
|
||||
inline void writeBinary(const StringRef & x, WriteBuffer & buf) { writeStringBinary(x, buf); }
|
||||
inline void writeBinary(const Int128 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
|
||||
inline void writeBinary(const UInt128 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
|
||||
inline void writeBinary(const UInt256 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
|
||||
inline void writeBinary(const DummyUInt256 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
|
||||
inline void writeBinary(const Decimal32 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
|
||||
inline void writeBinary(const Decimal64 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
|
||||
inline void writeBinary(const Decimal128 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
|
||||
@ -809,8 +809,8 @@ inline void writeBinary(const Decimal256 & x, WriteBuffer & buf) { writeBigIntBi
|
||||
inline void writeBinary(const LocalDate & x, WriteBuffer & buf) { writePODBinary(x, buf); }
|
||||
inline void writeBinary(const LocalDateTime & x, WriteBuffer & buf) { writePODBinary(x, buf); }
|
||||
|
||||
inline void writeBinary(const bUInt256 & x, WriteBuffer & buf) { writeBigIntBinary(x, buf); }
|
||||
inline void writeBinary(const bInt256 & x, WriteBuffer & buf) { writeBigIntBinary(x, buf); }
|
||||
inline void writeBinary(const UInt256 & x, WriteBuffer & buf) { writeBigIntBinary(x, buf); }
|
||||
inline void writeBinary(const Int256 & x, WriteBuffer & buf) { writeBigIntBinary(x, buf); }
|
||||
|
||||
/// Methods for outputting the value in text form for a tab-separated format.
|
||||
template <typename T>
|
||||
@ -835,13 +835,13 @@ inline void writeText(const LocalDate & x, WriteBuffer & buf) { writeDateText(x,
|
||||
inline void writeText(const LocalDateTime & x, WriteBuffer & buf) { writeDateTimeText(x, buf); }
|
||||
inline void writeText(const UUID & x, WriteBuffer & buf) { writeUUIDText(x, buf); }
|
||||
inline void writeText(const UInt128 & x, WriteBuffer & buf) { writeText(UUID(x), buf); }
|
||||
inline void writeText(const bUInt256 & x, WriteBuffer & buf) { writeText(x.str(), buf); }
|
||||
inline void writeText(const bInt256 & x, WriteBuffer & buf) { writeText(x.str(), buf); }
|
||||
inline void writeText(const UInt256 & x, WriteBuffer & buf) { writeText(bigintToString(x), buf); }
|
||||
inline void writeText(const Int256 & x, WriteBuffer & buf) { writeText(bigintToString(x), buf); }
|
||||
|
||||
template <typename T>
|
||||
String decimalFractional(const T & x, UInt32 scale)
|
||||
{
|
||||
if constexpr (std::is_same_v<T, bInt256>)
|
||||
if constexpr (std::is_same_v<T, Int256>)
|
||||
{
|
||||
static constexpr Int128 max_int128 = (Int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll;
|
||||
|
||||
@ -877,7 +877,7 @@ void writeText(Decimal<T> x, UInt32 scale, WriteBuffer & ostr)
|
||||
writeChar('-', ostr); /// avoid crop leading minus when whole part is zero
|
||||
}
|
||||
|
||||
if constexpr (std::is_same_v<T, bInt256>)
|
||||
if constexpr (std::is_same_v<T, Int256>)
|
||||
writeText(part, ostr);
|
||||
else
|
||||
writeIntText(part, ostr);
|
||||
@ -919,14 +919,14 @@ inline void writeQuoted(const UUID & x, WriteBuffer & buf)
|
||||
writeChar('\'', buf);
|
||||
}
|
||||
|
||||
inline void writeQuoted(const bUInt256 & x, WriteBuffer & buf)
|
||||
inline void writeQuoted(const UInt256 & x, WriteBuffer & buf)
|
||||
{
|
||||
writeChar('\'', buf);
|
||||
writeText(x, buf);
|
||||
writeChar('\'', buf);
|
||||
}
|
||||
|
||||
inline void writeQuoted(const bInt256 & x, WriteBuffer & buf)
|
||||
inline void writeQuoted(const Int256 & x, WriteBuffer & buf)
|
||||
{
|
||||
writeChar('\'', buf);
|
||||
writeText(x, buf);
|
||||
|
@ -304,7 +304,7 @@ AggregatedDataVariants::Type Aggregator::chooseAggregationMethod()
|
||||
/// into a fixed 16- or 32-byte blob.
|
||||
if (std::tuple_size<KeysNullMap<UInt128>>::value + keys_bytes <= 16)
|
||||
return AggregatedDataVariants::Type::nullable_keys128;
|
||||
if (std::tuple_size<KeysNullMap<UInt256>>::value + keys_bytes <= 32)
|
||||
if (std::tuple_size<KeysNullMap<DummyUInt256>>::value + keys_bytes <= 32)
|
||||
return AggregatedDataVariants::Type::nullable_keys256;
|
||||
}
|
||||
|
||||
|
@ -79,7 +79,7 @@ using AggregatedDataWithShortStringKey = StringHashMap<AggregateDataPtr>;
|
||||
using AggregatedDataWithStringKey = HashMapWithSavedHash<StringRef, AggregateDataPtr>;
|
||||
|
||||
using AggregatedDataWithKeys128 = HashMap<UInt128, AggregateDataPtr, UInt128HashCRC32>;
|
||||
using AggregatedDataWithKeys256 = HashMap<UInt256, AggregateDataPtr, UInt256HashCRC32>;
|
||||
using AggregatedDataWithKeys256 = HashMap<DummyUInt256, AggregateDataPtr, UInt256HashCRC32>;
|
||||
|
||||
using AggregatedDataWithUInt32KeyTwoLevel = TwoLevelHashMap<UInt32, AggregateDataPtr, HashCRC32<UInt32>>;
|
||||
using AggregatedDataWithUInt64KeyTwoLevel = TwoLevelHashMap<UInt64, AggregateDataPtr, HashCRC32<UInt64>>;
|
||||
@ -89,7 +89,7 @@ using AggregatedDataWithShortStringKeyTwoLevel = TwoLevelStringHashMap<Aggregate
|
||||
using AggregatedDataWithStringKeyTwoLevel = TwoLevelHashMapWithSavedHash<StringRef, AggregateDataPtr>;
|
||||
|
||||
using AggregatedDataWithKeys128TwoLevel = TwoLevelHashMap<UInt128, AggregateDataPtr, UInt128HashCRC32>;
|
||||
using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap<UInt256, AggregateDataPtr, UInt256HashCRC32>;
|
||||
using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap<DummyUInt256, AggregateDataPtr, UInt256HashCRC32>;
|
||||
|
||||
/** Variants with better hash function, using more than 32 bits for hash.
|
||||
* Using for merging phase of external aggregation, where number of keys may be far greater than 4 billion,
|
||||
@ -101,7 +101,7 @@ using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap<UInt256, AggregateData
|
||||
using AggregatedDataWithUInt64KeyHash64 = HashMap<UInt64, AggregateDataPtr, DefaultHash<UInt64>>;
|
||||
using AggregatedDataWithStringKeyHash64 = HashMapWithSavedHash<StringRef, AggregateDataPtr, StringRefHash64>;
|
||||
using AggregatedDataWithKeys128Hash64 = HashMap<UInt128, AggregateDataPtr, UInt128Hash>;
|
||||
using AggregatedDataWithKeys256Hash64 = HashMap<UInt256, AggregateDataPtr, UInt256Hash>;
|
||||
using AggregatedDataWithKeys256Hash64 = HashMap<DummyUInt256, AggregateDataPtr, UInt256Hash>;
|
||||
|
||||
template <typename Base>
|
||||
struct AggregationDataWithNullKey : public Base
|
||||
|
@ -327,7 +327,7 @@ template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<HashJoin:
|
||||
};
|
||||
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<HashJoin::Type::keys256, Value, Mapped>
|
||||
{
|
||||
using Type = ColumnsHashing::HashMethodKeysFixed<Value, UInt256, Mapped, false, false, false>;
|
||||
using Type = ColumnsHashing::HashMethodKeysFixed<Value, DummyUInt256, Mapped, false, false, false>;
|
||||
};
|
||||
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<HashJoin::Type::hashed, Value, Mapped>
|
||||
{
|
||||
|
@ -245,7 +245,7 @@ public:
|
||||
std::unique_ptr<HashMapWithSavedHash<StringRef, Mapped>> key_string;
|
||||
std::unique_ptr<HashMapWithSavedHash<StringRef, Mapped>> key_fixed_string;
|
||||
std::unique_ptr<HashMap<UInt128, Mapped, UInt128HashCRC32>> keys128;
|
||||
std::unique_ptr<HashMap<UInt256, Mapped, UInt256HashCRC32>> keys256;
|
||||
std::unique_ptr<HashMap<DummyUInt256, Mapped, UInt256HashCRC32>> keys256;
|
||||
std::unique_ptr<HashMap<UInt128, Mapped, UInt128TrivialHash>> hashed;
|
||||
|
||||
void create(Type which)
|
||||
|
@ -18,7 +18,6 @@
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
#include <Parsers/ASTIndexDeclaration.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <Parsers/ASTNameTypePair.h>
|
||||
#include <Parsers/ASTInsertQuery.h>
|
||||
#include <Parsers/ParserCreateQuery.h>
|
||||
#include <Parsers/formatAST.h>
|
||||
@ -30,11 +29,9 @@
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Interpreters/DDLWorker.h>
|
||||
#include <Interpreters/ExpressionAnalyzer.h>
|
||||
#include <Interpreters/TreeRewriter.h>
|
||||
#include <Interpreters/InterpreterCreateQuery.h>
|
||||
#include <Interpreters/InterpreterSelectWithUnionQuery.h>
|
||||
#include <Interpreters/InterpreterInsertQuery.h>
|
||||
#include <Interpreters/ExpressionActions.h>
|
||||
#include <Interpreters/AddDefaultDatabaseVisitor.h>
|
||||
|
||||
#include <Access/AccessRightsElement.h>
|
||||
@ -70,6 +67,7 @@ namespace ErrorCodes
|
||||
extern const int UNKNOWN_DATABASE_ENGINE;
|
||||
extern const int DUPLICATE_COLUMN;
|
||||
extern const int DATABASE_ALREADY_EXISTS;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
extern const int BAD_DATABASE_FOR_TEMPORARY_TABLE;
|
||||
extern const int SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY;
|
||||
extern const int DICTIONARY_ALREADY_EXISTS;
|
||||
@ -415,7 +413,12 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(
|
||||
column.comment = col_decl.comment->as<ASTLiteral &>().value.get<String>();
|
||||
|
||||
if (col_decl.codec)
|
||||
column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(col_decl.codec, column.type, sanity_check_compression_codecs);
|
||||
{
|
||||
if (col_decl.default_specifier == "ALIAS")
|
||||
throw Exception{"Cannot specify codec for column type ALIAS", ErrorCodes::BAD_ARGUMENTS};
|
||||
column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(
|
||||
col_decl.codec, column.type, sanity_check_compression_codecs);
|
||||
}
|
||||
|
||||
if (col_decl.ttl)
|
||||
column.ttl = col_decl.ttl;
|
||||
|
@ -1,5 +1,3 @@
|
||||
#include "MutationsInterpreter.h"
|
||||
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Interpreters/InDepthNodeVisitor.h>
|
||||
@ -7,11 +5,12 @@
|
||||
#include <Interpreters/MutationsInterpreter.h>
|
||||
#include <Interpreters/TreeRewriter.h>
|
||||
#include <Storages/MergeTree/MergeTreeData.h>
|
||||
#include <DataStreams/FilterBlockInputStream.h>
|
||||
#include <DataStreams/ExpressionBlockInputStream.h>
|
||||
#include <DataStreams/CreatingSetsBlockInputStream.h>
|
||||
#include <DataStreams/MaterializingBlockInputStream.h>
|
||||
#include <DataStreams/NullBlockInputStream.h>
|
||||
#include <Processors/Transforms/FilterTransform.h>
|
||||
#include <Processors/Transforms/ExpressionTransform.h>
|
||||
#include <Processors/Transforms/CreatingSetsTransform.h>
|
||||
#include <Processors/Transforms/MaterializingTransform.h>
|
||||
#include <Processors/Sources/NullSource.h>
|
||||
#include <Processors/QueryPipeline.h>
|
||||
#include <DataStreams/CheckSortedBlockInputStream.h>
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
#include <Parsers/ASTFunction.h>
|
||||
@ -160,7 +159,7 @@ ColumnDependencies getAllColumnDependencies(const StorageMetadataPtr & metadata_
|
||||
return dependencies;
|
||||
}
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
bool isStorageTouchedByMutations(
|
||||
StoragePtr storage,
|
||||
@ -525,8 +524,10 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run)
|
||||
SelectQueryOptions().analyze(/* dry_run = */ false).ignoreLimits()};
|
||||
|
||||
auto first_stage_header = interpreter.getSampleBlock();
|
||||
auto in = std::make_shared<NullBlockInputStream>(first_stage_header);
|
||||
updated_header = std::make_unique<Block>(addStreamsForLaterStages(stages_copy, in)->getHeader());
|
||||
QueryPipeline pipeline;
|
||||
pipeline.init(Pipe(std::make_shared<NullSource>(first_stage_header)));
|
||||
addStreamsForLaterStages(stages_copy, pipeline);
|
||||
updated_header = std::make_unique<Block>(pipeline.getHeader());
|
||||
}
|
||||
|
||||
/// Special step to recalculate affected indices and TTL expressions.
|
||||
@ -655,7 +656,7 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector<Stage> &
|
||||
return select;
|
||||
}
|
||||
|
||||
BlockInputStreamPtr MutationsInterpreter::addStreamsForLaterStages(const std::vector<Stage> & prepared_stages, BlockInputStreamPtr in) const
|
||||
void MutationsInterpreter::addStreamsForLaterStages(const std::vector<Stage> & prepared_stages, QueryPipeline & pipeline) const
|
||||
{
|
||||
for (size_t i_stage = 1; i_stage < prepared_stages.size(); ++i_stage)
|
||||
{
|
||||
@ -667,23 +668,36 @@ BlockInputStreamPtr MutationsInterpreter::addStreamsForLaterStages(const std::ve
|
||||
if (i < stage.filter_column_names.size())
|
||||
{
|
||||
/// Execute DELETEs.
|
||||
in = std::make_shared<FilterBlockInputStream>(in, step->actions(), stage.filter_column_names[i]);
|
||||
pipeline.addSimpleTransform([&](const Block & header)
|
||||
{
|
||||
return std::make_shared<FilterTransform>(header, step->actions(), stage.filter_column_names[i], false);
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
/// Execute UPDATE or final projection.
|
||||
in = std::make_shared<ExpressionBlockInputStream>(in, step->actions());
|
||||
pipeline.addSimpleTransform([&](const Block & header)
|
||||
{
|
||||
return std::make_shared<ExpressionTransform>(header, step->actions());
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const SubqueriesForSets & subqueries_for_sets = stage.analyzer->getSubqueriesForSets();
|
||||
if (!subqueries_for_sets.empty())
|
||||
in = std::make_shared<CreatingSetsBlockInputStream>(in, subqueries_for_sets, context);
|
||||
{
|
||||
const Settings & settings = context.getSettingsRef();
|
||||
SizeLimits network_transfer_limits(
|
||||
settings.max_rows_to_transfer, settings.max_bytes_to_transfer, settings.transfer_overflow_mode);
|
||||
pipeline.addCreatingSetsTransform(std::make_shared<CreatingSetsTransform>(
|
||||
pipeline.getHeader(), subqueries_for_sets, network_transfer_limits, context));
|
||||
}
|
||||
}
|
||||
|
||||
in = std::make_shared<MaterializingBlockInputStream>(in);
|
||||
|
||||
return in;
|
||||
pipeline.addSimpleTransform([&](const Block & header)
|
||||
{
|
||||
return std::make_shared<MaterializingTransform>(header);
|
||||
});
|
||||
}
|
||||
|
||||
void MutationsInterpreter::validate()
|
||||
@ -705,10 +719,8 @@ void MutationsInterpreter::validate()
|
||||
}
|
||||
}
|
||||
|
||||
/// Do not use getSampleBlock in order to check the whole pipeline.
|
||||
Block first_stage_header = select_interpreter->execute().getInputStream()->getHeader();
|
||||
BlockInputStreamPtr in = std::make_shared<NullBlockInputStream>(first_stage_header);
|
||||
addStreamsForLaterStages(stages, in)->getHeader();
|
||||
auto block_io = select_interpreter->execute();
|
||||
addStreamsForLaterStages(stages, block_io.pipeline);
|
||||
}
|
||||
|
||||
BlockInputStreamPtr MutationsInterpreter::execute()
|
||||
@ -716,9 +728,10 @@ BlockInputStreamPtr MutationsInterpreter::execute()
|
||||
if (!can_execute)
|
||||
throw Exception("Cannot execute mutations interpreter because can_execute flag set to false", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
BlockInputStreamPtr in = select_interpreter->execute().getInputStream();
|
||||
auto block_io = select_interpreter->execute();
|
||||
addStreamsForLaterStages(stages, block_io.pipeline);
|
||||
|
||||
auto result_stream = addStreamsForLaterStages(stages, in);
|
||||
auto result_stream = block_io.getInputStream();
|
||||
|
||||
/// Sometimes we update just part of columns (for example UPDATE mutation)
|
||||
/// in this case we don't read sorting key, so just we don't check anything.
|
||||
|
@ -13,6 +13,7 @@ namespace DB
|
||||
{
|
||||
|
||||
class Context;
|
||||
class QueryPipeline;
|
||||
|
||||
/// Return false if the data isn't going to be changed by mutations.
|
||||
bool isStorageTouchedByMutations(
|
||||
@ -51,7 +52,7 @@ private:
|
||||
struct Stage;
|
||||
|
||||
ASTPtr prepareInterpreterSelectQuery(std::vector<Stage> &prepared_stages, bool dry_run);
|
||||
BlockInputStreamPtr addStreamsForLaterStages(const std::vector<Stage> & prepared_stages, BlockInputStreamPtr in) const;
|
||||
void addStreamsForLaterStages(const std::vector<Stage> & prepared_stages, QueryPipeline & pipeline) const;
|
||||
|
||||
std::optional<SortDescription> getStorageSortDescriptionIfPossible(const Block & header) const;
|
||||
|
||||
|
@ -123,7 +123,7 @@ typename SetVariantsTemplate<Variant>::Type SetVariantsTemplate<Variant>::choose
|
||||
throw Exception{"Aggregator: keys sizes overflow", ErrorCodes::LOGICAL_ERROR};
|
||||
if ((std::tuple_size<KeysNullMap<UInt128>>::value + keys_bytes) <= 16)
|
||||
return Type::nullable_keys128;
|
||||
if ((std::tuple_size<KeysNullMap<UInt256>>::value + keys_bytes) <= 32)
|
||||
if ((std::tuple_size<KeysNullMap<DummyUInt256>>::value + keys_bytes) <= 32)
|
||||
return Type::nullable_keys256;
|
||||
}
|
||||
|
||||
|
@ -202,12 +202,12 @@ struct NonClearableSet
|
||||
std::unique_ptr<SetMethodString<HashSetWithSavedHash<StringRef>>> key_string;
|
||||
std::unique_ptr<SetMethodFixedString<HashSetWithSavedHash<StringRef>>> key_fixed_string;
|
||||
std::unique_ptr<SetMethodKeysFixed<HashSet<UInt128, UInt128HashCRC32>>> keys128;
|
||||
std::unique_ptr<SetMethodKeysFixed<HashSet<UInt256, UInt256HashCRC32>>> keys256;
|
||||
std::unique_ptr<SetMethodKeysFixed<HashSet<DummyUInt256, UInt256HashCRC32>>> keys256;
|
||||
std::unique_ptr<SetMethodHashed<HashSet<UInt128, UInt128TrivialHash>>> hashed;
|
||||
|
||||
/// Support for nullable keys (for DISTINCT implementation).
|
||||
std::unique_ptr<SetMethodKeysFixed<HashSet<UInt128, UInt128HashCRC32>, true>> nullable_keys128;
|
||||
std::unique_ptr<SetMethodKeysFixed<HashSet<UInt256, UInt256HashCRC32>, true>> nullable_keys256;
|
||||
std::unique_ptr<SetMethodKeysFixed<HashSet<DummyUInt256, UInt256HashCRC32>, true>> nullable_keys256;
|
||||
/** Unlike Aggregator, `concat` method is not used here.
|
||||
* This is done because `hashed` method, although slower, but in this case, uses less RAM.
|
||||
* since when you use it, the key values themselves are not stored.
|
||||
@ -224,12 +224,12 @@ struct ClearableSet
|
||||
std::unique_ptr<SetMethodString<ClearableHashSetWithSavedHash<StringRef>>> key_string;
|
||||
std::unique_ptr<SetMethodFixedString<ClearableHashSetWithSavedHash<StringRef>>> key_fixed_string;
|
||||
std::unique_ptr<SetMethodKeysFixed<ClearableHashSet<UInt128, UInt128HashCRC32>>> keys128;
|
||||
std::unique_ptr<SetMethodKeysFixed<ClearableHashSet<UInt256, UInt256HashCRC32>>> keys256;
|
||||
std::unique_ptr<SetMethodKeysFixed<ClearableHashSet<DummyUInt256, UInt256HashCRC32>>> keys256;
|
||||
std::unique_ptr<SetMethodHashed<ClearableHashSet<UInt128, UInt128TrivialHash>>> hashed;
|
||||
|
||||
/// Support for nullable keys (for DISTINCT implementation).
|
||||
std::unique_ptr<SetMethodKeysFixed<ClearableHashSet<UInt128, UInt128HashCRC32>, true>> nullable_keys128;
|
||||
std::unique_ptr<SetMethodKeysFixed<ClearableHashSet<UInt256, UInt256HashCRC32>, true>> nullable_keys256;
|
||||
std::unique_ptr<SetMethodKeysFixed<ClearableHashSet<DummyUInt256, UInt256HashCRC32>, true>> nullable_keys256;
|
||||
/** Unlike Aggregator, `concat` method is not used here.
|
||||
* This is done because `hashed` method, although slower, but in this case, uses less RAM.
|
||||
* since when you use it, the key values themselves are not stored.
|
||||
|
@ -469,7 +469,16 @@ void PipelineExecutor::wakeUpExecutor(size_t thread_num)
|
||||
|
||||
void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads)
|
||||
{
|
||||
executeStepImpl(thread_num, num_threads);
|
||||
try
|
||||
{
|
||||
executeStepImpl(thread_num, num_threads);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
/// In case of exception from executor itself, stop other threads.
|
||||
finish();
|
||||
throw;
|
||||
}
|
||||
|
||||
#ifndef NDEBUG
|
||||
auto & context = executor_contexts[thread_num];
|
||||
|
@ -74,8 +74,11 @@ std::optional<AlterCommand> AlterCommand::parse(const ASTAlterCommand * command_
|
||||
}
|
||||
|
||||
if (ast_col_decl.codec)
|
||||
{
|
||||
if (ast_col_decl.default_specifier == "ALIAS")
|
||||
throw Exception{"Cannot specify codec for column type ALIAS", ErrorCodes::BAD_ARGUMENTS};
|
||||
command.codec = ast_col_decl.codec;
|
||||
|
||||
}
|
||||
if (command_ast->column)
|
||||
command.after_column = getIdentifierName(command_ast->column);
|
||||
|
||||
|
@ -404,6 +404,13 @@ std::optional<ColumnDefault> ColumnsDescription::getDefault(const String & colum
|
||||
}
|
||||
|
||||
|
||||
bool ColumnsDescription::hasCompressionCodec(const String & column_name) const
|
||||
{
|
||||
const auto it = columns.get<1>().find(column_name);
|
||||
|
||||
return it != columns.get<1>().end() && it->codec != nullptr;
|
||||
}
|
||||
|
||||
CompressionCodecPtr ColumnsDescription::getCodecOrDefault(const String & column_name, CompressionCodecPtr default_codec) const
|
||||
{
|
||||
const auto it = columns.get<1>().find(column_name);
|
||||
|
@ -111,6 +111,8 @@ public:
|
||||
bool hasDefault(const String & column_name) const;
|
||||
std::optional<ColumnDefault> getDefault(const String & column_name) const;
|
||||
|
||||
/// Does column has non default specified compression codec
|
||||
bool hasCompressionCodec(const String & column_name) const;
|
||||
CompressionCodecPtr getCodecOrDefault(const String & column_name, CompressionCodecPtr default_codec) const;
|
||||
CompressionCodecPtr getCodecOrDefault(const String & column_name) const;
|
||||
|
||||
|
@ -43,6 +43,7 @@ namespace
|
||||
constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE = 1;
|
||||
constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE_AND_TTL_INFOS = 2;
|
||||
constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_TYPE = 3;
|
||||
constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_DEFAULT_COMPRESSION = 4;
|
||||
|
||||
|
||||
std::string getEndpointId(const std::string & node_id)
|
||||
@ -83,7 +84,7 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo
|
||||
}
|
||||
|
||||
/// We pretend to work as older server version, to be sure that client will correctly process our version
|
||||
response.addCookie({"server_protocol_version", toString(std::min(client_protocol_version, REPLICATION_PROTOCOL_VERSION_WITH_PARTS_TYPE))});
|
||||
response.addCookie({"server_protocol_version", toString(std::min(client_protocol_version, REPLICATION_PROTOCOL_VERSION_WITH_PARTS_DEFAULT_COMPRESSION))});
|
||||
|
||||
++total_sends;
|
||||
SCOPE_EXIT({--total_sends;});
|
||||
@ -115,7 +116,10 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo
|
||||
if (isInMemoryPart(part))
|
||||
sendPartFromMemory(part, out);
|
||||
else
|
||||
sendPartFromDisk(part, out);
|
||||
{
|
||||
bool send_default_compression_file = client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_DEFAULT_COMPRESSION;
|
||||
sendPartFromDisk(part, out, send_default_compression_file);
|
||||
}
|
||||
}
|
||||
catch (const NetException &)
|
||||
{
|
||||
@ -147,14 +151,20 @@ void Service::sendPartFromMemory(const MergeTreeData::DataPartPtr & part, WriteB
|
||||
block_out.write(part_in_memory->block);
|
||||
}
|
||||
|
||||
void Service::sendPartFromDisk(const MergeTreeData::DataPartPtr & part, WriteBuffer & out)
|
||||
void Service::sendPartFromDisk(const MergeTreeData::DataPartPtr & part, WriteBuffer & out, bool send_default_compression_file)
|
||||
{
|
||||
/// We'll take a list of files from the list of checksums.
|
||||
MergeTreeData::DataPart::Checksums checksums = part->checksums;
|
||||
/// Add files that are not in the checksum list.
|
||||
checksums.files["checksums.txt"] = {};
|
||||
checksums.files["columns.txt"] = {};
|
||||
auto file_names_without_checksums = part->getFileNamesWithoutChecksums();
|
||||
for (const auto & file_name : file_names_without_checksums)
|
||||
{
|
||||
if (!send_default_compression_file && file_name == IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME)
|
||||
continue;
|
||||
checksums.files[file_name] = {};
|
||||
}
|
||||
|
||||
auto disk = part->volume->getDisk();
|
||||
MergeTreeData::DataPart::Checksums data_checksums;
|
||||
|
||||
writeBinary(checksums.files.size(), out);
|
||||
@ -162,7 +172,6 @@ void Service::sendPartFromDisk(const MergeTreeData::DataPartPtr & part, WriteBuf
|
||||
{
|
||||
String file_name = it.first;
|
||||
|
||||
auto disk = part->volume->getDisk();
|
||||
String path = part->getFullRelativePath() + file_name;
|
||||
|
||||
UInt64 size = disk->getFileSize(path);
|
||||
@ -182,8 +191,7 @@ void Service::sendPartFromDisk(const MergeTreeData::DataPartPtr & part, WriteBuf
|
||||
|
||||
writePODBinary(hashing_out.getHash(), out);
|
||||
|
||||
if (file_name != "checksums.txt" &&
|
||||
file_name != "columns.txt")
|
||||
if (!file_names_without_checksums.count(file_name))
|
||||
data_checksums.addFile(file_name, hashing_out.count(), hashing_out.getHash());
|
||||
}
|
||||
|
||||
@ -230,7 +238,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart(
|
||||
{
|
||||
{"endpoint", getEndpointId(replica_path)},
|
||||
{"part", part_name},
|
||||
{"client_protocol_version", toString(REPLICATION_PROTOCOL_VERSION_WITH_PARTS_TYPE)},
|
||||
{"client_protocol_version", toString(REPLICATION_PROTOCOL_VERSION_WITH_PARTS_DEFAULT_COMPRESSION)},
|
||||
{"compress", "false"}
|
||||
});
|
||||
|
||||
@ -308,7 +316,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToMemory(
|
||||
new_data_part->minmax_idx.update(block, data.minmax_idx_columns);
|
||||
new_data_part->partition.create(metadata_snapshot, block, 0);
|
||||
|
||||
MergedBlockOutputStream part_out(new_data_part, metadata_snapshot, block.getNamesAndTypesList(), {}, nullptr);
|
||||
MergedBlockOutputStream part_out(new_data_part, metadata_snapshot, block.getNamesAndTypesList(), {}, CompressionCodecFactory::instance().get("NONE", {}));
|
||||
part_out.writePrefix();
|
||||
part_out.write(block);
|
||||
part_out.writeSuffixAndFinalizePart(new_data_part);
|
||||
@ -381,7 +389,8 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk(
|
||||
ErrorCodes::CHECKSUM_DOESNT_MATCH);
|
||||
|
||||
if (file_name != "checksums.txt" &&
|
||||
file_name != "columns.txt")
|
||||
file_name != "columns.txt" &&
|
||||
file_name != IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME)
|
||||
checksums.addFile(file_name, file_size, expected_hash);
|
||||
}
|
||||
|
||||
|
@ -32,7 +32,7 @@ public:
|
||||
private:
|
||||
MergeTreeData::DataPartPtr findPart(const String & name);
|
||||
void sendPartFromMemory(const MergeTreeData::DataPartPtr & part, WriteBuffer & out);
|
||||
void sendPartFromDisk(const MergeTreeData::DataPartPtr & part, WriteBuffer & out);
|
||||
void sendPartFromDisk(const MergeTreeData::DataPartPtr & part, WriteBuffer & out, bool send_default_compression_file);
|
||||
|
||||
private:
|
||||
/// StorageReplicatedMergeTree::shutdown() waits for all parts exchange handlers to finish,
|
||||
|
@ -13,6 +13,8 @@
|
||||
#include <Common/escapeForFileName.h>
|
||||
#include <common/JSON.h>
|
||||
#include <common/logger_useful.h>
|
||||
#include <Compression/getCompressionCodecForFile.h>
|
||||
#include <Parsers/queryToString.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -31,10 +33,6 @@ namespace ErrorCodes
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
|
||||
extern const char * DELETE_ON_DESTROY_MARKER_PATH;
|
||||
|
||||
|
||||
static std::unique_ptr<ReadBufferFromFileBase> openForReading(const DiskPtr & disk, const String & path)
|
||||
{
|
||||
return disk->readFile(path, std::min(size_t(DBMS_DEFAULT_BUFFER_SIZE), disk->getFileSize(path)));
|
||||
@ -418,9 +416,10 @@ void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checks
|
||||
loadRowsCount(); /// Must be called after loadIndexGranularity() as it uses the value of `index_granularity`.
|
||||
loadPartitionAndMinMaxIndex();
|
||||
loadTTLInfos();
|
||||
|
||||
if (check_consistency)
|
||||
checkConsistency(require_columns_checksums);
|
||||
loadDefaultCompressionCodec();
|
||||
|
||||
}
|
||||
|
||||
void IMergeTreeDataPart::loadIndexGranularity()
|
||||
@ -474,6 +473,89 @@ void IMergeTreeDataPart::loadIndex()
|
||||
}
|
||||
}
|
||||
|
||||
NameSet IMergeTreeDataPart::getFileNamesWithoutChecksums() const
|
||||
{
|
||||
if (!isStoredOnDisk())
|
||||
return {};
|
||||
|
||||
NameSet result = {"checksums.txt", "columns.txt"};
|
||||
String default_codec_path = getFullRelativePath() + DEFAULT_COMPRESSION_CODEC_FILE_NAME;
|
||||
if (volume->getDisk()->exists(default_codec_path))
|
||||
result.emplace(DEFAULT_COMPRESSION_CODEC_FILE_NAME);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
void IMergeTreeDataPart::loadDefaultCompressionCodec()
|
||||
{
|
||||
/// In memory parts doesn't have any compression
|
||||
if (!isStoredOnDisk())
|
||||
{
|
||||
default_codec = CompressionCodecFactory::instance().get("NONE", {});
|
||||
return;
|
||||
}
|
||||
|
||||
String path = getFullRelativePath() + DEFAULT_COMPRESSION_CODEC_FILE_NAME;
|
||||
if (!volume->getDisk()->exists(path))
|
||||
{
|
||||
default_codec = detectDefaultCompressionCodec();
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
auto file_buf = openForReading(volume->getDisk(), path);
|
||||
String codec_line;
|
||||
readEscapedStringUntilEOL(codec_line, *file_buf);
|
||||
|
||||
ReadBufferFromString buf(codec_line);
|
||||
|
||||
if (!checkString("CODEC", buf))
|
||||
{
|
||||
LOG_WARNING(storage.log, "Cannot parse default codec for part {} from file {}, content '{}'. Default compression codec will be deduced automatically, from data on disk", name, path, codec_line);
|
||||
default_codec = detectDefaultCompressionCodec();
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
ParserCodec codec_parser;
|
||||
auto codec_ast = parseQuery(codec_parser, codec_line.data() + buf.getPosition(), codec_line.data() + codec_line.length(), "codec parser", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
|
||||
default_codec = CompressionCodecFactory::instance().get(codec_ast, {});
|
||||
}
|
||||
catch (const DB::Exception & ex)
|
||||
{
|
||||
LOG_WARNING(storage.log, "Cannot parse default codec for part {} from file {}, content '{}', error '{}'. Default compression codec will be deduced automatically, from data on disk.", name, path, codec_line, ex.what());
|
||||
default_codec = detectDefaultCompressionCodec();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
CompressionCodecPtr IMergeTreeDataPart::detectDefaultCompressionCodec() const
|
||||
{
|
||||
/// In memory parts doesn't have any compression
|
||||
if (!isStoredOnDisk())
|
||||
return CompressionCodecFactory::instance().get("NONE", {});
|
||||
|
||||
auto metadata_snapshot = storage.getInMemoryMetadataPtr();
|
||||
|
||||
const auto & storage_columns = metadata_snapshot->getColumns();
|
||||
CompressionCodecPtr result = nullptr;
|
||||
for (const auto & part_column : columns)
|
||||
{
|
||||
/// It was compressed with default codec and it's not empty
|
||||
auto column_size = getColumnSize(part_column.name, *part_column.type);
|
||||
if (column_size.data_compressed != 0 && !storage_columns.hasCompressionCodec(part_column.name))
|
||||
{
|
||||
result = getCompressionCodecForFile(volume->getDisk(), getFullRelativePath() + getFileNameForColumn(part_column) + ".bin");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!result)
|
||||
result = CompressionCodecFactory::instance().getDefaultCodec();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
void IMergeTreeDataPart::loadPartitionAndMinMaxIndex()
|
||||
{
|
||||
if (storage.format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
|
||||
@ -795,7 +877,9 @@ void IMergeTreeDataPart::remove() const
|
||||
|
||||
for (const auto & file : {"checksums.txt", "columns.txt"})
|
||||
volume->getDisk()->remove(to + "/" + file);
|
||||
volume->getDisk()->removeIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_PATH);
|
||||
|
||||
volume->getDisk()->removeIfExists(to + "/" + DEFAULT_COMPRESSION_CODEC_FILE_NAME);
|
||||
volume->getDisk()->removeIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME);
|
||||
|
||||
volume->getDisk()->remove(to);
|
||||
}
|
||||
@ -850,7 +934,7 @@ void IMergeTreeDataPart::makeCloneInDetached(const String & prefix, const Storag
|
||||
|
||||
/// Backup is not recursive (max_level is 0), so do not copy inner directories
|
||||
localBackup(volume->getDisk(), getFullRelativePath(), destination_path, 0);
|
||||
volume->getDisk()->removeIfExists(destination_path + "/" + DELETE_ON_DESTROY_MARKER_PATH);
|
||||
volume->getDisk()->removeIfExists(destination_path + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME);
|
||||
}
|
||||
|
||||
void IMergeTreeDataPart::makeCloneOnDiskDetached(const ReservationPtr & reservation) const
|
||||
@ -867,7 +951,7 @@ void IMergeTreeDataPart::makeCloneOnDiskDetached(const ReservationPtr & reservat
|
||||
reserved_disk->createDirectory(path_to_clone);
|
||||
|
||||
volume->getDisk()->copy(getFullRelativePath(), reserved_disk, path_to_clone);
|
||||
volume->getDisk()->removeIfExists(path_to_clone + "/" + DELETE_ON_DESTROY_MARKER_PATH);
|
||||
volume->getDisk()->removeIfExists(path_to_clone + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME);
|
||||
}
|
||||
|
||||
void IMergeTreeDataPart::checkConsistencyBase() const
|
||||
|
@ -288,6 +288,8 @@ public:
|
||||
/// Columns with values, that all have been zeroed by expired ttl
|
||||
NameSet expired_columns;
|
||||
|
||||
CompressionCodecPtr default_codec;
|
||||
|
||||
/// For data in RAM ('index')
|
||||
UInt64 getIndexSizeInBytes() const;
|
||||
UInt64 getIndexSizeInAllocatedBytes() const;
|
||||
@ -330,7 +332,20 @@ public:
|
||||
|
||||
String getRelativePathForPrefix(const String & prefix) const;
|
||||
|
||||
|
||||
/// Return set of metadat file names without checksums. For example,
|
||||
/// columns.txt or checksums.txt itself.
|
||||
NameSet getFileNamesWithoutChecksums() const;
|
||||
|
||||
/// File with compression codec name which was used to compress part columns
|
||||
/// by default. Some columns may have their own compression codecs, but
|
||||
/// default will be stored in this file.
|
||||
static inline constexpr auto DEFAULT_COMPRESSION_CODEC_FILE_NAME = "default_compression_codec.txt";
|
||||
|
||||
static inline constexpr auto DELETE_ON_DESTROY_MARKER_FILE_NAME = "delete-on-destroy.txt";
|
||||
|
||||
protected:
|
||||
|
||||
/// Total size of all columns, calculated once in calcuateColumnSizesOnDisk
|
||||
ColumnSize total_columns_size;
|
||||
|
||||
@ -380,6 +395,15 @@ private:
|
||||
void loadTTLInfos();
|
||||
|
||||
void loadPartitionAndMinMaxIndex();
|
||||
|
||||
/// Load default compression codec from file default_compression_codec.txt
|
||||
/// if it not exists tries to deduce codec from compressed column without
|
||||
/// any specifial compression.
|
||||
void loadDefaultCompressionCodec();
|
||||
|
||||
/// Found column without specific compression and return codec
|
||||
/// for this column with default parameters.
|
||||
CompressionCodecPtr detectDefaultCompressionCodec() const;
|
||||
};
|
||||
|
||||
using MergeTreeDataPartState = IMergeTreeDataPart::State;
|
||||
|
@ -113,9 +113,6 @@ namespace ErrorCodes
|
||||
}
|
||||
|
||||
|
||||
const char * DELETE_ON_DESTROY_MARKER_PATH = "delete-on-destroy.txt";
|
||||
|
||||
|
||||
static void checkSampleExpression(const StorageInMemoryMetadata & metadata, bool allow_sampling_expression_not_in_primary_key)
|
||||
{
|
||||
const auto & pk_sample_block = metadata.getPrimaryKey().sample_block;
|
||||
@ -781,7 +778,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
|
||||
bool broken = false;
|
||||
|
||||
String part_path = relative_data_path + "/" + part_name;
|
||||
String marker_path = part_path + "/" + DELETE_ON_DESTROY_MARKER_PATH;
|
||||
String marker_path = part_path + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME;
|
||||
if (part_disk_ptr->exists(marker_path))
|
||||
{
|
||||
LOG_WARNING(log, "Detaching stale part {}{}, which should have been deleted after a move. That can only happen after unclean restart of ClickHouse after move of a part having an operation blocking that stale copy of part.", getFullPathOnDisk(part_disk_ptr), part_name);
|
||||
@ -908,6 +905,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
|
||||
for (auto & part : broken_parts_to_detach)
|
||||
part->renameToDetached("");
|
||||
|
||||
|
||||
/// Delete from the set of current parts those parts that are covered by another part (those parts that
|
||||
/// were merged), but that for some reason are still not deleted from the filesystem.
|
||||
/// Deletion of files will be performed later in the clearOldParts() method.
|
||||
@ -958,6 +956,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
|
||||
|
||||
calculateColumnSizesImpl();
|
||||
|
||||
|
||||
LOG_DEBUG(log, "Loaded data parts ({} items)", data_parts_indexes.size());
|
||||
}
|
||||
|
||||
@ -2370,7 +2369,7 @@ void MergeTreeData::swapActivePart(MergeTreeData::DataPartPtr part_copy)
|
||||
modifyPartState(part_it, DataPartState::Committed);
|
||||
|
||||
auto disk = original_active_part->volume->getDisk();
|
||||
String marker_path = original_active_part->getFullRelativePath() + DELETE_ON_DESTROY_MARKER_PATH;
|
||||
String marker_path = original_active_part->getFullRelativePath() + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME;
|
||||
try
|
||||
{
|
||||
disk->createFile(marker_path);
|
||||
@ -2441,14 +2440,6 @@ static void loadPartAndFixMetadataImpl(MergeTreeData::MutableDataPartPtr part)
|
||||
part->modification_time = disk->getLastModified(full_part_path).epochTime();
|
||||
}
|
||||
|
||||
MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartAndFixMetadata(const VolumePtr & volume, const String & relative_path) const
|
||||
{
|
||||
MutableDataPartPtr part = createPart(Poco::Path(relative_path).getFileName(), volume, relative_path);
|
||||
loadPartAndFixMetadataImpl(part);
|
||||
return part;
|
||||
}
|
||||
|
||||
|
||||
void MergeTreeData::calculateColumnSizesImpl()
|
||||
{
|
||||
column_sizes.clear();
|
||||
@ -2919,6 +2910,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const
|
||||
LOG_DEBUG(log, "Checking parts");
|
||||
MutableDataPartsVector loaded_parts;
|
||||
loaded_parts.reserve(renamed_parts.old_and_new_names.size());
|
||||
|
||||
for (const auto & part_names : renamed_parts.old_and_new_names)
|
||||
{
|
||||
LOG_DEBUG(log, "Checking part {}", part_names.second);
|
||||
@ -3286,7 +3278,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPartOnSameDisk(
|
||||
|
||||
LOG_DEBUG(log, "Cloning part {} to {}", fullPath(disk, src_part_path), fullPath(disk, dst_part_path));
|
||||
localBackup(disk, src_part_path, dst_part_path);
|
||||
disk->removeIfExists(dst_part_path + "/" + DELETE_ON_DESTROY_MARKER_PATH);
|
||||
disk->removeIfExists(dst_part_path + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME);
|
||||
|
||||
auto single_disk_volume = std::make_shared<SingleDiskVolume>(disk->getName(), disk);
|
||||
auto dst_data_part = createPart(dst_part_name, dst_part_info, single_disk_volume, tmp_dst_part_name);
|
||||
@ -3377,7 +3369,7 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher(MatcherFn m
|
||||
else
|
||||
localBackup(part->volume->getDisk(), part->getFullRelativePath(), backup_part_path);
|
||||
|
||||
part->volume->getDisk()->removeIfExists(backup_part_path + "/" + DELETE_ON_DESTROY_MARKER_PATH);
|
||||
part->volume->getDisk()->removeIfExists(backup_part_path + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME);
|
||||
|
||||
part->is_frozen.store(true, std::memory_order_relaxed);
|
||||
result.push_back(PartitionCommandResultInfo{
|
||||
|
@ -534,9 +534,6 @@ public:
|
||||
return DB::extractKeyExpressionList(node);
|
||||
}
|
||||
|
||||
/// Check that the part is not broken and calculate the checksums for it if they are not present.
|
||||
MutableDataPartPtr loadPartAndFixMetadata(const VolumePtr & volume, const String & relative_path) const;
|
||||
|
||||
/** Create local backup (snapshot) for parts with specified prefix.
|
||||
* Backup is created in directory clickhouse_dir/shadow/i/, where i - incremental number,
|
||||
* or if 'with_name' is specified - backup is created in directory with specified name.
|
||||
|
@ -29,6 +29,7 @@
|
||||
#include <Common/interpolate.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/escapeForFileName.h>
|
||||
#include <Parsers/queryToString.h>
|
||||
|
||||
#include <cmath>
|
||||
#include <ctime>
|
||||
@ -1122,7 +1123,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor
|
||||
/// We will modify only some of the columns. Other columns and key values can be copied as-is.
|
||||
auto indices_to_recalc = getIndicesToRecalculate(in, updated_header.getNamesAndTypesList(), metadata_snapshot, context);
|
||||
|
||||
NameSet files_to_skip = collectFilesToSkip(updated_header, indices_to_recalc, mrk_extension);
|
||||
NameSet files_to_skip = collectFilesToSkip(source_part, updated_header, indices_to_recalc, mrk_extension);
|
||||
NameToNameVector files_to_rename = collectFilesForRenames(source_part, for_file_renames, mrk_extension);
|
||||
|
||||
if (need_remove_expired_values)
|
||||
@ -1184,7 +1185,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor
|
||||
}
|
||||
}
|
||||
|
||||
finalizeMutatedPart(source_part, new_data_part, need_remove_expired_values);
|
||||
finalizeMutatedPart(source_part, new_data_part, need_remove_expired_values, compression_codec);
|
||||
}
|
||||
|
||||
return new_data_part;
|
||||
@ -1448,9 +1449,12 @@ NameToNameVector MergeTreeDataMergerMutator::collectFilesForRenames(
|
||||
}
|
||||
|
||||
NameSet MergeTreeDataMergerMutator::collectFilesToSkip(
|
||||
const Block & updated_header, const std::set<MergeTreeIndexPtr> & indices_to_recalc, const String & mrk_extension)
|
||||
const MergeTreeDataPartPtr & source_part,
|
||||
const Block & updated_header,
|
||||
const std::set<MergeTreeIndexPtr> & indices_to_recalc,
|
||||
const String & mrk_extension)
|
||||
{
|
||||
NameSet files_to_skip = {"checksums.txt", "columns.txt"};
|
||||
NameSet files_to_skip = source_part->getFileNamesWithoutChecksums();
|
||||
|
||||
/// Skip updated files
|
||||
for (const auto & entry : updated_header)
|
||||
@ -1737,7 +1741,8 @@ void MergeTreeDataMergerMutator::mutateSomePartColumns(
|
||||
void MergeTreeDataMergerMutator::finalizeMutatedPart(
|
||||
const MergeTreeDataPartPtr & source_part,
|
||||
MergeTreeData::MutableDataPartPtr new_data_part,
|
||||
bool need_remove_expired_values)
|
||||
bool need_remove_expired_values,
|
||||
const CompressionCodecPtr & codec)
|
||||
{
|
||||
auto disk = new_data_part->volume->getDisk();
|
||||
if (need_remove_expired_values)
|
||||
@ -1756,6 +1761,10 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart(
|
||||
new_data_part->checksums.write(*out_checksums);
|
||||
} /// close fd
|
||||
|
||||
{
|
||||
auto out = disk->writeFile(new_data_part->getFullRelativePath() + IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME, 4096);
|
||||
DB::writeText(queryToString(codec->getFullCodecDesc()), *out);
|
||||
}
|
||||
|
||||
{
|
||||
/// Write a file with a description of columns.
|
||||
@ -1770,6 +1779,7 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart(
|
||||
new_data_part->modification_time = time(nullptr);
|
||||
new_data_part->setBytesOnDisk(
|
||||
MergeTreeData::DataPart::calculateTotalSizeOnDisk(new_data_part->volume->getDisk(), new_data_part->getFullRelativePath()));
|
||||
new_data_part->default_codec = codec;
|
||||
new_data_part->calculateColumnsSizesOnDisk();
|
||||
}
|
||||
|
||||
|
@ -156,7 +156,11 @@ private:
|
||||
|
||||
/// Files, that we don't need to remove and don't need to hardlink, for example columns.txt and checksums.txt.
|
||||
/// Because we will generate new versions of them after we perform mutation.
|
||||
static NameSet collectFilesToSkip(const Block & updated_header, const std::set<MergeTreeIndexPtr> & indices_to_recalc, const String & mrk_extension);
|
||||
static NameSet collectFilesToSkip(
|
||||
const MergeTreeDataPartPtr & source_part,
|
||||
const Block & updated_header,
|
||||
const std::set<MergeTreeIndexPtr> & indices_to_recalc,
|
||||
const String & mrk_extension);
|
||||
|
||||
/// Get the columns list of the resulting part in the same order as storage_columns.
|
||||
static NamesAndTypesList getColumnsForNewDataPart(
|
||||
@ -209,7 +213,8 @@ private:
|
||||
static void finalizeMutatedPart(
|
||||
const MergeTreeDataPartPtr & source_part,
|
||||
MergeTreeData::MutableDataPartPtr new_data_part,
|
||||
bool need_remove_expired_values);
|
||||
bool need_remove_expired_values,
|
||||
const CompressionCodecPtr & codec);
|
||||
|
||||
public :
|
||||
/** Is used to cancel all merges and mutations. On cancel() call all currently running actions will throw exception soon.
|
||||
|
@ -56,7 +56,7 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter(
|
||||
const NamesAndTypesList & columns_list,
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
|
||||
const CompressionCodecPtr & default_codec,
|
||||
const CompressionCodecPtr & default_codec_,
|
||||
const MergeTreeWriterSettings & writer_settings,
|
||||
const MergeTreeIndexGranularity & computed_index_granularity) const
|
||||
{
|
||||
@ -71,7 +71,7 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter(
|
||||
return std::make_unique<MergeTreeDataPartWriterCompact>(
|
||||
shared_from_this(), ordered_columns_list, metadata_snapshot,
|
||||
indices_to_recalc, index_granularity_info.marks_file_extension,
|
||||
default_codec, writer_settings, computed_index_granularity);
|
||||
default_codec_, writer_settings, computed_index_granularity);
|
||||
}
|
||||
|
||||
|
||||
|
@ -24,6 +24,7 @@ MergeTreeDataPartInMemory::MergeTreeDataPartInMemory(
|
||||
const std::optional<String> & relative_path_)
|
||||
: IMergeTreeDataPart(storage_, name_, volume_, relative_path_, Type::IN_MEMORY)
|
||||
{
|
||||
default_codec = CompressionCodecFactory::instance().get("NONE", {});
|
||||
}
|
||||
|
||||
MergeTreeDataPartInMemory::MergeTreeDataPartInMemory(
|
||||
@ -34,6 +35,7 @@ MergeTreeDataPartInMemory::MergeTreeDataPartInMemory(
|
||||
const std::optional<String> & relative_path_)
|
||||
: IMergeTreeDataPart(storage_, name_, info_, volume_, relative_path_, Type::IN_MEMORY)
|
||||
{
|
||||
default_codec = CompressionCodecFactory::instance().get("NONE", {});
|
||||
}
|
||||
|
||||
IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartInMemory::getReader(
|
||||
@ -129,5 +131,4 @@ DataPartInMemoryPtr asInMemoryPart(const MergeTreeDataPartPtr & part)
|
||||
{
|
||||
return std::dynamic_pointer_cast<const MergeTreeDataPartInMemory>(part);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -57,14 +57,14 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartWide::getWriter(
|
||||
const NamesAndTypesList & columns_list,
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
|
||||
const CompressionCodecPtr & default_codec,
|
||||
const CompressionCodecPtr & default_codec_,
|
||||
const MergeTreeWriterSettings & writer_settings,
|
||||
const MergeTreeIndexGranularity & computed_index_granularity) const
|
||||
{
|
||||
return std::make_unique<MergeTreeDataPartWriterWide>(
|
||||
shared_from_this(), columns_list, metadata_snapshot, indices_to_recalc,
|
||||
index_granularity_info.marks_file_extension,
|
||||
default_codec, writer_settings, computed_index_granularity);
|
||||
default_codec_, writer_settings, computed_index_granularity);
|
||||
}
|
||||
|
||||
|
||||
|
@ -151,7 +151,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeWriteAheadLog::restore(const Stor
|
||||
|
||||
if (action_type == ActionType::ADD_PART)
|
||||
{
|
||||
MergedBlockOutputStream part_out(part, metadata_snapshot, block.getNamesAndTypesList(), {}, nullptr);
|
||||
MergedBlockOutputStream part_out(part, metadata_snapshot, block.getNamesAndTypesList(), {}, CompressionCodecFactory::instance().get("NONE", {}));
|
||||
|
||||
part->minmax_idx.update(block, storage.minmax_idx_columns);
|
||||
part->partition.create(metadata_snapshot, block, 0);
|
||||
|
@ -1,6 +1,7 @@
|
||||
#include <Storages/MergeTree/MergedBlockOutputStream.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Poco/File.h>
|
||||
#include <Parsers/queryToString.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -18,14 +19,14 @@ MergedBlockOutputStream::MergedBlockOutputStream(
|
||||
const StorageMetadataPtr & metadata_snapshot_,
|
||||
const NamesAndTypesList & columns_list_,
|
||||
const MergeTreeIndices & skip_indices,
|
||||
CompressionCodecPtr default_codec,
|
||||
CompressionCodecPtr default_codec_,
|
||||
bool blocks_are_granules_size)
|
||||
: MergedBlockOutputStream(
|
||||
data_part,
|
||||
metadata_snapshot_,
|
||||
columns_list_,
|
||||
skip_indices,
|
||||
default_codec,
|
||||
default_codec_,
|
||||
{},
|
||||
data_part->storage.global_context.getSettings().min_bytes_to_use_direct_io,
|
||||
blocks_are_granules_size)
|
||||
@ -37,12 +38,13 @@ MergedBlockOutputStream::MergedBlockOutputStream(
|
||||
const StorageMetadataPtr & metadata_snapshot_,
|
||||
const NamesAndTypesList & columns_list_,
|
||||
const MergeTreeIndices & skip_indices,
|
||||
CompressionCodecPtr default_codec,
|
||||
CompressionCodecPtr default_codec_,
|
||||
const MergeTreeData::DataPart::ColumnToSize & merged_column_to_size,
|
||||
size_t aio_threshold,
|
||||
bool blocks_are_granules_size)
|
||||
: IMergedBlockOutputStream(data_part, metadata_snapshot_)
|
||||
, columns_list(columns_list_)
|
||||
, default_codec(default_codec_)
|
||||
{
|
||||
MergeTreeWriterSettings writer_settings(
|
||||
storage.global_context.getSettings(),
|
||||
@ -120,6 +122,8 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart(
|
||||
new_part->setBytesOnDisk(checksums.getTotalSizeOnDisk());
|
||||
new_part->index_granularity = writer->getIndexGranularity();
|
||||
new_part->calculateColumnsSizesOnDisk();
|
||||
if (default_codec != nullptr)
|
||||
new_part->default_codec = default_codec;
|
||||
}
|
||||
|
||||
void MergedBlockOutputStream::finalizePartOnDisk(
|
||||
@ -162,6 +166,17 @@ void MergedBlockOutputStream::finalizePartOnDisk(
|
||||
part_columns.writeText(*out);
|
||||
}
|
||||
|
||||
if (default_codec != nullptr)
|
||||
{
|
||||
auto out = volume->getDisk()->writeFile(part_path + IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME, 4096);
|
||||
DB::writeText(queryToString(default_codec->getFullCodecDesc()), *out);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw Exception("Compression codec have to be specified for part on disk, empty for" + new_part->name
|
||||
+ ". It is a bug.", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
{
|
||||
/// Write file with checksums.
|
||||
auto out = volume->getDisk()->writeFile(part_path + "checksums.txt", 4096);
|
||||
|
@ -18,7 +18,7 @@ public:
|
||||
const StorageMetadataPtr & metadata_snapshot_,
|
||||
const NamesAndTypesList & columns_list_,
|
||||
const MergeTreeIndices & skip_indices,
|
||||
CompressionCodecPtr default_codec,
|
||||
CompressionCodecPtr default_codec_,
|
||||
bool blocks_are_granules_size = false);
|
||||
|
||||
MergedBlockOutputStream(
|
||||
@ -26,7 +26,7 @@ public:
|
||||
const StorageMetadataPtr & metadata_snapshot_,
|
||||
const NamesAndTypesList & columns_list_,
|
||||
const MergeTreeIndices & skip_indices,
|
||||
CompressionCodecPtr default_codec,
|
||||
CompressionCodecPtr default_codec_,
|
||||
const MergeTreeData::DataPart::ColumnToSize & merged_column_to_size,
|
||||
size_t aio_threshold,
|
||||
bool blocks_are_granules_size = false);
|
||||
@ -64,6 +64,7 @@ private:
|
||||
NamesAndTypesList columns_list;
|
||||
IMergeTreeDataPart::MinMaxIndex minmax_idx;
|
||||
size_t rows_count = 0;
|
||||
CompressionCodecPtr default_codec;
|
||||
};
|
||||
|
||||
}
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user