Merge branch 'master' into fix-fuzz-test6

This commit is contained in:
Alexey Milovidov 2020-09-02 04:40:42 +03:00
commit deb483eee5
115 changed files with 985 additions and 594 deletions

View File

@ -121,3 +121,18 @@ template <> struct is_big_int<bInt256> { static constexpr bool value = true; };
template <typename T>
inline constexpr bool is_big_int_v = is_big_int<T>::value;
template <typename T>
inline std::string bigintToString(const T & x)
{
return x.str();
}
template <typename To, typename From>
inline To bigint_cast(const From & x [[maybe_unused]])
{
if constexpr ((is_big_int_v<From> && std::is_same_v<To, UInt8>) || (is_big_int_v<To> && std::is_same_v<From, UInt8>))
return static_cast<uint8_t>(x);
else
return static_cast<To>(x);
}

View File

@ -2,6 +2,15 @@
<profiles>
<default>
<max_execution_time>10</max_execution_time>
<!--
Don't let the fuzzer change this setting (I've actually seen it
do this before).
-->
<constraints>
<max_execution_time>
<max>10</max>
</max_execution_time>
</constraints>
</default>
</profiles>
</yandex>

View File

@ -91,7 +91,7 @@ function fuzz
./clickhouse-client --query "select elapsed, query from system.processes" ||:
killall clickhouse-server ||:
for x in {1..10}
for _ in {1..10}
do
if ! pgrep -f clickhouse-server
then
@ -172,8 +172,60 @@ case "$stage" in
echo "failure" > status.txt
echo "Fuzzer failed ($fuzzer_exit_code). See the logs" > description.txt
fi
;&
"report")
cat > report.html <<EOF ||:
<!DOCTYPE html>
<html lang="en">
<link rel="preload" as="font" href="https://yastatic.net/adv-www/_/sUYVCPUAQE7ExrvMS7FoISoO83s.woff2" type="font/woff2" crossorigin="anonymous"/>
<style>
@font-face {
font-family:'Yandex Sans Display Web';
src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot);
src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot?#iefix) format('embedded-opentype'),
url(https://yastatic.net/adv-www/_/sUYVCPUAQE7ExrvMS7FoISoO83s.woff2) format('woff2'),
url(https://yastatic.net/adv-www/_/v2Sve_obH3rKm6rKrtSQpf-eB7U.woff) format('woff'),
url(https://yastatic.net/adv-www/_/PzD8hWLMunow5i3RfJ6WQJAL7aI.ttf) format('truetype'),
url(https://yastatic.net/adv-www/_/lF_KG5g4tpQNlYIgA0e77fBSZ5s.svg#YandexSansDisplayWeb-Regular) format('svg');
font-weight:400;
font-style:normal;
font-stretch:normal
}
exit $task_exit_code
body { font-family: "Yandex Sans Display Web", Arial, sans-serif; background: #EEE; }
h1 { margin-left: 10px; }
th, td { border: 0; padding: 5px 10px 5px 10px; text-align: left; vertical-align: top; line-height: 1.5; background-color: #FFF;
td { white-space: pre; font-family: Monospace, Courier New; }
border: 0; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }
a { color: #06F; text-decoration: none; }
a:hover, a:active { color: #F40; text-decoration: underline; }
table { border: 0; }
.main { margin-left: 10%; }
p.links a { padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-space: nowrap; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }
th { cursor: pointer; }
</style>
<title>AST Fuzzer for PR #${PR_TO_TEST} @ ${SHA_TO_TEST}</title>
</head>
<body>
<div class="main">
<h1>AST Fuzzer for PR #${PR_TO_TEST} @ ${SHA_TO_TEST}</h1>
<p class="links">
<a href="fuzzer.log">fuzzer.log</a>
<a href="server.log">server.log</a>
<a href="main.log">main.log</a>
</p>
<table>
<tr><th>Test name</th><th>Test status</th><th>Description</th></tr>
<tr><td>AST Fuzzer</td><td>$(cat status.txt)</td><td>$(cat description.txt)</td></tr>
</table>
</body>
</html>
EOF
;&
esac
exit $task_exit_code

View File

@ -5,3 +5,4 @@ services:
restart: always
ports:
- 6380:6379
command: redis-server --requirepass "clickhouse"

View File

@ -536,7 +536,9 @@ create table queries engine File(TSVWithNamesAndTypes, 'report/queries.tsv')
left join query_display_names
on query_metric_stats.test = query_display_names.test
and query_metric_stats.query_index = query_display_names.query_index
where metric_name = 'server_time'
-- 'server_time' is rounded down to ms, which might be bad for very short queries.
-- Use 'client_time' instead.
where metric_name = 'client_time'
order by test, query_index, metric_name
;
@ -888,7 +890,10 @@ for log in *-err.log
do
test=$(basename "$log" "-err.log")
{
grep -H -m2 -i '\(Exception\|Error\):[^:]' "$log" \
# The second grep is a heuristic for error messages like
# "socket.timeout: timed out".
grep -h -m2 -i '\(Exception\|Error\):[^:]' "$log" \
|| grep -h -m2 -i '^[^ ]\+: ' "$log" \
|| head -2 "$log"
} | sed "s/^/$test\t/" >> run-errors.tsv ||:
done

View File

@ -168,12 +168,6 @@ def nextRowAnchor():
global table_anchor
return f'{table_anchor}.{row_anchor + 1}'
def setRowAnchor(anchor_row_part):
global row_anchor
global table_anchor
row_anchor = anchor_row_part
return currentRowAnchor()
def advanceRowAnchor():
global row_anchor
global table_anchor
@ -480,11 +474,12 @@ if args.report == 'main':
total_runs = (nominal_runs + 1) * 2 # one prewarm run, two servers
attrs = ['' for c in columns]
for r in rows:
anchor = f'{currentTableAnchor()}.{r[0]}'
if float(r[6]) > 1.5 * total_runs:
# FIXME should be 15s max -- investigate parallel_insert
slow_average_tests += 1
attrs[6] = f'style="background: {color_bad}"'
errors_explained.append([f'<a href="./all-queries.html#all-query-times.{r[0]}.0">The test \'{r[0]}\' is too slow to run as a whole. Investigate whether the create and fill queries can be sped up'])
errors_explained.append([f'<a href="#{anchor}">The test \'{r[0]}\' is too slow to run as a whole. Investigate whether the create and fill queries can be sped up'])
else:
attrs[6] = ''
@ -495,7 +490,7 @@ if args.report == 'main':
else:
attrs[5] = ''
text += tableRow(r, attrs)
text += tableRow(r, attrs, anchor)
text += tableEnd()
tables.append(text)

View File

@ -31,7 +31,7 @@ For a description of request parameters, see [statement description](../../../sq
**ReplacingMergeTree Parameters**
- `ver` — column with version. Type `UInt*`, `Date`, `DateTime` or `DateTime64`. Optional parameter.
- `ver` — column with version. Type `UInt*`, `Date` or `DateTime`. Optional parameter.
When merging, `ReplacingMergeTree` from all the rows with the same sorting key leaves only one:

View File

@ -136,7 +136,7 @@ ENGINE = <Engine>
...
```
If a codec is specified, the default codec doesnt apply. Codecs can be combined in a pipeline, for example, `CODEC(Delta, ZSTD)`. To select the best codec combination for you project, pass benchmarks similar to described in the Altinity [New Encodings to Improve ClickHouse Efficiency](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse) article.
If a codec is specified, the default codec doesnt apply. Codecs can be combined in a pipeline, for example, `CODEC(Delta, ZSTD)`. To select the best codec combination for you project, pass benchmarks similar to described in the Altinity [New Encodings to Improve ClickHouse Efficiency](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse) article. One thing to note is that codec can't be applied for ALIAS column type.
!!! warning "Warning"
You cant decompress ClickHouse database files with external utilities like `lz4`. Instead, use the special [clickhouse-compressor](https://github.com/ClickHouse/ClickHouse/tree/master/programs/compressor) utility.

View File

@ -33,7 +33,7 @@ Para obtener una descripción de los parámetros de solicitud, consulte [descrip
**ReplacingMergeTree Parámetros**
- `ver` — column with version. Type `UInt*`, `Date`, `DateTime` o `DateTime64`. Parámetro opcional.
- `ver` — column with version. Type `UInt*`, `Date` o `DateTime`. Parámetro opcional.
Al fusionar, `ReplacingMergeTree` de todas las filas con la misma clave primaria deja solo una:

View File

@ -33,7 +33,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
**پارامترهای جایگزین**
- `ver` — column with version. Type `UInt*`, `Date`, `DateTime` یا `DateTime64`. پارامتر اختیاری.
- `ver` — column with version. Type `UInt*`, `Date` یا `DateTime`. پارامتر اختیاری.
هنگام ادغام, `ReplacingMergeTree` از تمام ردیف ها با همان کلید اصلی تنها یک برگ دارد:

View File

@ -33,7 +33,7 @@ Pour une description des paramètres de requête, voir [demande de description](
**ReplacingMergeTree Paramètres**
- `ver` — column with version. Type `UInt*`, `Date`, `DateTime` ou `DateTime64`. Paramètre facultatif.
- `ver` — column with version. Type `UInt*`, `Date` ou `DateTime`. Paramètre facultatif.
Lors de la fusion, `ReplacingMergeTree` de toutes les lignes avec la même clé primaire ne laisse qu'un:

View File

@ -33,7 +33,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
**ReplacingMergeTreeパラメータ**
- `ver` — column with version. Type `UInt*`, `Date`, `DateTime` または `DateTime64`. 任意パラメータ。
- `ver` — column with version. Type `UInt*`, `Date` または `DateTime`. 任意パラメータ。
マージ時, `ReplacingMergeTree` 同じ主キーを持つすべての行から、一つだけを残します:

View File

@ -25,7 +25,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
**Параметры ReplacingMergeTree**
- `ver` — столбец с версией, тип `UInt*`, `Date`, `DateTime` или `DateTime64`. Необязательный параметр.
- `ver` — столбец с версией, тип `UInt*`, `Date` или `DateTime`. Необязательный параметр.
При слиянии, из всех строк с одинаковым значением ключа сортировки `ReplacingMergeTree` оставляет только одну:

View File

@ -33,7 +33,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
**ReplacingMergeTree Parametreleri**
- `ver` — column with version. Type `UInt*`, `Date`, `DateTime` veya `DateTime64`. İsteğe bağlı parametre.
- `ver` — column with version. Type `UInt*`, `Date` veya `DateTime`. İsteğe bağlı parametre.
Birleş whenirken, `ReplacingMergeTree` aynı birincil anahtara sahip tüm satırlardan sadece bir tane bırakır:

View File

@ -25,7 +25,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
**参数**
- `ver` — 版本列。类型为 `UInt*`, `Date`, `DateTime``DateTime64`。可选参数。
- `ver` — 版本列。类型为 `UInt*`, `Date``DateTime`。可选参数。
合并的时候,`ReplacingMergeTree` 从所有具有相同主键的行中选择一行留下:
- 如果 `ver` 列未指定,选择最后一条。

View File

@ -847,32 +847,26 @@ private:
}
// Parse and execute what we've read.
fprintf(stderr, "will now parse '%s'\n", text.c_str());
const auto * new_end = processWithFuzzing(text);
if (new_end > &text[0])
{
const auto rest_size = text.size() - (new_end - &text[0]);
fprintf(stderr, "total %zd, rest %zd\n", text.size(), rest_size);
memcpy(&text[0], new_end, rest_size);
text.resize(rest_size);
}
else
{
fprintf(stderr, "total %zd, can't parse\n", text.size());
// We didn't read enough text to parse a query. Will read more.
}
if (!connection->isConnected())
{
// Uh-oh...
std::cerr << "Lost connection to the server." << std::endl;
last_exception_received_from_server
= std::make_unique<Exception>(210, "~");
return;
}
// Ensure that we're still connected to the server. If the server died,
// the reconnect is going to fail with an exception, and the fuzzer
// will exit. The ping() would be the best match here, but it's
// private, probably for a good reason that the protocol doesn't allow
// pings at any possible moment.
connection->forceConnected(connection_parameters.timeouts);
if (text.size() > 4 * 1024)
{
@ -880,9 +874,6 @@ private:
// and we still cannot parse a single query in it. Abort.
std::cerr << "Read too much text and still can't parse a query."
" Aborting." << std::endl;
last_exception_received_from_server
= std::make_unique<Exception>(1, "~");
// return;
exit(1);
}
}

View File

@ -106,8 +106,8 @@ AggregateFunctionPtr createAggregateFunctionQuantile(const std::string & name, c
if constexpr (supportBigInt<Function>())
{
if (which.idx == TypeIndex::Int128) return std::make_shared<Function<Int128, true>>(argument_types, params);
if (which.idx == TypeIndex::bInt256) return std::make_shared<Function<bInt256, true>>(argument_types, params);
if (which.idx == TypeIndex::bUInt256) return std::make_shared<Function<bUInt256, true>>(argument_types, params);
if (which.idx == TypeIndex::Int256) return std::make_shared<Function<Int256, true>>(argument_types, params);
if (which.idx == TypeIndex::UInt256) return std::make_shared<Function<UInt256, true>>(argument_types, params);
}
throw Exception("Illegal type " + argument_type->getName() + " of argument for aggregate function " + name,

View File

@ -137,23 +137,23 @@ struct AggregateFunctionUniqUpToData<UInt128> : AggregateFunctionUniqUpToData<UI
};
template <>
struct AggregateFunctionUniqUpToData<bUInt256> : AggregateFunctionUniqUpToData<UInt64>
struct AggregateFunctionUniqUpToData<UInt256> : AggregateFunctionUniqUpToData<UInt64>
{
/// ALWAYS_INLINE is required to have better code layout for uniqUpTo function
void ALWAYS_INLINE add(const IColumn & column, size_t row_num, UInt8 threshold)
{
bUInt256 value = assert_cast<const ColumnVector<bUInt256> &>(column).getData()[row_num];
UInt256 value = assert_cast<const ColumnVector<UInt256> &>(column).getData()[row_num];
insert(sipHash64(value), threshold);
}
};
template <>
struct AggregateFunctionUniqUpToData<bInt256> : AggregateFunctionUniqUpToData<UInt64>
struct AggregateFunctionUniqUpToData<Int256> : AggregateFunctionUniqUpToData<UInt64>
{
/// ALWAYS_INLINE is required to have better code layout for uniqUpTo function
void ALWAYS_INLINE add(const IColumn & column, size_t row_num, UInt8 threshold)
{
bInt256 value = assert_cast<const ColumnVector<bInt256> &>(column).getData()[row_num];
Int256 value = assert_cast<const ColumnVector<Int256> &>(column).getData()[row_num];
insert(sipHash64(value), threshold);
}
};

View File

@ -20,13 +20,13 @@
M(UInt16) \
M(UInt32) \
M(UInt64) \
M(bUInt256) \
M(UInt256) \
M(Int8) \
M(Int16) \
M(Int32) \
M(Int64) \
M(Int128) \
M(bInt256) \
M(Int256) \
M(Float32) \
M(Float64)

View File

@ -61,7 +61,7 @@ StringRef ColumnDecimal<T>::serializeValueIntoArena(size_t n, Arena & arena, cha
else
{
char * pos = arena.allocContinue(BigInt<T>::size, begin);
return BigInt<bInt256>::serialize(data[n], pos);
return BigInt<Int256>::serialize(data[n], pos);
}
}
@ -75,8 +75,8 @@ const char * ColumnDecimal<T>::deserializeAndInsertFromArena(const char * pos)
}
else
{
data.push_back(BigInt<bInt256>::deserialize(pos));
return pos + BigInt<bInt256>::size;
data.push_back(BigInt<Int256>::deserialize(pos));
return pos + BigInt<Int256>::size;
}
}
@ -273,7 +273,7 @@ void ColumnDecimal<T>::insertData(const char * src, size_t /*length*/)
}
else
{
data.push_back(BigInt<bInt256>::deserialize(src));
data.push_back(BigInt<Int256>::deserialize(src));
}
}

View File

@ -598,13 +598,13 @@ template class ColumnVector<UInt16>;
template class ColumnVector<UInt32>;
template class ColumnVector<UInt64>;
template class ColumnVector<UInt128>;
template class ColumnVector<bUInt256>;
template class ColumnVector<UInt256>;
template class ColumnVector<Int8>;
template class ColumnVector<Int16>;
template class ColumnVector<Int32>;
template class ColumnVector<Int64>;
template class ColumnVector<Int128>;
template class ColumnVector<bInt256>;
template class ColumnVector<Int256>;
template class ColumnVector<Float32>;
template class ColumnVector<Float64>;
}

View File

@ -14,14 +14,14 @@ using ColumnUInt16 = ColumnVector<UInt16>;
using ColumnUInt32 = ColumnVector<UInt32>;
using ColumnUInt64 = ColumnVector<UInt64>;
using ColumnUInt128 = ColumnVector<UInt128>;
using ColumnUInt256 = ColumnVector<bUInt256>;
using ColumnUInt256 = ColumnVector<UInt256>;
using ColumnInt8 = ColumnVector<Int8>;
using ColumnInt16 = ColumnVector<Int16>;
using ColumnInt32 = ColumnVector<Int32>;
using ColumnInt64 = ColumnVector<Int64>;
using ColumnInt128 = ColumnVector<Int128>;
using ColumnInt256 = ColumnVector<bInt256>;
using ColumnInt256 = ColumnVector<Int256>;
using ColumnFloat32 = ColumnVector<Float32>;
using ColumnFloat64 = ColumnVector<Float64>;

View File

@ -48,8 +48,8 @@ String FieldVisitorDump::operator() (const DecimalField<Decimal32> & x) const {
String FieldVisitorDump::operator() (const DecimalField<Decimal64> & x) const { return formatQuotedWithPrefix(x, "Decimal64_"); }
String FieldVisitorDump::operator() (const DecimalField<Decimal128> & x) const { return formatQuotedWithPrefix(x, "Decimal128_"); }
String FieldVisitorDump::operator() (const DecimalField<Decimal256> & x) const { return formatQuotedWithPrefix(x, "Decimal256_"); }
String FieldVisitorDump::operator() (const bUInt256 & x) const { return formatQuotedWithPrefix(x, "UInt256_"); }
String FieldVisitorDump::operator() (const bInt256 & x) const { return formatQuotedWithPrefix(x, "Int256_"); }
String FieldVisitorDump::operator() (const UInt256 & x) const { return formatQuotedWithPrefix(x, "UInt256_"); }
String FieldVisitorDump::operator() (const Int256 & x) const { return formatQuotedWithPrefix(x, "Int256_"); }
String FieldVisitorDump::operator() (const Int128 & x) const { return formatQuotedWithPrefix(x, "Int128_"); }
String FieldVisitorDump::operator() (const UInt128 & x) const { return formatQuotedWithPrefix(UUID(x), "UUID_"); }
@ -141,8 +141,8 @@ String FieldVisitorToString::operator() (const AggregateFunctionStateData & x) c
{
return formatQuoted(x.data);
}
String FieldVisitorToString::operator() (const bUInt256 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const bInt256 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const UInt256 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const Int256 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const Array & x) const
{
@ -286,16 +286,16 @@ void FieldVisitorHash::operator() (const AggregateFunctionStateData & x) const
hash.update(x.data.data(), x.data.size());
}
void FieldVisitorHash::operator() (const bUInt256 & x) const
void FieldVisitorHash::operator() (const UInt256 & x) const
{
UInt8 type = Field::Types::bUInt256;
UInt8 type = Field::Types::UInt256;
hash.update(type);
hash.update(x);
}
void FieldVisitorHash::operator() (const bInt256 & x) const
void FieldVisitorHash::operator() (const Int256 & x) const
{
UInt8 type = Field::Types::bInt256;
UInt8 type = Field::Types::Int256;
hash.update(type);
hash.update(x);
}

View File

@ -82,8 +82,8 @@ public:
String operator() (const DecimalField<Decimal256> & x) const;
String operator() (const AggregateFunctionStateData & x) const;
String operator() (const bUInt256 & x) const;
String operator() (const bInt256 & x) const;
String operator() (const UInt256 & x) const;
String operator() (const Int256 & x) const;
};
@ -106,8 +106,8 @@ public:
String operator() (const DecimalField<Decimal256> & x) const;
String operator() (const AggregateFunctionStateData & x) const;
String operator() (const bUInt256 & x) const;
String operator() (const bInt256 & x) const;
String operator() (const UInt256 & x) const;
String operator() (const Int256 & x) const;
};
@ -143,7 +143,7 @@ public:
T operator() (const Float64 & x) const
{
if constexpr (std::is_same_v<Decimal256, T>)
return bInt256(x);
return Int256(x);
else
return T(x);
}
@ -187,12 +187,10 @@ public:
{
if constexpr (IsDecimalNumber<T>)
return static_cast<T>(static_cast<typename T::NativeType>(x));
else if constexpr (std::is_same_v<T, UInt8>)
return static_cast<T>(static_cast<UInt16>(x));
else if constexpr (std::is_same_v<T, UInt128>)
throw Exception("No conversion to old UInt128 from " + demangle(typeid(U).name()), ErrorCodes::NOT_IMPLEMENTED);
else
return static_cast<T>(x);
return bigint_cast<T>(x);
}
};
@ -220,8 +218,8 @@ public:
void operator() (const DecimalField<Decimal256> & x) const;
void operator() (const AggregateFunctionStateData & x) const;
void operator() (const bUInt256 & x) const;
void operator() (const bInt256 & x) const;
void operator() (const UInt256 & x) const;
void operator() (const Int256 & x) const;
};

View File

@ -280,13 +280,13 @@ DEFINE_HASH(DB::UInt16)
DEFINE_HASH(DB::UInt32)
DEFINE_HASH(DB::UInt64)
DEFINE_HASH(DB::UInt128)
DEFINE_HASH(DB::bUInt256)
DEFINE_HASH(DB::UInt256)
DEFINE_HASH(DB::Int8)
DEFINE_HASH(DB::Int16)
DEFINE_HASH(DB::Int32)
DEFINE_HASH(DB::Int64)
DEFINE_HASH(DB::Int128)
DEFINE_HASH(DB::bInt256)
DEFINE_HASH(DB::Int256)
DEFINE_HASH(DB::Float32)
DEFINE_HASH(DB::Float64)
@ -297,7 +297,7 @@ template <>
struct DefaultHash<DB::UInt128> : public DB::UInt128Hash {};
template <>
struct DefaultHash<DB::UInt256> : public DB::UInt256Hash {};
struct DefaultHash<DB::DummyUInt256> : public DB::UInt256Hash {};
/// It is reasonable to use for UInt8, UInt16 with sufficient hash table size.

View File

@ -136,7 +136,7 @@ struct UInt128TrivialHash
/** Used for aggregation, for putting a large number of constant-length keys in a hash table.
*/
struct UInt256
struct DummyUInt256
{
/// Suppress gcc7 warnings: 'prev_key.DB::UInt256::a' may be used uninitialized in this function
@ -150,7 +150,7 @@ struct UInt256
UInt64 c;
UInt64 d;
bool operator== (const UInt256 rhs) const
bool operator== (const DummyUInt256 rhs) const
{
return a == rhs.a && b == rhs.b && c == rhs.c && d == rhs.d;
@ -164,7 +164,7 @@ struct UInt256
_mm_loadu_si128(reinterpret_cast<const __m128i *>(&rhs.c)))));*/
}
bool operator!= (const UInt256 rhs) const { return !operator==(rhs); }
bool operator!= (const DummyUInt256 rhs) const { return !operator==(rhs); }
bool operator== (const UInt64 rhs) const { return a == rhs && b == 0 && c == 0 && d == 0; }
bool operator!= (const UInt64 rhs) const { return !operator==(rhs); }
@ -173,12 +173,12 @@ struct UInt256
#pragma GCC diagnostic pop
#endif
UInt256 & operator= (const UInt64 rhs) { a = rhs; b = 0; c = 0; d = 0; return *this; }
DummyUInt256 & operator = (const UInt64 rhs) { a = rhs; b = 0; c = 0; d = 0; return *this; }
};
struct UInt256Hash
{
size_t operator()(UInt256 x) const
size_t operator()(DummyUInt256 x) const
{
/// NOTE suboptimal
return CityHash_v1_0_2::Hash128to64({CityHash_v1_0_2::Hash128to64({x.a, x.b}), CityHash_v1_0_2::Hash128to64({x.c, x.d})});
@ -189,7 +189,7 @@ struct UInt256Hash
struct UInt256HashCRC32
{
size_t operator()(UInt256 x) const
size_t operator()(DummyUInt256 x) const
{
UInt64 crc = -1ULL;
crc = _mm_crc32_u64(crc, x.a);

View File

@ -106,6 +106,15 @@ void CompressionCodecMultiple::doDecompressData(const char * source, UInt32 sour
memcpy(dest, compressed_buf.data(), decompressed_size);
}
std::vector<uint8_t> CompressionCodecMultiple::getCodecsBytesFromData(const char * source)
{
std::vector<uint8_t> result;
uint8_t compression_methods_size = source[0];
for (size_t i = 0; i < compression_methods_size; ++i)
result.push_back(source[1 + i]);
return result;
}
bool CompressionCodecMultiple::isCompression() const
{
for (const auto & codec : codecs)

View File

@ -17,6 +17,8 @@ public:
UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override;
static std::vector<uint8_t> getCodecsBytesFromData(const char * source);
protected:
UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;

View File

@ -5,6 +5,7 @@
#include <Parsers/ASTFunction.h>
#include <common/unaligned.h>
#include <Common/Exception.h>
#include <Parsers/queryToString.h>
namespace DB
@ -20,7 +21,17 @@ ASTPtr ICompressionCodec::getFullCodecDesc() const
{
std::shared_ptr<ASTFunction> result = std::make_shared<ASTFunction>();
result->name = "CODEC";
result->arguments = getCodecDesc();
ASTPtr codec_desc = getCodecDesc();
if (codec_desc->as<ASTExpressionList>())
{
result->arguments = codec_desc;
}
else
{
result->arguments = std::make_shared<ASTExpressionList>();
result->arguments->children.push_back(codec_desc);
}
result->children.push_back(result->arguments);
return result;
}

View File

@ -0,0 +1,43 @@
#include <Compression/getCompressionCodecForFile.h>
#include <Compression/CompressionInfo.h>
#include <Compression/CompressionFactory.h>
#include <IO/ReadBufferFromFileBase.h>
#include <Compression/CompressionCodecMultiple.h>
#include <Common/PODArray.h>
#include <common/logger_useful.h>
#include <Common/UInt128.h>
namespace DB
{
using Checksum = CityHash_v1_0_2::uint128;
CompressionCodecPtr getCompressionCodecForFile(const DiskPtr & disk, const String & relative_path)
{
auto read_buffer = disk->readFile(relative_path);
read_buffer->ignore(sizeof(Checksum));
UInt8 header_size = ICompressionCodec::getHeaderSize();
PODArray<char> compressed_buffer;
compressed_buffer.resize(header_size);
read_buffer->readStrict(compressed_buffer.data(), header_size);
uint8_t method = ICompressionCodec::readMethod(compressed_buffer.data());
if (method == static_cast<uint8_t>(CompressionMethodByte::Multiple))
{
compressed_buffer.resize(1);
read_buffer->readStrict(compressed_buffer.data(), 1);
compressed_buffer.resize(1 + compressed_buffer[0]);
read_buffer->readStrict(compressed_buffer.data() + 1, compressed_buffer[0]);
auto codecs_bytes = CompressionCodecMultiple::getCodecsBytesFromData(compressed_buffer.data());
Codecs codecs;
for (auto byte : codecs_bytes)
codecs.push_back(CompressionCodecFactory::instance().get(byte));
return std::make_shared<CompressionCodecMultiple>(codecs);
}
return CompressionCodecFactory::instance().get(method);
}
}

View File

@ -0,0 +1,15 @@
#pragma once
#include <Compression/ICompressionCodec.h>
#include <Disks/IDisk.h>
namespace DB
{
/// Return compression codec with default parameters for file compressed in
/// clickhouse fashion (with checksums, headers for each block, etc). This
/// method should be used as fallback when we cannot deduce compression codec
/// from metadata.
CompressionCodecPtr getCompressionCodecForFile(const DiskPtr & disk, const String & relative_path);
}

View File

@ -27,6 +27,7 @@ SRCS(
CompressionCodecT64.cpp
CompressionCodecZSTD.cpp
CompressionFactory.cpp
getCompressionCodecForFile.cpp
ICompressionCodec.cpp
LZ4_decompress_faster.cpp

View File

@ -52,7 +52,7 @@ template <typename TInt, typename TUInt>
constexpr bool is_le_int_vs_uint = is_any_int_vs_uint<TInt, TUInt> && (sizeof(TInt) <= sizeof(TUInt));
static_assert(is_le_int_vs_uint<Int128, DB::UInt128>);
static_assert(is_le_int_vs_uint<Int128, bUInt256>);
static_assert(is_le_int_vs_uint<Int128, DB::UInt256>);
template <typename TInt, typename TUInt>
using bool_if_le_int_vs_uint_t = std::enable_if_t<is_le_int_vs_uint<TInt, TUInt>, bool>;
@ -92,39 +92,27 @@ using bool_if_gt_int_vs_uint = std::enable_if_t<is_gt_int_vs_uint<TInt, TUInt>,
template <typename TInt, typename TUInt>
inline bool_if_gt_int_vs_uint<TInt, TUInt> greaterOpTmpl(TInt a, TUInt b)
{
if constexpr (is_big_int_v<TInt> && std::is_same_v<TUInt, UInt8>)
return static_cast<TInt>(a) > static_cast<TInt>(static_cast<UInt16>(b));
else
return static_cast<TInt>(a) > static_cast<TInt>(b);
return bigint_cast<TInt>(a) > bigint_cast<TInt>(b);
}
template <typename TInt, typename TUInt>
inline bool_if_gt_int_vs_uint<TInt, TUInt> greaterOpTmpl(TUInt a, TInt b)
{
if constexpr (is_big_int_v<TInt> && std::is_same_v<TUInt, UInt8>)
return static_cast<TInt>(static_cast<UInt16>(a)) > static_cast<TInt>(b);
else if constexpr (is_big_int_v<TInt> && std::is_same_v<TUInt, DB::UInt128>)
return static_cast<bUInt256>(a) > b;
else
return static_cast<TInt>(a) > b;
using CastA = std::conditional_t<is_big_int_v<TInt> && std::is_same_v<TUInt, DB::UInt128>, DB::UInt256, TInt>;
return bigint_cast<CastA>(a) > b;
}
template <typename TInt, typename TUInt>
inline bool_if_gt_int_vs_uint<TInt, TUInt> equalsOpTmpl(TInt a, TUInt b)
{
if constexpr (is_big_int_v<TInt> && std::is_same_v<TUInt, UInt8>)
return static_cast<TInt>(a) == static_cast<TInt>(static_cast<UInt16>(b));
else
return static_cast<TInt>(a) == static_cast<TInt>(b);
return bigint_cast<TInt>(a) == bigint_cast<TInt>(b);
}
template <typename TInt, typename TUInt>
inline bool_if_gt_int_vs_uint<TInt, TUInt> equalsOpTmpl(TUInt a, TInt b)
{
if constexpr (is_big_int_v<TInt> && std::is_same_v<TUInt, UInt8>)
return static_cast<TInt>(static_cast<UInt16>(a)) == static_cast<TInt>(b);
else
return static_cast<TInt>(a) == static_cast<TInt>(b);
return bigint_cast<TInt>(a) == bigint_cast<TInt>(b);
}
@ -200,16 +188,14 @@ inline bool_if_not_safe_conversion<A, B> greaterOp(A a, B b)
template <typename A, typename B>
inline bool_if_safe_conversion<A, B> greaterOp(A a, B b)
{
if constexpr (is_big_int_v<A> && std::is_same_v<B, UInt8>)
return a > static_cast<UInt16>(b);
else if constexpr (is_big_int_v<B> && std::is_same_v<A, UInt8>)
return static_cast<UInt16>(a) > b;
else if constexpr (std::is_same_v<A, DB::UInt128> && is_big_int_v<B>)
// hack for UInt128 and bUInt128
return static_cast<B>(a) > b;
else if constexpr (std::is_same_v<B, DB::UInt128> && is_big_int_v<A>)
// hack for UInt128 and bUInt128
return a > static_cast<A>(b);
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
return bigint_cast<CastA>(a) > bigint_cast<CastB>(b);
else
return a > b;
}
@ -317,17 +303,9 @@ inline bool_if_not_safe_conversion<A, B> equalsOp(A a, B b)
template <typename A, typename B>
inline bool_if_safe_conversion<A, B> equalsOp(A a, B b)
{
using LargestType = std::conditional_t<sizeof(A) >= sizeof(B), A, B>;
using LargestType = std::conditional_t<(sizeof(A) > sizeof(B)) || ((sizeof(A) == sizeof(B)) && !std::is_same_v<A, DB::UInt128>), A, B>;
if constexpr (is_big_int_v<LargestType> && std::is_same_v<A, UInt8>)
return static_cast<LargestType>(static_cast<UInt16>(a)) == static_cast<LargestType>(b);
else if constexpr (is_big_int_v<LargestType> && std::is_same_v<B, UInt8>)
return static_cast<LargestType>(a) == static_cast<LargestType>(static_cast<UInt16>(b));
else if constexpr (std::is_same_v<LargestType, DB::UInt128> && is_big_int_v<B>)
// hack for UInt128 and bUInt128
return static_cast<B>(a) == static_cast<B>(b);
else
return static_cast<LargestType>(a) == static_cast<LargestType>(b);
return bigint_cast<LargestType>(a) == bigint_cast<LargestType>(b);
}
template <>
@ -443,16 +421,14 @@ inline bool_if_not_safe_conversion<A, B> notEqualsOp(A a, B b)
template <typename A, typename B>
inline bool_if_safe_conversion<A, B> notEqualsOp(A a, B b)
{
if constexpr (std::is_same_v<A, UInt8> && is_big_int_v<B>)
return static_cast<UInt16>(a) != b;
else if constexpr (std::is_same_v<B, UInt8> && is_big_int_v<A>)
return a != static_cast<UInt16>(b);
else if constexpr (std::is_same_v<A, DB::UInt128> && is_big_int_v<B>)
// hack for UInt128 and bUInt128
return static_cast<B>(a) != b;
else if constexpr (std::is_same_v<B, DB::UInt128> && is_big_int_v<A>)
// hack for UInt128 and bUInt128
return a != static_cast<A>(b);
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
return bigint_cast<CastA>(a) != bigint_cast<CastB>(b);
else
return a != b;
}
@ -467,16 +443,14 @@ inline bool_if_not_safe_conversion<A, B> lessOp(A a, B b)
template <typename A, typename B>
inline bool_if_safe_conversion<A, B> lessOp(A a, B b)
{
if constexpr (std::is_same_v<A, UInt8> && is_big_int_v<B>)
return static_cast<UInt16>(a) < b;
else if constexpr (std::is_same_v<B, UInt8> && is_big_int_v<A>)
return a < static_cast<UInt16>(b);
else if constexpr (std::is_same_v<A, DB::UInt128> && is_big_int_v<B>)
// hack for UInt128 and bUInt128
return static_cast<B>(a) < b;
else if constexpr (std::is_same_v<B, DB::UInt128> && is_big_int_v<A>)
// hack for UInt128 and bUInt128
return a < static_cast<A>(b);
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
return bigint_cast<CastA>(a) < bigint_cast<CastB>(b);
else
return a < b;
}
@ -493,16 +467,14 @@ inline bool_if_not_safe_conversion<A, B> lessOrEqualsOp(A a, B b)
template <typename A, typename B>
inline bool_if_safe_conversion<A, B> lessOrEqualsOp(A a, B b)
{
if constexpr (std::is_same_v<A, UInt8> && is_big_int_v<B>)
return static_cast<UInt16>(a) <= b;
else if constexpr (std::is_same_v<B, UInt8> && is_big_int_v<A>)
return a <= static_cast<UInt16>(b);
else if constexpr (std::is_same_v<A, DB::UInt128> && is_big_int_v<B>)
// hack for UInt128 and bUInt128
return static_cast<B>(a) <= b;
else if constexpr (std::is_same_v<B, DB::UInt128> && is_big_int_v<A>)
// hack for UInt128 and bUInt128
return a <= static_cast<A>(b);
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
return bigint_cast<CastA>(a) <= bigint_cast<CastB>(b);
else
return a <= b;
}
@ -519,16 +491,14 @@ inline bool_if_not_safe_conversion<A, B> greaterOrEqualsOp(A a, B b)
template <typename A, typename B>
inline bool_if_safe_conversion<A, B> greaterOrEqualsOp(A a, B b)
{
if constexpr (std::is_same_v<A, UInt8> && is_big_int_v<B>)
return static_cast<UInt16>(a) >= b;
else if constexpr (std::is_same_v<B, UInt8> && is_big_int_v<A>)
return a >= static_cast<UInt16>(b);
else if constexpr (std::is_same_v<A, DB::UInt128> && is_big_int_v<B>)
// hack for UInt128 and bUInt128
return static_cast<B>(a) >= b;
else if constexpr (std::is_same_v<B, DB::UInt128> && is_big_int_v<A>)
// hack for UInt128 and bUInt128
return a >= static_cast<A>(b);
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
return bigint_cast<CastA>(a) >= bigint_cast<CastB>(b);
else
return a >= b;
}

View File

@ -37,7 +37,7 @@ inline bool allowDecimalComparison(const DataTypePtr & left_type, const DataType
template <size_t > struct ConstructDecInt { using Type = Int32; };
template <> struct ConstructDecInt<8> { using Type = Int64; };
template <> struct ConstructDecInt<16> { using Type = Int128; };
template <> struct ConstructDecInt<48> { using Type = bInt256; };
template <> struct ConstructDecInt<48> { using Type = Int256; };
template <typename T, typename U>
struct DecCompareInt
@ -228,18 +228,14 @@ private:
CompareInt x;
if constexpr (is_big_int_v<CompareInt> && IsDecimalNumber<A>)
x = a.value;
else if constexpr (is_big_int_v<CompareInt> && std::is_same_v<A, UInt8>)
x = static_cast<UInt16>(a);
else
x = static_cast<CompareInt>(a);
x = bigint_cast<CompareInt>(a);
CompareInt y;
if constexpr (is_big_int_v<CompareInt> && IsDecimalNumber<B>)
y = b.value;
else if constexpr (is_big_int_v<CompareInt> && std::is_same_v<B, UInt8>)
y = static_cast<UInt16>(b);
else
y = static_cast<CompareInt>(b);
y = bigint_cast<CompareInt>(b);
if constexpr (_check_overflow)
{

View File

@ -36,7 +36,7 @@ inline auto scaleMultiplier(UInt32 scale)
return common::exp10_i64(scale);
else if constexpr (std::is_same_v<T, Int128> || std::is_same_v<T, Decimal128>)
return common::exp10_i128(scale);
else if constexpr (std::is_same_v<T, bInt256> || std::is_same_v<T, Decimal256>)
else if constexpr (std::is_same_v<T, Int256> || std::is_same_v<T, Decimal256>)
return common::exp10_i256(scale);
}

View File

@ -218,16 +218,16 @@ void readBinary(Tuple & x, ReadBuffer & buf)
x.push_back(value);
break;
}
case Field::Types::bUInt256:
case Field::Types::UInt256:
{
bUInt256 value;
UInt256 value;
DB::readBinary(value, buf);
x.push_back(value);
break;
}
case Field::Types::bInt256:
case Field::Types::Int256:
{
bInt256 value;
Int256 value;
DB::readBinary(value, buf);
x.push_back(value);
break;
@ -301,14 +301,14 @@ void writeBinary(const Tuple & x, WriteBuffer & buf)
DB::writeStringBinary(get<std::string>(elem), buf);
break;
}
case Field::Types::bUInt256:
case Field::Types::UInt256:
{
DB::writeBinary(get<bUInt256>(elem), buf);
DB::writeBinary(get<UInt256>(elem), buf);
break;
}
case Field::Types::bInt256:
case Field::Types::Int256:
{
DB::writeBinary(get<bInt256>(elem), buf);
DB::writeBinary(get<Int256>(elem), buf);
break;
}
case Field::Types::Array:
@ -411,14 +411,14 @@ Field Field::restoreFromDump(const std::string_view & dump_)
prefix = std::string_view{"Int256_"};
if (dump.starts_with(prefix))
{
bInt256 value = parseFromString<bInt256>(dump.substr(prefix.length()));
Int256 value = parseFromString<Int256>(dump.substr(prefix.length()));
return value;
}
prefix = std::string_view{"UInt256_"};
if (dump.starts_with(prefix))
{
bUInt256 value = parseFromString<bUInt256>(dump.substr(prefix.length()));
UInt256 value = parseFromString<UInt256>(dump.substr(prefix.length()));
return value;
}

View File

@ -185,8 +185,8 @@ template <> struct NearestFieldTypeImpl<long long> { using Type = Int64; };
template <> struct NearestFieldTypeImpl<unsigned long> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<unsigned long long> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<bUInt256> { using Type = bUInt256; };
template <> struct NearestFieldTypeImpl<bInt256> { using Type = bInt256; };
template <> struct NearestFieldTypeImpl<UInt256> { using Type = UInt256; };
template <> struct NearestFieldTypeImpl<Int256> { using Type = Int256; };
template <> struct NearestFieldTypeImpl<Int128> { using Type = Int128; };
template <> struct NearestFieldTypeImpl<Decimal32> { using Type = DecimalField<Decimal32>; };
@ -255,8 +255,8 @@ public:
Decimal128 = 21,
AggregateFunctionState = 22,
Decimal256 = 23,
bUInt256 = 24,
bInt256 = 25,
UInt256 = 24,
Int256 = 25,
};
static const int MIN_NON_POD = 16;
@ -279,8 +279,8 @@ public:
case Decimal128: return "Decimal128";
case Decimal256: return "Decimal256";
case AggregateFunctionState: return "AggregateFunctionState";
case bUInt256: return "UInt256";
case bInt256: return "Int256";
case UInt256: return "UInt256";
case Int256: return "Int256";
}
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
@ -467,8 +467,8 @@ public:
case Types::Decimal128: return get<DecimalField<Decimal128>>() < rhs.get<DecimalField<Decimal128>>();
case Types::Decimal256: return get<DecimalField<Decimal256>>() < rhs.get<DecimalField<Decimal256>>();
case Types::AggregateFunctionState: return get<AggregateFunctionStateData>() < rhs.get<AggregateFunctionStateData>();
case Types::bUInt256: return get<bUInt256>() < rhs.get<bUInt256>();
case Types::bInt256: return get<bInt256>() < rhs.get<bInt256>();
case Types::UInt256: return get<UInt256>() < rhs.get<UInt256>();
case Types::Int256: return get<Int256>() < rhs.get<Int256>();
}
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
@ -502,8 +502,8 @@ public:
case Types::Decimal128: return get<DecimalField<Decimal128>>() <= rhs.get<DecimalField<Decimal128>>();
case Types::Decimal256: return get<DecimalField<Decimal256>>() <= rhs.get<DecimalField<Decimal256>>();
case Types::AggregateFunctionState: return get<AggregateFunctionStateData>() <= rhs.get<AggregateFunctionStateData>();
case Types::bUInt256: return get<bUInt256>() <= rhs.get<bUInt256>();
case Types::bInt256: return get<bInt256>() <= rhs.get<bInt256>();
case Types::UInt256: return get<UInt256>() <= rhs.get<UInt256>();
case Types::Int256: return get<Int256>() <= rhs.get<Int256>();
}
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
@ -541,8 +541,8 @@ public:
case Types::Decimal128: return get<DecimalField<Decimal128>>() == rhs.get<DecimalField<Decimal128>>();
case Types::Decimal256: return get<DecimalField<Decimal256>>() == rhs.get<DecimalField<Decimal256>>();
case Types::AggregateFunctionState: return get<AggregateFunctionStateData>() == rhs.get<AggregateFunctionStateData>();
case Types::bUInt256: return get<bUInt256>() == rhs.get<bUInt256>();
case Types::bInt256: return get<bInt256>() == rhs.get<bInt256>();
case Types::UInt256: return get<UInt256>() == rhs.get<UInt256>();
case Types::Int256: return get<Int256>() == rhs.get<Int256>();
}
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
@ -579,8 +579,8 @@ public:
case Types::Decimal256: return f(field.template get<DecimalField<Decimal256>>());
case Types::AggregateFunctionState: return f(field.template get<AggregateFunctionStateData>());
case Types::Int128: return f(field.template get<Int128>());
case Types::bUInt256: return f(field.template get<bUInt256>());
case Types::bInt256: return f(field.template get<bInt256>());
case Types::UInt256: return f(field.template get<UInt256>());
case Types::Int256: return f(field.template get<Int256>());
#if !__clang__
#pragma GCC diagnostic pop
#endif
@ -601,7 +601,7 @@ private:
Null, UInt64, UInt128, Int64, Int128, Float64, String, Array, Tuple,
DecimalField<Decimal32>, DecimalField<Decimal64>, DecimalField<Decimal128>, DecimalField<Decimal256>,
AggregateFunctionStateData,
bUInt256, bInt256
UInt256, Int256
> storage;
Types::Which which;
@ -732,8 +732,8 @@ template <> struct Field::TypeToEnum<DecimalField<Decimal64>>{ static const Type
template <> struct Field::TypeToEnum<DecimalField<Decimal128>>{ static const Types::Which value = Types::Decimal128; };
template <> struct Field::TypeToEnum<DecimalField<Decimal256>>{ static const Types::Which value = Types::Decimal256; };
template <> struct Field::TypeToEnum<AggregateFunctionStateData>{ static const Types::Which value = Types::AggregateFunctionState; };
template <> struct Field::TypeToEnum<bUInt256> { static const Types::Which value = Types::bUInt256; };
template <> struct Field::TypeToEnum<bInt256> { static const Types::Which value = Types::bInt256; };
template <> struct Field::TypeToEnum<UInt256> { static const Types::Which value = Types::UInt256; };
template <> struct Field::TypeToEnum<Int256> { static const Types::Which value = Types::Int256; };
template <> struct Field::EnumToType<Field::Types::Null> { using Type = Null; };
template <> struct Field::EnumToType<Field::Types::UInt64> { using Type = UInt64; };
@ -749,8 +749,8 @@ template <> struct Field::EnumToType<Field::Types::Decimal64> { using Type = Dec
template <> struct Field::EnumToType<Field::Types::Decimal128> { using Type = DecimalField<Decimal128>; };
template <> struct Field::EnumToType<Field::Types::Decimal256> { using Type = DecimalField<Decimal256>; };
template <> struct Field::EnumToType<Field::Types::AggregateFunctionState> { using Type = DecimalField<AggregateFunctionStateData>; };
template <> struct Field::EnumToType<Field::Types::bUInt256> { using Type = bUInt256; };
template <> struct Field::EnumToType<Field::Types::bInt256> { using Type = bInt256; };
template <> struct Field::EnumToType<Field::Types::UInt256> { using Type = UInt256; };
template <> struct Field::EnumToType<Field::Types::Int256> { using Type = Int256; };
inline constexpr bool isInt64FieldType(Field::Types::Which t)
{

View File

@ -8,7 +8,7 @@ namespace DB
{
using TypeListNativeNumbers = TypeList<UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64>;
using TypeListExtendedNumbers = TypeList<Int128, bUInt256, bInt256>;
using TypeListExtendedNumbers = TypeList<Int128, UInt256, Int256>;
using TypeListDecimalNumbers = TypeList<Decimal32, Decimal64, Decimal128, Decimal256>;
using TypeListGeneralNumbers = typename TypeListConcat<TypeListNativeNumbers, TypeListExtendedNumbers>::Type;

View File

@ -22,13 +22,13 @@ enum class TypeIndex
UInt32,
UInt64,
UInt128,
bUInt256,
UInt256,
Int8,
Int16,
Int32,
Int64,
Int128,
bInt256,
Int256,
Float32,
Float64,
Date,
@ -58,14 +58,14 @@ using UInt8 = ::UInt8;
using UInt16 = ::UInt16;
using UInt32 = ::UInt32;
using UInt64 = ::UInt64;
using bUInt256 = ::bUInt256;
using UInt256 = ::bUInt256;
using Int8 = ::Int8;
using Int16 = ::Int16;
using Int32 = ::Int32;
using Int64 = ::Int64;
using Int128 = ::Int128;
using bInt256 = ::bInt256;
using Int256 = ::bInt256;
using Float32 = float;
using Float64 = double;
@ -81,13 +81,13 @@ template <> inline constexpr bool IsNumber<UInt8> = true;
template <> inline constexpr bool IsNumber<UInt16> = true;
template <> inline constexpr bool IsNumber<UInt32> = true;
template <> inline constexpr bool IsNumber<UInt64> = true;
template <> inline constexpr bool IsNumber<bUInt256> = true;
template <> inline constexpr bool IsNumber<UInt256> = true;
template <> inline constexpr bool IsNumber<Int8> = true;
template <> inline constexpr bool IsNumber<Int16> = true;
template <> inline constexpr bool IsNumber<Int32> = true;
template <> inline constexpr bool IsNumber<Int64> = true;
template <> inline constexpr bool IsNumber<Int128> = true;
template <> inline constexpr bool IsNumber<bInt256> = true;
template <> inline constexpr bool IsNumber<Int256> = true;
template <> inline constexpr bool IsNumber<Float32> = true;
template <> inline constexpr bool IsNumber<Float64> = true;
@ -97,13 +97,13 @@ template <> struct TypeName<UInt8> { static constexpr const char * get() { ret
template <> struct TypeName<UInt16> { static constexpr const char * get() { return "UInt16"; } };
template <> struct TypeName<UInt32> { static constexpr const char * get() { return "UInt32"; } };
template <> struct TypeName<UInt64> { static constexpr const char * get() { return "UInt64"; } };
template <> struct TypeName<bUInt256> { static constexpr const char * get() { return "UInt256"; } };
template <> struct TypeName<UInt256> { static constexpr const char * get() { return "UInt256"; } };
template <> struct TypeName<Int8> { static constexpr const char * get() { return "Int8"; } };
template <> struct TypeName<Int16> { static constexpr const char * get() { return "Int16"; } };
template <> struct TypeName<Int32> { static constexpr const char * get() { return "Int32"; } };
template <> struct TypeName<Int64> { static constexpr const char * get() { return "Int64"; } };
template <> struct TypeName<Int128> { static constexpr const char * get() { return "Int128"; } };
template <> struct TypeName<bInt256> { static constexpr const char * get() { return "Int256"; } };
template <> struct TypeName<Int256> { static constexpr const char * get() { return "Int256"; } };
template <> struct TypeName<Float32> { static constexpr const char * get() { return "Float32"; } };
template <> struct TypeName<Float64> { static constexpr const char * get() { return "Float64"; } };
template <> struct TypeName<String> { static constexpr const char * get() { return "String"; } };
@ -113,13 +113,13 @@ template <> struct TypeId<UInt8> { static constexpr const TypeIndex value = T
template <> struct TypeId<UInt16> { static constexpr const TypeIndex value = TypeIndex::UInt16; };
template <> struct TypeId<UInt32> { static constexpr const TypeIndex value = TypeIndex::UInt32; };
template <> struct TypeId<UInt64> { static constexpr const TypeIndex value = TypeIndex::UInt64; };
template <> struct TypeId<bUInt256> { static constexpr const TypeIndex value = TypeIndex::bUInt256; };
template <> struct TypeId<UInt256> { static constexpr const TypeIndex value = TypeIndex::UInt256; };
template <> struct TypeId<Int8> { static constexpr const TypeIndex value = TypeIndex::Int8; };
template <> struct TypeId<Int16> { static constexpr const TypeIndex value = TypeIndex::Int16; };
template <> struct TypeId<Int32> { static constexpr const TypeIndex value = TypeIndex::Int32; };
template <> struct TypeId<Int64> { static constexpr const TypeIndex value = TypeIndex::Int64; };
template <> struct TypeId<Int128> { static constexpr const TypeIndex value = TypeIndex::Int128; };
template <> struct TypeId<bInt256> { static constexpr const TypeIndex value = TypeIndex::bInt256; };
template <> struct TypeId<Int256> { static constexpr const TypeIndex value = TypeIndex::Int256; };
template <> struct TypeId<Float32> { static constexpr const TypeIndex value = TypeIndex::Float32; };
template <> struct TypeId<Float64> { static constexpr const TypeIndex value = TypeIndex::Float64; };
@ -158,16 +158,12 @@ struct Decimal
if constexpr (std::is_same_v<U, Decimal<Int32>> ||
std::is_same_v<U, Decimal<Int64>> ||
std::is_same_v<U, Decimal<Int128>> ||
std::is_same_v<U, Decimal<bInt256>>)
std::is_same_v<U, Decimal<Int256>>)
{
return convertTo<typename U::NativeType>();
}
else if constexpr (is_big_int_v<NativeType>)
{
return value. template convert_to<U>();
}
else
return static_cast<U>(value);
return bigint_cast<U>(value);
}
const Decimal<T> & operator += (const T & x) { value += x; return *this; }
@ -193,7 +189,7 @@ template <typename T> inline Decimal<T> operator- (const Decimal<T> & x) { retur
using Decimal32 = Decimal<Int32>;
using Decimal64 = Decimal<Int64>;
using Decimal128 = Decimal<Int128>;
using Decimal256 = Decimal<bInt256>;
using Decimal256 = Decimal<Int256>;
using DateTime64 = Decimal64;
@ -217,11 +213,11 @@ template <typename T> struct NativeType { using Type = T; };
template <> struct NativeType<Decimal32> { using Type = Int32; };
template <> struct NativeType<Decimal64> { using Type = Int64; };
template <> struct NativeType<Decimal128> { using Type = Int128; };
template <> struct NativeType<Decimal256> { using Type = bInt256; };
template <> struct NativeType<Decimal256> { using Type = Int256; };
template <typename T> constexpr bool OverBigInt = false;
template <> inline constexpr bool OverBigInt<bInt256> = true;
template <> inline constexpr bool OverBigInt<bUInt256> = true;
template <> inline constexpr bool OverBigInt<Int256> = true;
template <> inline constexpr bool OverBigInt<UInt256> = true;
template <> inline constexpr bool OverBigInt<Decimal256> = true;
inline constexpr const char * getTypeName(TypeIndex idx)
@ -234,13 +230,13 @@ inline constexpr const char * getTypeName(TypeIndex idx)
case TypeIndex::UInt32: return TypeName<UInt32>::get();
case TypeIndex::UInt64: return TypeName<UInt64>::get();
case TypeIndex::UInt128: return "UInt128";
case TypeIndex::bUInt256: return TypeName<bUInt256>::get();
case TypeIndex::UInt256: return TypeName<UInt256>::get();
case TypeIndex::Int8: return TypeName<Int8>::get();
case TypeIndex::Int16: return TypeName<Int16>::get();
case TypeIndex::Int32: return TypeName<Int32>::get();
case TypeIndex::Int64: return TypeName<Int64>::get();
case TypeIndex::Int128: return TypeName<Int128>::get();
case TypeIndex::bInt256: return TypeName<bInt256>::get();
case TypeIndex::Int256: return TypeName<Int256>::get();
case TypeIndex::Float32: return TypeName<Float32>::get();
case TypeIndex::Float64: return TypeName<Float64>::get();
case TypeIndex::Date: return "Date";

View File

@ -26,14 +26,14 @@ bool callOnBasicType(TypeIndex number, F && f)
case TypeIndex::UInt16: return f(TypePair<T, UInt16>());
case TypeIndex::UInt32: return f(TypePair<T, UInt32>());
case TypeIndex::UInt64: return f(TypePair<T, UInt64>());
case TypeIndex::bUInt256: return f(TypePair<T, bUInt256>());
case TypeIndex::UInt256: return f(TypePair<T, UInt256>());
case TypeIndex::Int8: return f(TypePair<T, Int8>());
case TypeIndex::Int16: return f(TypePair<T, Int16>());
case TypeIndex::Int32: return f(TypePair<T, Int32>());
case TypeIndex::Int64: return f(TypePair<T, Int64>());
case TypeIndex::Int128: return f(TypePair<T, Int128>());
case TypeIndex::bInt256: return f(TypePair<T, bInt256>());
case TypeIndex::Int256: return f(TypePair<T, Int256>());
case TypeIndex::Enum8: return f(TypePair<T, Int8>());
case TypeIndex::Enum16: return f(TypePair<T, Int16>());
@ -94,14 +94,14 @@ inline bool callOnBasicTypes(TypeIndex type_num1, TypeIndex type_num2, F && f)
case TypeIndex::UInt16: return callOnBasicType<UInt16, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::UInt32: return callOnBasicType<UInt32, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::UInt64: return callOnBasicType<UInt64, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::bUInt256: return callOnBasicType<bUInt256, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::UInt256: return callOnBasicType<UInt256, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::Int8: return callOnBasicType<Int8, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::Int16: return callOnBasicType<Int16, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::Int32: return callOnBasicType<Int32, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::Int64: return callOnBasicType<Int64, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::Int128: return callOnBasicType<Int128, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::bInt256: return callOnBasicType<bInt256, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::Int256: return callOnBasicType<Int256, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::Enum8: return callOnBasicType<Int8, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::Enum16: return callOnBasicType<Int16, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
@ -171,14 +171,14 @@ bool callOnIndexAndDataType(TypeIndex number, F && f)
case TypeIndex::UInt16: return f(TypePair<DataTypeNumber<UInt16>, T>());
case TypeIndex::UInt32: return f(TypePair<DataTypeNumber<UInt32>, T>());
case TypeIndex::UInt64: return f(TypePair<DataTypeNumber<UInt64>, T>());
case TypeIndex::bUInt256: return f(TypePair<DataTypeNumber<bUInt256>, T>());
case TypeIndex::UInt256: return f(TypePair<DataTypeNumber<UInt256>, T>());
case TypeIndex::Int8: return f(TypePair<DataTypeNumber<Int8>, T>());
case TypeIndex::Int16: return f(TypePair<DataTypeNumber<Int16>, T>());
case TypeIndex::Int32: return f(TypePair<DataTypeNumber<Int32>, T>());
case TypeIndex::Int64: return f(TypePair<DataTypeNumber<Int64>, T>());
case TypeIndex::Int128: return f(TypePair<DataTypeNumber<Int128>, T>());
case TypeIndex::bInt256: return f(TypePair<DataTypeNumber<bInt256>, T>());
case TypeIndex::Int256: return f(TypePair<DataTypeNumber<Int256>, T>());
case TypeIndex::Float32: return f(TypePair<DataTypeNumber<Float32>, T>());
case TypeIndex::Float64: return f(TypePair<DataTypeNumber<Float64>, T>());

View File

@ -257,14 +257,14 @@ template class DataTypeNumberBase<UInt8>;
template class DataTypeNumberBase<UInt16>;
template class DataTypeNumberBase<UInt32>;
template class DataTypeNumberBase<UInt64>;
template class DataTypeNumberBase<UInt128>; // used only in UUID
template class DataTypeNumberBase<bUInt256>;
template class DataTypeNumberBase<UInt128>; // base for UUID
template class DataTypeNumberBase<UInt256>;
template class DataTypeNumberBase<Int8>;
template class DataTypeNumberBase<Int16>;
template class DataTypeNumberBase<Int32>;
template class DataTypeNumberBase<Int64>;
template class DataTypeNumberBase<Int128>;
template class DataTypeNumberBase<bInt256>;
template class DataTypeNumberBase<Int256>;
template class DataTypeNumberBase<Float32>;
template class DataTypeNumberBase<Float64>;

View File

@ -171,7 +171,7 @@ convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
else
{
if constexpr (is_big_int_v<FromFieldType>)
return convertDecimals<DataTypeDecimal<Decimal256>, ToDataType>(static_cast<bInt256>(value), 0, scale);
return convertDecimals<DataTypeDecimal<Decimal256>, ToDataType>(static_cast<Int256>(value), 0, scale);
else if constexpr (std::is_same_v<FromFieldType, UInt64>)
return convertDecimals<DataTypeDecimal<Decimal128>, ToDataType>(value, 0, scale);
else

View File

@ -39,7 +39,7 @@ using DataTypeFloat32 = DataTypeNumber<Float32>;
using DataTypeFloat64 = DataTypeNumber<Float64>;
using DataTypeInt128 = DataTypeNumber<Int128>;
using DataTypeUInt256 = DataTypeNumber<bUInt256>;
using DataTypeInt256 = DataTypeNumber<bInt256>;
using DataTypeUInt256 = DataTypeNumber<UInt256>;
using DataTypeInt256 = DataTypeNumber<Int256>;
}

View File

@ -124,12 +124,12 @@ DataTypePtr FieldToDataType::operator() (const AggregateFunctionStateData & x) c
return DataTypeFactory::instance().get(name);
}
DataTypePtr FieldToDataType::operator() (const bUInt256 &) const
DataTypePtr FieldToDataType::operator() (const UInt256 &) const
{
throw Exception("There are no UInt256 literals in SQL", ErrorCodes::NOT_IMPLEMENTED);
}
DataTypePtr FieldToDataType::operator() (const bInt256 &) const
DataTypePtr FieldToDataType::operator() (const Int256 &) const
{
throw Exception("There are no Int256 literals in SQL", ErrorCodes::NOT_IMPLEMENTED);
}

View File

@ -31,8 +31,8 @@ public:
DataTypePtr operator() (const DecimalField<Decimal128> & x) const;
DataTypePtr operator() (const DecimalField<Decimal256> & x) const;
DataTypePtr operator() (const AggregateFunctionStateData & x) const;
DataTypePtr operator() (const bUInt256 & x) const;
DataTypePtr operator() (const bInt256 & x) const;
DataTypePtr operator() (const UInt256 & x) const;
DataTypePtr operator() (const Int256 & x) const;
};
}

View File

@ -482,8 +482,8 @@ struct WhichDataType
bool isUInt32() const { return idx == TypeIndex::UInt32; }
bool isUInt64() const { return idx == TypeIndex::UInt64; }
bool isUInt128() const { return idx == TypeIndex::UInt128; }
bool isbUInt256() const { return idx == TypeIndex::bUInt256; }
bool isUInt() const { return isUInt8() || isUInt16() || isUInt32() || isUInt64() || isUInt128() || isbUInt256(); }
bool isUInt256() const { return idx == TypeIndex::UInt256; }
bool isUInt() const { return isUInt8() || isUInt16() || isUInt32() || isUInt64() || isUInt128() || isUInt256(); }
bool isNativeUInt() const { return isUInt8() || isUInt16() || isUInt32() || isUInt64(); }
bool isInt8() const { return idx == TypeIndex::Int8; }
@ -491,8 +491,8 @@ struct WhichDataType
bool isInt32() const { return idx == TypeIndex::Int32; }
bool isInt64() const { return idx == TypeIndex::Int64; }
bool isInt128() const { return idx == TypeIndex::Int128; }
bool isbInt256() const { return idx == TypeIndex::bInt256; }
bool isInt() const { return isInt8() || isInt16() || isInt32() || isInt64() || isInt128() || isbInt256(); }
bool isInt256() const { return idx == TypeIndex::Int256; }
bool isInt() const { return isInt8() || isInt16() || isInt32() || isInt64() || isInt128() || isInt256(); }
bool isNativeInt() const { return isInt8() || isInt16() || isInt32() || isInt64(); }
bool isDecimal32() const { return idx == TypeIndex::Decimal32; }
@ -529,7 +529,7 @@ struct WhichDataType
bool isFunction() const { return idx == TypeIndex::Function; }
bool isAggregateFunction() const { return idx == TypeIndex::AggregateFunction; }
bool IsBigIntOrDeimal() const { return isInt128() || isbInt256() || isbUInt256() || isDecimal256(); }
bool IsBigIntOrDeimal() const { return isInt128() || isInt256() || isUInt256() || isDecimal256(); }
};
/// IDataType helpers (alternative for IDataType virtual methods with single point of truth)

View File

@ -55,9 +55,9 @@ template <> struct Construct<false, false, 1> { using Type = UInt8; };
template <> struct Construct<false, false, 2> { using Type = UInt16; };
template <> struct Construct<false, false, 4> { using Type = UInt32; };
template <> struct Construct<false, false, 8> { using Type = UInt64; };
template <> struct Construct<false, false, 16> { using Type = bUInt256; };
template <> struct Construct<false, false, 32> { using Type = bUInt256; };
template <> struct Construct<false, false, 48> { using Type = bUInt256; };
template <> struct Construct<false, false, 16> { using Type = UInt256; };
template <> struct Construct<false, false, 32> { using Type = UInt256; };
template <> struct Construct<false, false, 48> { using Type = UInt256; };
template <> struct Construct<false, true, 1> { using Type = Float32; };
template <> struct Construct<false, true, 2> { using Type = Float32; };
template <> struct Construct<false, true, 4> { using Type = Float32; };
@ -68,7 +68,7 @@ template <> struct Construct<true, false, 4> { using Type = Int32; };
template <> struct Construct<true, false, 8> { using Type = Int64; };
template <> struct Construct<true, false, 16> { using Type = Int128; };
template <> struct Construct<true, false, 32> { using Type = Int128; };
template <> struct Construct<true, false, 48> { using Type = bInt256; };
template <> struct Construct<true, false, 48> { using Type = Int256; };
template <> struct Construct<true, true, 1> { using Type = Float32; };
template <> struct Construct<true, true, 2> { using Type = Float32; };
template <> struct Construct<true, true, 4> { using Type = Float32; };
@ -228,10 +228,7 @@ using ResultOfGreatest = std::conditional_t<LeastGreatestSpecialCase<A, B>,
template <typename T>
static inline auto littleBits(const T & x)
{
if constexpr (is_big_int_v<T>)
return x. template convert_to<UInt32>();
else
return UInt8(x);
return bigint_cast<UInt8>(x);
}
}

View File

@ -33,6 +33,11 @@ static constexpr size_t PRINT_MESSAGE_EACH_N_OBJECTS = 256;
static constexpr size_t PRINT_MESSAGE_EACH_N_SECONDS = 5;
static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768;
namespace ErrorCodes
{
extern const int NOT_IMPLEMENTED;
}
namespace
{
void tryAttachTable(
@ -249,6 +254,9 @@ void DatabaseOrdinary::alterTable(const Context & context, const StorageID & tab
auto & ast_create_query = ast->as<ASTCreateQuery &>();
if (ast_create_query.as_table_function)
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot alter table {} because it was created AS table function", backQuote(table_name));
ASTPtr new_columns = InterpreterCreateQuery::formatColumns(metadata.columns);
ASTPtr new_indices = InterpreterCreateQuery::formatIndices(metadata.secondary_indices);
ASTPtr new_constraints = InterpreterCreateQuery::formatConstraints(metadata.constraints);

View File

@ -52,12 +52,14 @@ namespace DB
const String & host_,
UInt16 port_,
UInt8 db_index_,
const String & password_,
RedisStorageType storage_type_,
const Block & sample_block_)
: dict_struct{dict_struct_}
, host{host_}
, port{port_}
, db_index{db_index_}
, password{password_}
, storage_type{storage_type_}
, sample_block{sample_block_}
, client{std::make_shared<Poco::Redis::Client>(host, port)}
@ -78,16 +80,22 @@ namespace DB
ErrorCodes::INVALID_CONFIG_PARAMETER};
// suppose key[0] is primary key, key[1] is secondary key
}
if (!password.empty())
{
RedisCommand command("AUTH");
command << password;
String reply = client->execute<String>(command);
if (reply != "OK")
throw Exception{"Authentication failed with reason "
+ reply, ErrorCodes::INTERNAL_REDIS_ERROR};
}
if (db_index != 0)
{
RedisCommand command("SELECT");
// Use poco's Int64, because it is defined as long long, and on
// MacOS, for the purposes of template instantiation, this type is
// distinct from int64_t, which is our Int64.
command << static_cast<Poco::Int64>(db_index);
command << std::to_string(db_index);
String reply = client->execute<String>(command);
if (reply != "+OK\r\n")
if (reply != "OK")
throw Exception{"Selecting database with index " + DB::toString(db_index)
+ " failed with reason " + reply, ErrorCodes::INTERNAL_REDIS_ERROR};
}
@ -104,6 +112,7 @@ namespace DB
config_.getString(config_prefix_ + ".host"),
config_.getUInt(config_prefix_ + ".port"),
config_.getUInt(config_prefix_ + ".db_index", 0),
config_.getString(config_prefix_ + ".password",""),
parseStorageType(config_.getString(config_prefix_ + ".storage_type", "")),
sample_block_)
{
@ -115,6 +124,7 @@ namespace DB
other.host,
other.port,
other.db_index,
other.password,
other.storage_type,
other.sample_block}
{

View File

@ -41,6 +41,7 @@ namespace ErrorCodes
const std::string & host,
UInt16 port,
UInt8 db_index,
const std::string & password,
RedisStorageType storage_type,
const Block & sample_block);
@ -91,6 +92,7 @@ namespace ErrorCodes
const std::string host;
const UInt16 port;
const UInt8 db_index;
const std::string password;
const RedisStorageType storage_type;
Block sample_block;

View File

@ -456,8 +456,8 @@ public:
}
bool readInt128(Int128 &) override { cannotConvertType("Int128"); }
bool readbInt256(bInt256 &) override { cannotConvertType("Int256"); }
bool readbUInt256(bUInt256 &) override { cannotConvertType("UInt256"); }
bool readInt256(Int256 &) override { cannotConvertType("Int256"); }
bool readUInt256(UInt256 &) override { cannotConvertType("UInt256"); }
bool readFloat32(Float32 &) override
{

View File

@ -62,8 +62,8 @@ public:
bool readNumber(UInt64 & value) { return current_converter->readUInt64(value); }
bool readNumber(Int128 & value) { return current_converter->readInt128(value); }
bool readNumber(UInt128 & value) { return current_converter->readUInt128(value); }
bool readNumber(bInt256 & value) { return current_converter->readbInt256(value); }
bool readNumber(bUInt256 & value) { return current_converter->readbUInt256(value); }
bool readNumber(Int256 & value) { return current_converter->readInt256(value); }
bool readNumber(UInt256 & value) { return current_converter->readUInt256(value); }
bool readNumber(Float32 & value) { return current_converter->readFloat32(value); }
bool readNumber(Float64 & value) { return current_converter->readFloat64(value); }
@ -152,8 +152,8 @@ private:
virtual bool readInt128(Int128 &) = 0;
virtual bool readUInt128(UInt128 &) = 0;
virtual bool readbInt256(bInt256 &) = 0;
virtual bool readbUInt256(bUInt256 &) = 0;
virtual bool readInt256(Int256 &) = 0;
virtual bool readUInt256(UInt256 &) = 0;
virtual bool readFloat32(Float32 &) = 0;
virtual bool readFloat64(Float64 &) = 0;
@ -232,8 +232,8 @@ public:
bool readNumber(UInt64 &) { return false; }
bool readNumber(Int128 &) { return false; }
bool readNumber(UInt128 &) { return false; }
bool readNumber(bInt256 &) { return false; }
bool readNumber(bUInt256 &) { return false; }
bool readNumber(Int256 &) { return false; }
bool readNumber(UInt256 &) { return false; }
bool readNumber(Float32 &) { return false; }
bool readNumber(Float64 &) { return false; }
bool readStringInto(PaddedPODArray<UInt8> &) { return false; }

View File

@ -319,8 +319,8 @@ public:
virtual void writeUInt64(UInt64) override { cannotConvertType("UInt64"); }
virtual void writeInt128(Int128) override { cannotConvertType("Int128"); }
virtual void writeUInt128(const UInt128 &) override { cannotConvertType("UInt128"); }
virtual void writebInt256(const bInt256 &) override { cannotConvertType("Int256"); }
virtual void writebUInt256(const bUInt256 &) override { cannotConvertType("UInt256"); }
virtual void writeInt256(const Int256 &) override { cannotConvertType("Int256"); }
virtual void writeUInt256(const UInt256 &) override { cannotConvertType("UInt256"); }
virtual void writeFloat32(Float32) override { cannotConvertType("Float32"); }
virtual void writeFloat64(Float64) override { cannotConvertType("Float64"); }
virtual void prepareEnumMapping8(const std::vector<std::pair<std::string, Int8>> &) override {}

View File

@ -65,8 +65,8 @@ public:
bool writeNumber(Int128 value) { return writeValueIfPossible(&IConverter::writeInt128, value); }
bool writeNumber(UInt128 value) { return writeValueIfPossible(&IConverter::writeUInt128, value); }
bool writeNumber(bInt256 value) { return writeValueIfPossible(&IConverter::writebInt256, value); }
bool writeNumber(bUInt256 value) { return writeValueIfPossible(&IConverter::writebUInt256, value); }
bool writeNumber(Int256 value) { return writeValueIfPossible(&IConverter::writeInt256, value); }
bool writeNumber(UInt256 value) { return writeValueIfPossible(&IConverter::writeUInt256, value); }
bool writeNumber(Float32 value) { return writeValueIfPossible(&IConverter::writeFloat32, value); }
bool writeNumber(Float64 value) { return writeValueIfPossible(&IConverter::writeFloat64, value); }
@ -156,8 +156,8 @@ private:
virtual void writeInt128(Int128) = 0;
virtual void writeUInt128(const UInt128 &) = 0;
virtual void writebInt256(const bInt256 &) = 0;
virtual void writebUInt256(const bUInt256 &) = 0;
virtual void writeInt256(const Int256 &) = 0;
virtual void writeUInt256(const UInt256 &) = 0;
virtual void writeFloat32(Float32) = 0;
virtual void writeFloat64(Float64) = 0;
@ -267,8 +267,8 @@ public:
bool writeNumber(UInt64 /* value */) { return false; }
bool writeNumber(Int128 /* value */) { return false; }
bool writeNumber(UInt128 /* value */) { return false; }
bool writeNumber(bInt256 /* value */) { return false; }
bool writeNumber(bUInt256 /* value */) { return false; }
bool writeNumber(Int256 /* value */) { return false; }
bool writeNumber(UInt256 /* value */) { return false; }
bool writeNumber(Float32 /* value */) { return false; }
bool writeNumber(Float64 /* value */) { return false; }
bool writeString(const StringRef & /* value */) { return false; }

View File

@ -50,7 +50,11 @@ inline auto checkedDivision(A a, B b)
{
throwIfDivisionLeadsToFPE(a, b);
if constexpr (is_big_int_v<A> && is_big_int_v<B>)
if constexpr (is_big_int_v<A> && std::is_floating_point_v<B>)
return bigint_cast<B>(a) / b;
else if constexpr (is_big_int_v<B> && std::is_floating_point_v<A>)
return a / bigint_cast<A>(b);
else if constexpr (is_big_int_v<A> && is_big_int_v<B>)
return static_cast<A>(a / b);
else if constexpr (!is_big_int_v<A> && is_big_int_v<B>)
return static_cast<A>(B(a) / b);
@ -70,28 +74,18 @@ struct DivideIntegralImpl
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
if constexpr (is_big_int_v<A> && std::is_floating_point_v<B>)
return Result(static_cast<B>(a) / b);
else if constexpr (is_big_int_v<B> && std::is_floating_point_v<A>)
return a / static_cast<Result>(b);
else if constexpr (is_big_int_v<A> && std::is_same_v<B, UInt8>)
return static_cast<Result>(checkedDivision(make_signed_t<A>(a), Int16(b)));
else if constexpr (is_big_int_v<B> && std::is_same_v<A, UInt8>)
return checkedDivision(Int16(a), make_signed_t<B>(b));
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
/// Otherwise overflow may occur due to integer promotion. Example: int8_t(-1) / uint64_t(2).
/// NOTE: overflow is still possible when dividing large signed number to large unsigned number or vice-versa. But it's less harmful.
else if constexpr (is_signed_v<A> || is_signed_v<B>)
if constexpr (is_integer_v<A> && is_integer_v<B> && (is_signed_v<A> || is_signed_v<B>))
{
if constexpr (is_integer_v<A> && is_integer_v<B>)
{
return checkedDivision(make_signed_t<A>(a),
sizeof(A) > sizeof(B) ? make_signed_t<A>(b) : make_signed_t<B>(b));
}
else
return checkedDivision(a, b);
return checkedDivision(make_signed_t<CastA>(a),
sizeof(A) > sizeof(B) ? make_signed_t<A>(CastB(b)) : make_signed_t<CastB>(b));
}
else
return checkedDivision(a, b);
return bigint_cast<Result>(checkedDivision(CastA(a), CastB(b)));
}
#if USE_EMBEDDED_COMPILER
@ -99,19 +93,6 @@ struct DivideIntegralImpl
#endif
};
template <typename Result, typename A, typename B>
inline Result applyBigIntModulo(A a, B b)
{
if constexpr (std::is_same_v<A, UInt8>)
return UInt16(a) % b;
else if constexpr (std::is_same_v<B, UInt8>)
return static_cast<UInt16>(a % UInt16(b));
else if constexpr (sizeof(A) > sizeof(B))
return static_cast<Result>(a % A(b));
else
return static_cast<Result>(B(a) % b);
}
template <typename A, typename B>
struct ModuloImpl
{
@ -120,7 +101,6 @@ struct ModuloImpl
using IntegerBType = typename NumberTraits::ToInteger<B>::Type;
static const constexpr bool allow_fixed_string = false;
static const constexpr bool is_special = is_big_int_v<IntegerAType> || is_big_int_v<IntegerBType>;
template <typename Result = ResultType>
static inline Result apply(A a, B b)
@ -134,8 +114,19 @@ struct ModuloImpl
{
throwIfDivisionLeadsToFPE(IntegerAType(a), IntegerBType(b));
if constexpr (is_special)
return applyBigIntModulo<Result>(IntegerAType(a), IntegerBType(b));
if constexpr (is_big_int_v<IntegerAType> || is_big_int_v<IntegerBType>)
{
using CastA = std::conditional_t<std::is_same_v<IntegerAType, UInt8>, uint8_t, IntegerAType>;
using CastB = std::conditional_t<std::is_same_v<IntegerBType, UInt8>, uint8_t, IntegerBType>;
CastA int_a(a);
CastB int_b(b);
if constexpr (is_big_int_v<IntegerBType> && sizeof(IntegerAType) <= sizeof(IntegerBType))
return bigint_cast<Result>(bigint_cast<CastB>(int_a) % int_b);
else
return bigint_cast<Result>(int_a % int_b);
}
else
return IntegerAType(a) % IntegerBType(b);
}

View File

@ -634,13 +634,13 @@ private:
|| executeNumRightType<T0, UInt32>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, UInt64>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, UInt128>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, bUInt256>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, UInt256>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Int8>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Int16>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Int32>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Int64>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Int128>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, bInt256>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Int256>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Float32>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Float64>(block, result, col_left, col_right_untyped))
return true;
@ -656,13 +656,13 @@ private:
|| executeNumConstRightType<T0, UInt32>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, UInt64>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, UInt128>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, bUInt256>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, UInt256>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Int8>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Int16>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Int32>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Int64>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Int128>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, bInt256>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Int256>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Float32>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Float64>(block, result, col_left_const, col_right_untyped))
return true;
@ -1223,13 +1223,13 @@ public:
|| executeNumLeftType<UInt32>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<UInt64>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<UInt128>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<bUInt256>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<UInt256>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Int8>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Int16>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Int32>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Int64>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Int128>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<bInt256>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Int256>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Float32>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Float64>(block, result, col_left_untyped, col_right_untyped)))
throw Exception("Illegal column " + col_left_untyped->getName()

View File

@ -150,13 +150,10 @@ struct ConvertImpl
}
else if constexpr (is_big_int_v<FromFieldType> || is_big_int_v<ToFieldType>)
{
using CastFrom = std::conditional_t<std::is_same_v<FromFieldType, UInt8>, uint8_t, FromFieldType>;
using CastTo = std::conditional_t<std::is_same_v<ToFieldType, UInt8>, uint8_t, ToFieldType>;
if constexpr (std::is_same_v<FromFieldType, UInt128> || std::is_same_v<ToFieldType, UInt128>)
throw Exception("Unexpected UInt128 to big int conversion", ErrorCodes::NOT_IMPLEMENTED);
else
vec_to[i] = static_cast<CastTo>(static_cast<CastFrom>(vec_from[i]));
vec_to[i] = bigint_cast<ToFieldType>(vec_from[i]);
}
else if constexpr (std::is_same_v<ToFieldType, UInt128> && sizeof(FromFieldType) <= sizeof(UInt64))
vec_to[i] = static_cast<ToFieldType>(static_cast<UInt64>(vec_from[i]));
@ -1630,13 +1627,13 @@ using FunctionToUInt8 = FunctionConvert<DataTypeUInt8, NameToUInt8, ToNumberMono
using FunctionToUInt16 = FunctionConvert<DataTypeUInt16, NameToUInt16, ToNumberMonotonicity<UInt16>>;
using FunctionToUInt32 = FunctionConvert<DataTypeUInt32, NameToUInt32, ToNumberMonotonicity<UInt32>>;
using FunctionToUInt64 = FunctionConvert<DataTypeUInt64, NameToUInt64, ToNumberMonotonicity<UInt64>>;
using FunctionToUInt256 = FunctionConvert<DataTypeUInt256, NameToUInt256, ToNumberMonotonicity<bUInt256>>;
using FunctionToUInt256 = FunctionConvert<DataTypeUInt256, NameToUInt256, ToNumberMonotonicity<UInt256>>;
using FunctionToInt8 = FunctionConvert<DataTypeInt8, NameToInt8, ToNumberMonotonicity<Int8>>;
using FunctionToInt16 = FunctionConvert<DataTypeInt16, NameToInt16, ToNumberMonotonicity<Int16>>;
using FunctionToInt32 = FunctionConvert<DataTypeInt32, NameToInt32, ToNumberMonotonicity<Int32>>;
using FunctionToInt64 = FunctionConvert<DataTypeInt64, NameToInt64, ToNumberMonotonicity<Int64>>;
using FunctionToInt128 = FunctionConvert<DataTypeInt128, NameToInt128, ToNumberMonotonicity<Int128>>;
using FunctionToInt256 = FunctionConvert<DataTypeInt256, NameToInt256, ToNumberMonotonicity<bInt256>>;
using FunctionToInt256 = FunctionConvert<DataTypeInt256, NameToInt256, ToNumberMonotonicity<Int256>>;
using FunctionToFloat32 = FunctionConvert<DataTypeFloat32, NameToFloat32, ToNumberMonotonicity<Float32>>;
using FunctionToFloat64 = FunctionConvert<DataTypeFloat64, NameToFloat64, ToNumberMonotonicity<Float64>>;
using FunctionToDate = FunctionConvert<DataTypeDate, NameToDate, ToDateMonotonicity>;

View File

@ -968,13 +968,13 @@ private:
else if (which.isUInt32()) executeIntType<UInt32, first>(icolumn, vec_to);
else if (which.isUInt64()) executeIntType<UInt64, first>(icolumn, vec_to);
else if (which.isUInt128() || which.isUUID()) executeBigIntType<UInt128, first>(icolumn, vec_to);
else if (which.isbUInt256()) executeBigIntType<bUInt256, first>(icolumn, vec_to);
else if (which.isUInt256()) executeBigIntType<UInt256, first>(icolumn, vec_to);
else if (which.isInt8()) executeIntType<Int8, first>(icolumn, vec_to);
else if (which.isInt16()) executeIntType<Int16, first>(icolumn, vec_to);
else if (which.isInt32()) executeIntType<Int32, first>(icolumn, vec_to);
else if (which.isInt64()) executeIntType<Int64, first>(icolumn, vec_to);
else if (which.isInt128()) executeBigIntType<Int128, first>(icolumn, vec_to);
else if (which.isbInt256()) executeBigIntType<bInt256, first>(icolumn, vec_to);
else if (which.isInt256()) executeBigIntType<Int256, first>(icolumn, vec_to);
else if (which.isEnum8()) executeIntType<Int8, first>(icolumn, vec_to);
else if (which.isEnum16()) executeIntType<Int16, first>(icolumn, vec_to);
else if (which.isDate()) executeIntType<UInt16, first>(icolumn, vec_to);

View File

@ -45,21 +45,14 @@ void writeSlice(const NumericArraySlice<T> & slice, NumericArraySink<U> & sink)
if constexpr (OverBigInt<T> || OverBigInt<U>)
{
if constexpr (std::is_same_v<T, UInt8> || std::is_same_v<U, UInt8>)
{
if constexpr (IsDecimalNumber<T>)
dst = static_cast<NativeU>(static_cast<UInt16>(src.value));
else
dst = static_cast<NativeU>(static_cast<UInt16>(src));
}
else if constexpr (std::is_same_v<U, UInt128>)
if constexpr (std::is_same_v<U, UInt128>)
{
throw Exception("No conversion between UInt128 and " + demangle(typeid(T).name()), ErrorCodes::NOT_IMPLEMENTED);
}
else if constexpr (IsDecimalNumber<T>)
dst = static_cast<NativeU>(src.value);
dst = bigint_cast<NativeU>(src.value);
else
dst = static_cast<NativeU>(src);
dst = bigint_cast<NativeU>(src);
}
else
dst = static_cast<NativeU>(src);

View File

@ -14,20 +14,11 @@ struct BitAndImpl
{
using ResultType = typename NumberTraits::ResultOfBit<A, B>::Type;
static constexpr const bool allow_fixed_string = true;
static constexpr bool need_uint8_cast = is_big_int_v<ResultType> && (std::is_same_v<A, UInt8> || std::is_same_v<B, UInt8>);
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
if constexpr (need_uint8_cast)
{
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
return static_cast<Result>(static_cast<CastA>(a)) & static_cast<Result>(static_cast<CastB>(b));
}
else
return static_cast<Result>(a) & static_cast<Result>(b);
return bigint_cast<Result>(a) & bigint_cast<Result>(b);
}
#if USE_EMBEDDED_COMPILER

View File

@ -13,20 +13,11 @@ struct BitOrImpl
{
using ResultType = typename NumberTraits::ResultOfBit<A, B>::Type;
static constexpr const bool allow_fixed_string = true;
static constexpr bool need_uint8_cast = is_big_int_v<ResultType> && (std::is_same_v<A, UInt8> || std::is_same_v<B, UInt8>);
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
if constexpr (need_uint8_cast)
{
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
return static_cast<Result>(static_cast<CastA>(a)) | static_cast<Result>(static_cast<CastB>(b));
}
else
return static_cast<Result>(a) | static_cast<Result>(b);
return bigint_cast<Result>(a) | bigint_cast<Result>(b);
}
#if USE_EMBEDDED_COMPILER

View File

@ -18,12 +18,10 @@ struct BitShiftLeftImpl
template <typename Result = ResultType>
static inline NO_SANITIZE_UNDEFINED Result apply(A a [[maybe_unused]], B b [[maybe_unused]])
{
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
if constexpr (is_big_int_v<B>)
throw Exception("BitShiftLeftImpl is not implemented for big integers as second argument", ErrorCodes::NOT_IMPLEMENTED);
else if constexpr (is_big_int_v<ResultType>)
return static_cast<Result>(a) << static_cast<CastB>(b);
else if constexpr (is_big_int_v<A>)
return static_cast<Result>(a) << bigint_cast<UInt32>(b);
else
return static_cast<Result>(a) << static_cast<Result>(b);
}

View File

@ -18,12 +18,10 @@ struct BitShiftRightImpl
template <typename Result = ResultType>
static inline NO_SANITIZE_UNDEFINED Result apply(A a [[maybe_unused]], B b [[maybe_unused]])
{
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
if constexpr (is_big_int_v<B>)
throw Exception("BitRotate is not implemented for big integers as second argument", ErrorCodes::NOT_IMPLEMENTED);
else if constexpr (is_big_int_v<ResultType>)
return static_cast<Result>(a) >> static_cast<CastB>(b);
else if constexpr (is_big_int_v<A>)
return static_cast<Result>(a) >> bigint_cast<UInt32>(b);
else
return static_cast<Result>(a) >> static_cast<Result>(b);
}

View File

@ -13,20 +13,11 @@ struct BitXorImpl
{
using ResultType = typename NumberTraits::ResultOfBit<A, B>::Type;
static constexpr bool allow_fixed_string = true;
static constexpr bool need_uint8_cast = is_big_int_v<ResultType> && (std::is_same_v<A, UInt8> || std::is_same_v<B, UInt8>);
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
if constexpr (need_uint8_cast)
{
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
return static_cast<Result>(static_cast<CastA>(a)) ^ static_cast<Result>(static_cast<CastB>(b));
}
else
return static_cast<Result>(a) ^ static_cast<Result>(b);
return bigint_cast<Result>(a) ^ bigint_cast<Result>(b);
}
#if USE_EMBEDDED_COMPILER

View File

@ -12,22 +12,12 @@ struct GreatestBaseImpl
{
using ResultType = NumberTraits::ResultOfGreatest<A, B>;
static const constexpr bool allow_fixed_string = false;
static constexpr bool need_uint8_cast = is_big_int_v<ResultType> && (std::is_same_v<A, UInt8> || std::is_same_v<B, UInt8>);
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
if constexpr (need_uint8_cast)
{
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
return static_cast<Result>(static_cast<CastA>(a)) > static_cast<Result>(static_cast<CastB>(b)) ?
static_cast<Result>(static_cast<CastA>(a)) : static_cast<Result>(static_cast<CastB>(b));
}
else
return static_cast<Result>(a) > static_cast<Result>(b) ?
static_cast<Result>(a) : static_cast<Result>(b);
return bigint_cast<Result>(a) > bigint_cast<Result>(b) ?
bigint_cast<Result>(a) : bigint_cast<Result>(b);
}
#if USE_EMBEDDED_COMPILER

View File

@ -44,15 +44,6 @@ using namespace GatherUtils;
* then, else - numeric types for which there is a general type, or dates, datetimes, or strings, or arrays of these types.
*/
template <typename From, typename To>
inline To special_cast(From x)
{
if constexpr (is_big_int_v<To> && std::is_same_v<From, UInt8>)
return static_cast<To>(static_cast<UInt16>(x));
else
return static_cast<To>(x);
}
template <typename A, typename B, typename ResultType>
struct NumIfImpl
{
@ -68,7 +59,7 @@ struct NumIfImpl
typename ColVecResult::Container & res = col_res->getData();
for (size_t i = 0; i < size; ++i)
res[i] = cond[i] ? special_cast<A, ResultType>(a[i]) : special_cast<B, ResultType>(b[i]);
res[i] = cond[i] ? bigint_cast<ResultType>(a[i]) : bigint_cast<ResultType>(b[i]);
block.getByPosition(result).column = std::move(col_res);
}
@ -79,7 +70,7 @@ struct NumIfImpl
typename ColVecResult::Container & res = col_res->getData();
for (size_t i = 0; i < size; ++i)
res[i] = cond[i] ? special_cast<A, ResultType>(a[i]) : special_cast<B, ResultType>(b);
res[i] = cond[i] ? bigint_cast<ResultType>(a[i]) : bigint_cast<ResultType>(b);
block.getByPosition(result).column = std::move(col_res);
}
@ -90,7 +81,7 @@ struct NumIfImpl
typename ColVecResult::Container & res = col_res->getData();
for (size_t i = 0; i < size; ++i)
res[i] = cond[i] ? special_cast<A, ResultType>(a) : special_cast<B, ResultType>(b[i]);
res[i] = cond[i] ? bigint_cast<ResultType>(a) : bigint_cast<ResultType>(b[i]);
block.getByPosition(result).column = std::move(col_res);
}
@ -101,7 +92,7 @@ struct NumIfImpl
typename ColVecResult::Container & res = col_res->getData();
for (size_t i = 0; i < size; ++i)
res[i] = cond[i] ? special_cast<A, ResultType>(a) : special_cast<B, ResultType>(b);
res[i] = cond[i] ? bigint_cast<ResultType>(a) : bigint_cast<ResultType>(b);
block.getByPosition(result).column = std::move(col_res);
}
};

View File

@ -12,22 +12,12 @@ struct LeastBaseImpl
{
using ResultType = NumberTraits::ResultOfLeast<A, B>;
static const constexpr bool allow_fixed_string = false;
static constexpr bool need_uint8_cast = is_big_int_v<ResultType> && (std::is_same_v<A, UInt8> || std::is_same_v<B, UInt8>);
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
if constexpr (need_uint8_cast)
{
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
return static_cast<Result>(static_cast<CastA>(a)) < static_cast<Result>(static_cast<CastB>(b)) ?
static_cast<Result>(static_cast<CastA>(a)) : static_cast<Result>(static_cast<CastB>(b));
}
else
/** gcc 4.9.2 successfully vectorizes a loop from this function. */
return static_cast<Result>(a) < static_cast<Result>(b) ? static_cast<Result>(a) : static_cast<Result>(b);
/** gcc 4.9.2 successfully vectorizes a loop from this function. */
return bigint_cast<Result>(a) < bigint_cast<Result>(b) ? bigint_cast<Result>(a) : bigint_cast<Result>(b);
}
#if USE_EMBEDDED_COMPILER

View File

@ -17,10 +17,10 @@ struct MinusImpl
{
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
{
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, std::conditional_t<std::is_floating_point_v<B>, B, A>>;
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, std::conditional_t<std::is_floating_point_v<A>, A, B>>;
using CastA = std::conditional_t<std::is_floating_point_v<B>, B, A>;
using CastB = std::conditional_t<std::is_floating_point_v<A>, A, B>;
return static_cast<Result>(static_cast<CastA>(a)) - static_cast<Result>(static_cast<CastB>(b));
return bigint_cast<Result>(bigint_cast<CastA>(a)) - bigint_cast<Result>(bigint_cast<CastB>(b));
}
else
return static_cast<Result>(a) - b;

View File

@ -17,10 +17,10 @@ struct MultiplyImpl
{
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
{
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, std::conditional_t<std::is_floating_point_v<B>, B, A>>;
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, std::conditional_t<std::is_floating_point_v<A>, A, B>>;
using CastA = std::conditional_t<std::is_floating_point_v<B>, B, A>;
using CastB = std::conditional_t<std::is_floating_point_v<A>, A, B>;
return static_cast<Result>(static_cast<CastA>(a)) * static_cast<Result>(static_cast<CastB>(b));
return bigint_cast<Result>(bigint_cast<CastA>(a)) * bigint_cast<Result>(bigint_cast<CastB>(b));
}
else
return static_cast<Result>(a) * b;

View File

@ -18,10 +18,10 @@ struct PlusImpl
/// Next everywhere, static_cast - so that there is no wrong result in expressions of the form Int64 c = UInt32(a) * Int32(-1).
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
{
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, std::conditional_t<std::is_floating_point_v<B>, B, A>>;
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, std::conditional_t<std::is_floating_point_v<A>, A, B>>;
using CastA = std::conditional_t<std::is_floating_point_v<B>, B, A>;
using CastB = std::conditional_t<std::is_floating_point_v<A>, A, B>;
return static_cast<Result>(static_cast<CastA>(a)) + static_cast<Result>(static_cast<CastB>(b));
return bigint_cast<Result>(bigint_cast<CastA>(a)) + bigint_cast<Result>(bigint_cast<CastB>(b));
}
else
return static_cast<Result>(a) + b;

View File

@ -13,169 +13,169 @@ static const std::map<std::pair<std::string, std::string>, std::string> answer =
{{"UInt8", "UInt16"}, "UInt16"},
{{"UInt8", "UInt32"}, "UInt32"},
{{"UInt8", "UInt64"}, "UInt64"},
{{"UInt8", "bUInt256"}, "bUInt256"},
{{"UInt8", "UInt256"}, "UInt256"},
{{"UInt8", "Int8"}, "Int16"},
{{"UInt8", "Int16"}, "Int16"},
{{"UInt8", "Int32"}, "Int32"},
{{"UInt8", "Int64"}, "Int64"},
{{"UInt8", "Int128"}, "Int128"},
{{"UInt8", "bInt256"}, "bInt256"},
{{"UInt8", "Int256"}, "Int256"},
{{"UInt8", "Float32"}, "Float32"},
{{"UInt8", "Float64"}, "Float64"},
{{"UInt16", "UInt8"}, "UInt16"},
{{"UInt16", "UInt16"}, "UInt16"},
{{"UInt16", "UInt32"}, "UInt32"},
{{"UInt16", "UInt64"}, "UInt64"},
{{"UInt16", "bUInt256"}, "bUInt256"},
{{"UInt16", "UInt256"}, "UInt256"},
{{"UInt16", "Int8"}, "Int32"},
{{"UInt16", "Int16"}, "Int32"},
{{"UInt16", "Int32"}, "Int32"},
{{"UInt16", "Int64"}, "Int64"},
{{"UInt16", "Int128"}, "Int128"},
{{"UInt16", "bInt256"}, "bInt256"},
{{"UInt16", "Int256"}, "Int256"},
{{"UInt16", "Float32"}, "Float32"},
{{"UInt16", "Float64"}, "Float64"},
{{"UInt32", "UInt8"}, "UInt32"},
{{"UInt32", "UInt16"}, "UInt32"},
{{"UInt32", "UInt32"}, "UInt32"},
{{"UInt32", "UInt64"}, "UInt64"},
{{"UInt32", "bUInt256"}, "bUInt256"},
{{"UInt32", "UInt256"}, "UInt256"},
{{"UInt32", "Int8"}, "Int64"},
{{"UInt32", "Int16"}, "Int64"},
{{"UInt32", "Int32"}, "Int64"},
{{"UInt32", "Int64"}, "Int64"},
{{"UInt32", "Int128"}, "Int128"},
{{"UInt32", "bInt256"}, "bInt256"},
{{"UInt32", "Int256"}, "Int256"},
{{"UInt32", "Float32"}, "Float64"},
{{"UInt32", "Float64"}, "Float64"},
{{"UInt64", "UInt8"}, "UInt64"},
{{"UInt64", "UInt16"}, "UInt64"},
{{"UInt64", "UInt32"}, "UInt64"},
{{"UInt64", "UInt64"}, "UInt64"},
{{"UInt64", "bUInt256"}, "bUInt256"},
{{"UInt64", "UInt256"}, "UInt256"},
{{"UInt64", "Int8"}, "Int128"},
{{"UInt64", "Int16"}, "Int128"},
{{"UInt64", "Int32"}, "Int128"},
{{"UInt64", "Int64"}, "Int128"},
{{"UInt64", "Int128"}, "Int128"},
{{"UInt64", "bInt256"}, "bInt256"},
{{"UInt64", "Int256"}, "Int256"},
{{"UInt64", "Float32"}, "Error"},
{{"UInt64", "Float64"}, "Error"},
{{"bUInt256", "UInt8"}, "bUInt256"},
{{"bUInt256", "UInt16"}, "bUInt256"},
{{"bUInt256", "UInt32"}, "bUInt256"},
{{"bUInt256", "UInt64"}, "bUInt256"},
{{"bUInt256", "bUInt256"}, "bUInt256"},
{{"bUInt256", "Int8"}, "Error"},
{{"bUInt256", "Int16"}, "Error"},
{{"bUInt256", "Int32"}, "Error"},
{{"bUInt256", "Int64"}, "Error"},
{{"bUInt256", "Int128"}, "Error"},
{{"bUInt256", "bInt256"}, "Error"},
{{"bUInt256", "Float32"}, "Error"},
{{"bUInt256", "Float64"}, "Error"},
{{"UInt256", "UInt8"}, "UInt256"},
{{"UInt256", "UInt16"}, "UInt256"},
{{"UInt256", "UInt32"}, "UInt256"},
{{"UInt256", "UInt64"}, "UInt256"},
{{"UInt256", "UInt256"}, "UInt256"},
{{"UInt256", "Int8"}, "Error"},
{{"UInt256", "Int16"}, "Error"},
{{"UInt256", "Int32"}, "Error"},
{{"UInt256", "Int64"}, "Error"},
{{"UInt256", "Int128"}, "Error"},
{{"UInt256", "Int256"}, "Error"},
{{"UInt256", "Float32"}, "Error"},
{{"UInt256", "Float64"}, "Error"},
{{"Int8", "UInt8"}, "Int16"},
{{"Int8", "UInt16"}, "Int32"},
{{"Int8", "UInt32"}, "Int64"},
{{"Int8", "UInt64"}, "Int128"},
{{"Int8", "bUInt256"}, "Error"},
{{"Int8", "UInt256"}, "Error"},
{{"Int8", "Int8"}, "Int8"},
{{"Int8", "Int16"}, "Int16"},
{{"Int8", "Int32"}, "Int32"},
{{"Int8", "Int64"}, "Int64"},
{{"Int8", "Int128"}, "Int128"},
{{"Int8", "bInt256"}, "bInt256"},
{{"Int8", "Int256"}, "Int256"},
{{"Int8", "Float32"}, "Float32"},
{{"Int8", "Float64"}, "Float64"},
{{"Int16", "UInt8"}, "Int16"},
{{"Int16", "UInt16"}, "Int32"},
{{"Int16", "UInt32"}, "Int64"},
{{"Int16", "UInt64"}, "Int128"},
{{"Int16", "bUInt256"}, "Error"},
{{"Int16", "UInt256"}, "Error"},
{{"Int16", "Int8"}, "Int16"},
{{"Int16", "Int16"}, "Int16"},
{{"Int16", "Int32"}, "Int32"},
{{"Int16", "Int64"}, "Int64"},
{{"Int16", "Int128"}, "Int128"},
{{"Int16", "bInt256"}, "bInt256"},
{{"Int16", "Int256"}, "Int256"},
{{"Int16", "Float32"}, "Float32"},
{{"Int16", "Float64"}, "Float64"},
{{"Int32", "UInt8"}, "Int32"},
{{"Int32", "UInt16"}, "Int32"},
{{"Int32", "UInt32"}, "Int64"},
{{"Int32", "UInt64"}, "Int128"},
{{"Int32", "bUInt256"}, "Error"},
{{"Int32", "UInt256"}, "Error"},
{{"Int32", "Int8"}, "Int32"},
{{"Int32", "Int16"}, "Int32"},
{{"Int32", "Int32"}, "Int32"},
{{"Int32", "Int64"}, "Int64"},
{{"Int32", "Int128"}, "Int128"},
{{"Int32", "bInt256"}, "bInt256"},
{{"Int32", "Int256"}, "Int256"},
{{"Int32", "Float32"}, "Float64"},
{{"Int32", "Float64"}, "Float64"},
{{"Int64", "UInt8"}, "Int64"},
{{"Int64", "UInt16"}, "Int64"},
{{"Int64", "UInt32"}, "Int64"},
{{"Int64", "UInt64"}, "Int128"},
{{"Int64", "bUInt256"}, "Error"},
{{"Int64", "UInt256"}, "Error"},
{{"Int64", "Int8"}, "Int64"},
{{"Int64", "Int16"}, "Int64"},
{{"Int64", "Int32"}, "Int64"},
{{"Int64", "Int64"}, "Int64"},
{{"Int64", "Int128"}, "Int128"},
{{"Int64", "bInt256"}, "bInt256"},
{{"Int64", "Int256"}, "Int256"},
{{"Int64", "Float32"}, "Error"},
{{"Int64", "Float64"}, "Error"},
{{"Int128", "UInt8"}, "Int128"},
{{"Int128", "UInt16"}, "Int128"},
{{"Int128", "UInt32"}, "Int128"},
{{"Int128", "UInt64"}, "Int128"},
{{"Int128", "bUInt256"}, "Error"},
{{"Int128", "UInt256"}, "Error"},
{{"Int128", "Int8"}, "Int128"},
{{"Int128", "Int16"}, "Int128"},
{{"Int128", "Int32"}, "Int128"},
{{"Int128", "Int64"}, "Int128"},
{{"Int128", "Int128"}, "Int128"},
{{"Int128", "bInt256"}, "bInt256"},
{{"Int128", "Int256"}, "Int256"},
{{"Int128", "Float32"}, "Error"},
{{"Int128", "Float64"}, "Error"},
{{"bInt256", "UInt8"}, "bInt256"},
{{"bInt256", "UInt16"}, "bInt256"},
{{"bInt256", "UInt32"}, "bInt256"},
{{"bInt256", "UInt64"}, "bInt256"},
{{"bInt256", "bUInt256"}, "Error"},
{{"bInt256", "Int8"}, "bInt256"},
{{"bInt256", "Int16"}, "bInt256"},
{{"bInt256", "Int32"}, "bInt256"},
{{"bInt256", "Int64"}, "bInt256"},
{{"bInt256", "Int128"}, "bInt256"},
{{"bInt256", "bInt256"}, "bInt256"},
{{"bInt256", "Float32"}, "Error"},
{{"bInt256", "Float64"}, "Error"},
{{"Int256", "UInt8"}, "Int256"},
{{"Int256", "UInt16"}, "Int256"},
{{"Int256", "UInt32"}, "Int256"},
{{"Int256", "UInt64"}, "Int256"},
{{"Int256", "UInt256"}, "Error"},
{{"Int256", "Int8"}, "Int256"},
{{"Int256", "Int16"}, "Int256"},
{{"Int256", "Int32"}, "Int256"},
{{"Int256", "Int64"}, "Int256"},
{{"Int256", "Int128"}, "Int256"},
{{"Int256", "Int256"}, "Int256"},
{{"Int256", "Float32"}, "Error"},
{{"Int256", "Float64"}, "Error"},
{{"Float32", "UInt8"}, "Float32"},
{{"Float32", "UInt16"}, "Float32"},
{{"Float32", "UInt32"}, "Float64"},
{{"Float32", "UInt64"}, "Error"},
{{"Float32", "bUInt256"}, "Error"},
{{"Float32", "UInt256"}, "Error"},
{{"Float32", "Int8"}, "Float32"},
{{"Float32", "Int16"}, "Float32"},
{{"Float32", "Int32"}, "Float64"},
{{"Float32", "Int64"}, "Error"},
{{"Float32", "Int128"}, "Error"},
{{"Float32", "bInt256"}, "Error"},
{{"Float32", "Int256"}, "Error"},
{{"Float32", "Float32"}, "Float32"},
{{"Float32", "Float64"}, "Float64"},
{{"Float64", "UInt8"}, "Float64"},
{{"Float64", "UInt16"}, "Float64"},
{{"Float64", "UInt32"}, "Float64"},
{{"Float64", "UInt64"}, "Error"},
{{"Float64", "bUInt256"}, "Error"},
{{"Float64", "UInt256"}, "Error"},
{{"Float64", "Int8"}, "Float64"},
{{"Float64", "Int16"}, "Float64"},
{{"Float64", "Int32"}, "Float64"},
{{"Float64", "Int64"}, "Error"},
{{"Float64", "Int128"}, "Error"},
{{"Float64", "bInt256"}, "Error"},
{{"Float64", "Int256"}, "Error"},
{{"Float64", "Float32"}, "Float64"},
{{"Float64", "Float64"}, "Float64"}
};
@ -184,13 +184,13 @@ static std::string getTypeString(DB::UInt8) { return "UInt8"; }
static std::string getTypeString(DB::UInt16) { return "UInt16"; }
static std::string getTypeString(DB::UInt32) { return "UInt32"; }
static std::string getTypeString(DB::UInt64) { return "UInt64"; }
static std::string getTypeString(DB::bUInt256) { return "bUInt256"; }
static std::string getTypeString(DB::UInt256) { return "UInt256"; }
static std::string getTypeString(DB::Int8) { return "Int8"; }
static std::string getTypeString(DB::Int16) { return "Int16"; }
static std::string getTypeString(DB::Int32) { return "Int32"; }
static std::string getTypeString(DB::Int64) { return "Int64"; }
static std::string getTypeString(DB::Int128) { return "Int128"; }
static std::string getTypeString(DB::bInt256) { return "bInt256"; }
static std::string getTypeString(DB::Int256) { return "Int256"; }
static std::string getTypeString(DB::Float32) { return "Float32"; }
static std::string getTypeString(DB::Float64) { return "Float64"; }
static std::string getTypeString(DB::NumberTraits::Error) { return "Error"; }
@ -225,13 +225,13 @@ void ifLeftType()
ifRightType<T0, DB::UInt16>();
ifRightType<T0, DB::UInt32>();
ifRightType<T0, DB::UInt64>();
ifRightType<T0, DB::bUInt256>();
ifRightType<T0, DB::UInt256>();
ifRightType<T0, DB::Int8>();
ifRightType<T0, DB::Int16>();
ifRightType<T0, DB::Int32>();
ifRightType<T0, DB::Int64>();
ifRightType<T0, DB::Int128>();
ifRightType<T0, DB::bInt256>();
ifRightType<T0, DB::Int256>();
ifRightType<T0, DB::Float32>();
ifRightType<T0, DB::Float64>();
}
@ -268,13 +268,13 @@ TEST(NumberTraits, FunctionIf)
ifLeftType<DB::UInt16>();
ifLeftType<DB::UInt32>();
ifLeftType<DB::UInt64>();
ifLeftType<DB::bUInt256>();
ifLeftType<DB::UInt256>();
ifLeftType<DB::Int8>();
ifLeftType<DB::Int16>();
ifLeftType<DB::Int32>();
ifLeftType<DB::Int64>();
ifLeftType<DB::Int128>();
ifLeftType<DB::bInt256>();
ifLeftType<DB::Int256>();
ifLeftType<DB::Float32>();
ifLeftType<DB::Float64>();
}

View File

@ -801,15 +801,15 @@ readBinary(T & x, ReadBuffer & buf) { readPODBinary(x, buf); }
inline void readBinary(String & x, ReadBuffer & buf) { readStringBinary(x, buf); }
inline void readBinary(Int128 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
inline void readBinary(UInt128 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
inline void readBinary(UInt256 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
inline void readBinary(DummyUInt256 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
inline void readBinary(Decimal32 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
inline void readBinary(Decimal64 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
inline void readBinary(Decimal128 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
inline void readBinary(Decimal256 & x, ReadBuffer & buf) { readBigIntBinary(x.value, buf); }
inline void readBinary(LocalDate & x, ReadBuffer & buf) { readPODBinary(x, buf); }
inline void readBinary(bUInt256 & x, ReadBuffer & buf) { readBigIntBinary(x, buf); }
inline void readBinary(bInt256 & x, ReadBuffer & buf) { readBigIntBinary(x, buf); }
inline void readBinary(UInt256 & x, ReadBuffer & buf) { readBigIntBinary(x, buf); }
inline void readBinary(Int256 & x, ReadBuffer & buf) { readBigIntBinary(x, buf); }
template <typename T>
inline std::enable_if_t<is_arithmetic_v<T> && (sizeof(T) <= 8), void>
@ -943,8 +943,8 @@ inline void readCSV(UUID & x, ReadBuffer & buf) { readCSVSimple(x, buf); }
*/
throw Exception("UInt128 cannot be read as a text", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
inline void readCSV(bUInt256 & x, ReadBuffer & buf) { readCSVSimple(x, buf); }
inline void readCSV(bInt256 & x, ReadBuffer & buf) { readCSVSimple(x, buf); }
inline void readCSV(UInt256 & x, ReadBuffer & buf) { readCSVSimple(x, buf); }
inline void readCSV(Int256 & x, ReadBuffer & buf) { readCSVSimple(x, buf); }
template <typename T>
void readBinary(std::vector<T> & x, ReadBuffer & buf)

View File

@ -801,7 +801,7 @@ inline void writeBinary(const String & x, WriteBuffer & buf) { writeStringBinary
inline void writeBinary(const StringRef & x, WriteBuffer & buf) { writeStringBinary(x, buf); }
inline void writeBinary(const Int128 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const UInt128 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const UInt256 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const DummyUInt256 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const Decimal32 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const Decimal64 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const Decimal128 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
@ -809,8 +809,8 @@ inline void writeBinary(const Decimal256 & x, WriteBuffer & buf) { writeBigIntBi
inline void writeBinary(const LocalDate & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const LocalDateTime & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const bUInt256 & x, WriteBuffer & buf) { writeBigIntBinary(x, buf); }
inline void writeBinary(const bInt256 & x, WriteBuffer & buf) { writeBigIntBinary(x, buf); }
inline void writeBinary(const UInt256 & x, WriteBuffer & buf) { writeBigIntBinary(x, buf); }
inline void writeBinary(const Int256 & x, WriteBuffer & buf) { writeBigIntBinary(x, buf); }
/// Methods for outputting the value in text form for a tab-separated format.
template <typename T>
@ -835,13 +835,13 @@ inline void writeText(const LocalDate & x, WriteBuffer & buf) { writeDateText(x,
inline void writeText(const LocalDateTime & x, WriteBuffer & buf) { writeDateTimeText(x, buf); }
inline void writeText(const UUID & x, WriteBuffer & buf) { writeUUIDText(x, buf); }
inline void writeText(const UInt128 & x, WriteBuffer & buf) { writeText(UUID(x), buf); }
inline void writeText(const bUInt256 & x, WriteBuffer & buf) { writeText(x.str(), buf); }
inline void writeText(const bInt256 & x, WriteBuffer & buf) { writeText(x.str(), buf); }
inline void writeText(const UInt256 & x, WriteBuffer & buf) { writeText(bigintToString(x), buf); }
inline void writeText(const Int256 & x, WriteBuffer & buf) { writeText(bigintToString(x), buf); }
template <typename T>
String decimalFractional(const T & x, UInt32 scale)
{
if constexpr (std::is_same_v<T, bInt256>)
if constexpr (std::is_same_v<T, Int256>)
{
static constexpr Int128 max_int128 = (Int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll;
@ -877,7 +877,7 @@ void writeText(Decimal<T> x, UInt32 scale, WriteBuffer & ostr)
writeChar('-', ostr); /// avoid crop leading minus when whole part is zero
}
if constexpr (std::is_same_v<T, bInt256>)
if constexpr (std::is_same_v<T, Int256>)
writeText(part, ostr);
else
writeIntText(part, ostr);
@ -919,14 +919,14 @@ inline void writeQuoted(const UUID & x, WriteBuffer & buf)
writeChar('\'', buf);
}
inline void writeQuoted(const bUInt256 & x, WriteBuffer & buf)
inline void writeQuoted(const UInt256 & x, WriteBuffer & buf)
{
writeChar('\'', buf);
writeText(x, buf);
writeChar('\'', buf);
}
inline void writeQuoted(const bInt256 & x, WriteBuffer & buf)
inline void writeQuoted(const Int256 & x, WriteBuffer & buf)
{
writeChar('\'', buf);
writeText(x, buf);

View File

@ -304,7 +304,7 @@ AggregatedDataVariants::Type Aggregator::chooseAggregationMethod()
/// into a fixed 16- or 32-byte blob.
if (std::tuple_size<KeysNullMap<UInt128>>::value + keys_bytes <= 16)
return AggregatedDataVariants::Type::nullable_keys128;
if (std::tuple_size<KeysNullMap<UInt256>>::value + keys_bytes <= 32)
if (std::tuple_size<KeysNullMap<DummyUInt256>>::value + keys_bytes <= 32)
return AggregatedDataVariants::Type::nullable_keys256;
}

View File

@ -79,7 +79,7 @@ using AggregatedDataWithShortStringKey = StringHashMap<AggregateDataPtr>;
using AggregatedDataWithStringKey = HashMapWithSavedHash<StringRef, AggregateDataPtr>;
using AggregatedDataWithKeys128 = HashMap<UInt128, AggregateDataPtr, UInt128HashCRC32>;
using AggregatedDataWithKeys256 = HashMap<UInt256, AggregateDataPtr, UInt256HashCRC32>;
using AggregatedDataWithKeys256 = HashMap<DummyUInt256, AggregateDataPtr, UInt256HashCRC32>;
using AggregatedDataWithUInt32KeyTwoLevel = TwoLevelHashMap<UInt32, AggregateDataPtr, HashCRC32<UInt32>>;
using AggregatedDataWithUInt64KeyTwoLevel = TwoLevelHashMap<UInt64, AggregateDataPtr, HashCRC32<UInt64>>;
@ -89,7 +89,7 @@ using AggregatedDataWithShortStringKeyTwoLevel = TwoLevelStringHashMap<Aggregate
using AggregatedDataWithStringKeyTwoLevel = TwoLevelHashMapWithSavedHash<StringRef, AggregateDataPtr>;
using AggregatedDataWithKeys128TwoLevel = TwoLevelHashMap<UInt128, AggregateDataPtr, UInt128HashCRC32>;
using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap<UInt256, AggregateDataPtr, UInt256HashCRC32>;
using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap<DummyUInt256, AggregateDataPtr, UInt256HashCRC32>;
/** Variants with better hash function, using more than 32 bits for hash.
* Using for merging phase of external aggregation, where number of keys may be far greater than 4 billion,
@ -101,7 +101,7 @@ using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap<UInt256, AggregateData
using AggregatedDataWithUInt64KeyHash64 = HashMap<UInt64, AggregateDataPtr, DefaultHash<UInt64>>;
using AggregatedDataWithStringKeyHash64 = HashMapWithSavedHash<StringRef, AggregateDataPtr, StringRefHash64>;
using AggregatedDataWithKeys128Hash64 = HashMap<UInt128, AggregateDataPtr, UInt128Hash>;
using AggregatedDataWithKeys256Hash64 = HashMap<UInt256, AggregateDataPtr, UInt256Hash>;
using AggregatedDataWithKeys256Hash64 = HashMap<DummyUInt256, AggregateDataPtr, UInt256Hash>;
template <typename Base>
struct AggregationDataWithNullKey : public Base

View File

@ -327,7 +327,7 @@ template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<HashJoin:
};
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<HashJoin::Type::keys256, Value, Mapped>
{
using Type = ColumnsHashing::HashMethodKeysFixed<Value, UInt256, Mapped, false, false, false>;
using Type = ColumnsHashing::HashMethodKeysFixed<Value, DummyUInt256, Mapped, false, false, false>;
};
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<HashJoin::Type::hashed, Value, Mapped>
{

View File

@ -245,7 +245,7 @@ public:
std::unique_ptr<HashMapWithSavedHash<StringRef, Mapped>> key_string;
std::unique_ptr<HashMapWithSavedHash<StringRef, Mapped>> key_fixed_string;
std::unique_ptr<HashMap<UInt128, Mapped, UInt128HashCRC32>> keys128;
std::unique_ptr<HashMap<UInt256, Mapped, UInt256HashCRC32>> keys256;
std::unique_ptr<HashMap<DummyUInt256, Mapped, UInt256HashCRC32>> keys256;
std::unique_ptr<HashMap<UInt128, Mapped, UInt128TrivialHash>> hashed;
void create(Type which)

View File

@ -18,7 +18,6 @@
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTIndexDeclaration.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTNameTypePair.h>
#include <Parsers/ASTInsertQuery.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/formatAST.h>
@ -30,11 +29,9 @@
#include <Interpreters/Context.h>
#include <Interpreters/DDLWorker.h>
#include <Interpreters/ExpressionAnalyzer.h>
#include <Interpreters/TreeRewriter.h>
#include <Interpreters/InterpreterCreateQuery.h>
#include <Interpreters/InterpreterSelectWithUnionQuery.h>
#include <Interpreters/InterpreterInsertQuery.h>
#include <Interpreters/ExpressionActions.h>
#include <Interpreters/AddDefaultDatabaseVisitor.h>
#include <Access/AccessRightsElement.h>
@ -70,6 +67,7 @@ namespace ErrorCodes
extern const int UNKNOWN_DATABASE_ENGINE;
extern const int DUPLICATE_COLUMN;
extern const int DATABASE_ALREADY_EXISTS;
extern const int BAD_ARGUMENTS;
extern const int BAD_DATABASE_FOR_TEMPORARY_TABLE;
extern const int SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY;
extern const int DICTIONARY_ALREADY_EXISTS;
@ -415,7 +413,12 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(
column.comment = col_decl.comment->as<ASTLiteral &>().value.get<String>();
if (col_decl.codec)
column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(col_decl.codec, column.type, sanity_check_compression_codecs);
{
if (col_decl.default_specifier == "ALIAS")
throw Exception{"Cannot specify codec for column type ALIAS", ErrorCodes::BAD_ARGUMENTS};
column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(
col_decl.codec, column.type, sanity_check_compression_codecs);
}
if (col_decl.ttl)
column.ttl = col_decl.ttl;

View File

@ -1,5 +1,3 @@
#include "MutationsInterpreter.h"
#include <Functions/FunctionFactory.h>
#include <Functions/IFunction.h>
#include <Interpreters/InDepthNodeVisitor.h>
@ -7,11 +5,12 @@
#include <Interpreters/MutationsInterpreter.h>
#include <Interpreters/TreeRewriter.h>
#include <Storages/MergeTree/MergeTreeData.h>
#include <DataStreams/FilterBlockInputStream.h>
#include <DataStreams/ExpressionBlockInputStream.h>
#include <DataStreams/CreatingSetsBlockInputStream.h>
#include <DataStreams/MaterializingBlockInputStream.h>
#include <DataStreams/NullBlockInputStream.h>
#include <Processors/Transforms/FilterTransform.h>
#include <Processors/Transforms/ExpressionTransform.h>
#include <Processors/Transforms/CreatingSetsTransform.h>
#include <Processors/Transforms/MaterializingTransform.h>
#include <Processors/Sources/NullSource.h>
#include <Processors/QueryPipeline.h>
#include <DataStreams/CheckSortedBlockInputStream.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTFunction.h>
@ -160,7 +159,7 @@ ColumnDependencies getAllColumnDependencies(const StorageMetadataPtr & metadata_
return dependencies;
}
};
}
bool isStorageTouchedByMutations(
StoragePtr storage,
@ -525,8 +524,10 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run)
SelectQueryOptions().analyze(/* dry_run = */ false).ignoreLimits()};
auto first_stage_header = interpreter.getSampleBlock();
auto in = std::make_shared<NullBlockInputStream>(first_stage_header);
updated_header = std::make_unique<Block>(addStreamsForLaterStages(stages_copy, in)->getHeader());
QueryPipeline pipeline;
pipeline.init(Pipe(std::make_shared<NullSource>(first_stage_header)));
addStreamsForLaterStages(stages_copy, pipeline);
updated_header = std::make_unique<Block>(pipeline.getHeader());
}
/// Special step to recalculate affected indices and TTL expressions.
@ -655,7 +656,7 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector<Stage> &
return select;
}
BlockInputStreamPtr MutationsInterpreter::addStreamsForLaterStages(const std::vector<Stage> & prepared_stages, BlockInputStreamPtr in) const
void MutationsInterpreter::addStreamsForLaterStages(const std::vector<Stage> & prepared_stages, QueryPipeline & pipeline) const
{
for (size_t i_stage = 1; i_stage < prepared_stages.size(); ++i_stage)
{
@ -667,23 +668,36 @@ BlockInputStreamPtr MutationsInterpreter::addStreamsForLaterStages(const std::ve
if (i < stage.filter_column_names.size())
{
/// Execute DELETEs.
in = std::make_shared<FilterBlockInputStream>(in, step->actions(), stage.filter_column_names[i]);
pipeline.addSimpleTransform([&](const Block & header)
{
return std::make_shared<FilterTransform>(header, step->actions(), stage.filter_column_names[i], false);
});
}
else
{
/// Execute UPDATE or final projection.
in = std::make_shared<ExpressionBlockInputStream>(in, step->actions());
pipeline.addSimpleTransform([&](const Block & header)
{
return std::make_shared<ExpressionTransform>(header, step->actions());
});
}
}
const SubqueriesForSets & subqueries_for_sets = stage.analyzer->getSubqueriesForSets();
if (!subqueries_for_sets.empty())
in = std::make_shared<CreatingSetsBlockInputStream>(in, subqueries_for_sets, context);
{
const Settings & settings = context.getSettingsRef();
SizeLimits network_transfer_limits(
settings.max_rows_to_transfer, settings.max_bytes_to_transfer, settings.transfer_overflow_mode);
pipeline.addCreatingSetsTransform(std::make_shared<CreatingSetsTransform>(
pipeline.getHeader(), subqueries_for_sets, network_transfer_limits, context));
}
}
in = std::make_shared<MaterializingBlockInputStream>(in);
return in;
pipeline.addSimpleTransform([&](const Block & header)
{
return std::make_shared<MaterializingTransform>(header);
});
}
void MutationsInterpreter::validate()
@ -705,10 +719,8 @@ void MutationsInterpreter::validate()
}
}
/// Do not use getSampleBlock in order to check the whole pipeline.
Block first_stage_header = select_interpreter->execute().getInputStream()->getHeader();
BlockInputStreamPtr in = std::make_shared<NullBlockInputStream>(first_stage_header);
addStreamsForLaterStages(stages, in)->getHeader();
auto block_io = select_interpreter->execute();
addStreamsForLaterStages(stages, block_io.pipeline);
}
BlockInputStreamPtr MutationsInterpreter::execute()
@ -716,9 +728,10 @@ BlockInputStreamPtr MutationsInterpreter::execute()
if (!can_execute)
throw Exception("Cannot execute mutations interpreter because can_execute flag set to false", ErrorCodes::LOGICAL_ERROR);
BlockInputStreamPtr in = select_interpreter->execute().getInputStream();
auto block_io = select_interpreter->execute();
addStreamsForLaterStages(stages, block_io.pipeline);
auto result_stream = addStreamsForLaterStages(stages, in);
auto result_stream = block_io.getInputStream();
/// Sometimes we update just part of columns (for example UPDATE mutation)
/// in this case we don't read sorting key, so just we don't check anything.

View File

@ -13,6 +13,7 @@ namespace DB
{
class Context;
class QueryPipeline;
/// Return false if the data isn't going to be changed by mutations.
bool isStorageTouchedByMutations(
@ -51,7 +52,7 @@ private:
struct Stage;
ASTPtr prepareInterpreterSelectQuery(std::vector<Stage> &prepared_stages, bool dry_run);
BlockInputStreamPtr addStreamsForLaterStages(const std::vector<Stage> & prepared_stages, BlockInputStreamPtr in) const;
void addStreamsForLaterStages(const std::vector<Stage> & prepared_stages, QueryPipeline & pipeline) const;
std::optional<SortDescription> getStorageSortDescriptionIfPossible(const Block & header) const;

View File

@ -123,7 +123,7 @@ typename SetVariantsTemplate<Variant>::Type SetVariantsTemplate<Variant>::choose
throw Exception{"Aggregator: keys sizes overflow", ErrorCodes::LOGICAL_ERROR};
if ((std::tuple_size<KeysNullMap<UInt128>>::value + keys_bytes) <= 16)
return Type::nullable_keys128;
if ((std::tuple_size<KeysNullMap<UInt256>>::value + keys_bytes) <= 32)
if ((std::tuple_size<KeysNullMap<DummyUInt256>>::value + keys_bytes) <= 32)
return Type::nullable_keys256;
}

View File

@ -202,12 +202,12 @@ struct NonClearableSet
std::unique_ptr<SetMethodString<HashSetWithSavedHash<StringRef>>> key_string;
std::unique_ptr<SetMethodFixedString<HashSetWithSavedHash<StringRef>>> key_fixed_string;
std::unique_ptr<SetMethodKeysFixed<HashSet<UInt128, UInt128HashCRC32>>> keys128;
std::unique_ptr<SetMethodKeysFixed<HashSet<UInt256, UInt256HashCRC32>>> keys256;
std::unique_ptr<SetMethodKeysFixed<HashSet<DummyUInt256, UInt256HashCRC32>>> keys256;
std::unique_ptr<SetMethodHashed<HashSet<UInt128, UInt128TrivialHash>>> hashed;
/// Support for nullable keys (for DISTINCT implementation).
std::unique_ptr<SetMethodKeysFixed<HashSet<UInt128, UInt128HashCRC32>, true>> nullable_keys128;
std::unique_ptr<SetMethodKeysFixed<HashSet<UInt256, UInt256HashCRC32>, true>> nullable_keys256;
std::unique_ptr<SetMethodKeysFixed<HashSet<DummyUInt256, UInt256HashCRC32>, true>> nullable_keys256;
/** Unlike Aggregator, `concat` method is not used here.
* This is done because `hashed` method, although slower, but in this case, uses less RAM.
* since when you use it, the key values themselves are not stored.
@ -224,12 +224,12 @@ struct ClearableSet
std::unique_ptr<SetMethodString<ClearableHashSetWithSavedHash<StringRef>>> key_string;
std::unique_ptr<SetMethodFixedString<ClearableHashSetWithSavedHash<StringRef>>> key_fixed_string;
std::unique_ptr<SetMethodKeysFixed<ClearableHashSet<UInt128, UInt128HashCRC32>>> keys128;
std::unique_ptr<SetMethodKeysFixed<ClearableHashSet<UInt256, UInt256HashCRC32>>> keys256;
std::unique_ptr<SetMethodKeysFixed<ClearableHashSet<DummyUInt256, UInt256HashCRC32>>> keys256;
std::unique_ptr<SetMethodHashed<ClearableHashSet<UInt128, UInt128TrivialHash>>> hashed;
/// Support for nullable keys (for DISTINCT implementation).
std::unique_ptr<SetMethodKeysFixed<ClearableHashSet<UInt128, UInt128HashCRC32>, true>> nullable_keys128;
std::unique_ptr<SetMethodKeysFixed<ClearableHashSet<UInt256, UInt256HashCRC32>, true>> nullable_keys256;
std::unique_ptr<SetMethodKeysFixed<ClearableHashSet<DummyUInt256, UInt256HashCRC32>, true>> nullable_keys256;
/** Unlike Aggregator, `concat` method is not used here.
* This is done because `hashed` method, although slower, but in this case, uses less RAM.
* since when you use it, the key values themselves are not stored.

View File

@ -469,7 +469,16 @@ void PipelineExecutor::wakeUpExecutor(size_t thread_num)
void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads)
{
executeStepImpl(thread_num, num_threads);
try
{
executeStepImpl(thread_num, num_threads);
}
catch (...)
{
/// In case of exception from executor itself, stop other threads.
finish();
throw;
}
#ifndef NDEBUG
auto & context = executor_contexts[thread_num];

View File

@ -74,8 +74,11 @@ std::optional<AlterCommand> AlterCommand::parse(const ASTAlterCommand * command_
}
if (ast_col_decl.codec)
{
if (ast_col_decl.default_specifier == "ALIAS")
throw Exception{"Cannot specify codec for column type ALIAS", ErrorCodes::BAD_ARGUMENTS};
command.codec = ast_col_decl.codec;
}
if (command_ast->column)
command.after_column = getIdentifierName(command_ast->column);

View File

@ -404,6 +404,13 @@ std::optional<ColumnDefault> ColumnsDescription::getDefault(const String & colum
}
bool ColumnsDescription::hasCompressionCodec(const String & column_name) const
{
const auto it = columns.get<1>().find(column_name);
return it != columns.get<1>().end() && it->codec != nullptr;
}
CompressionCodecPtr ColumnsDescription::getCodecOrDefault(const String & column_name, CompressionCodecPtr default_codec) const
{
const auto it = columns.get<1>().find(column_name);

View File

@ -111,6 +111,8 @@ public:
bool hasDefault(const String & column_name) const;
std::optional<ColumnDefault> getDefault(const String & column_name) const;
/// Does column has non default specified compression codec
bool hasCompressionCodec(const String & column_name) const;
CompressionCodecPtr getCodecOrDefault(const String & column_name, CompressionCodecPtr default_codec) const;
CompressionCodecPtr getCodecOrDefault(const String & column_name) const;

View File

@ -43,6 +43,7 @@ namespace
constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE = 1;
constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE_AND_TTL_INFOS = 2;
constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_TYPE = 3;
constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_DEFAULT_COMPRESSION = 4;
std::string getEndpointId(const std::string & node_id)
@ -83,7 +84,7 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo
}
/// We pretend to work as older server version, to be sure that client will correctly process our version
response.addCookie({"server_protocol_version", toString(std::min(client_protocol_version, REPLICATION_PROTOCOL_VERSION_WITH_PARTS_TYPE))});
response.addCookie({"server_protocol_version", toString(std::min(client_protocol_version, REPLICATION_PROTOCOL_VERSION_WITH_PARTS_DEFAULT_COMPRESSION))});
++total_sends;
SCOPE_EXIT({--total_sends;});
@ -115,7 +116,10 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo
if (isInMemoryPart(part))
sendPartFromMemory(part, out);
else
sendPartFromDisk(part, out);
{
bool send_default_compression_file = client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_DEFAULT_COMPRESSION;
sendPartFromDisk(part, out, send_default_compression_file);
}
}
catch (const NetException &)
{
@ -147,14 +151,20 @@ void Service::sendPartFromMemory(const MergeTreeData::DataPartPtr & part, WriteB
block_out.write(part_in_memory->block);
}
void Service::sendPartFromDisk(const MergeTreeData::DataPartPtr & part, WriteBuffer & out)
void Service::sendPartFromDisk(const MergeTreeData::DataPartPtr & part, WriteBuffer & out, bool send_default_compression_file)
{
/// We'll take a list of files from the list of checksums.
MergeTreeData::DataPart::Checksums checksums = part->checksums;
/// Add files that are not in the checksum list.
checksums.files["checksums.txt"] = {};
checksums.files["columns.txt"] = {};
auto file_names_without_checksums = part->getFileNamesWithoutChecksums();
for (const auto & file_name : file_names_without_checksums)
{
if (!send_default_compression_file && file_name == IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME)
continue;
checksums.files[file_name] = {};
}
auto disk = part->volume->getDisk();
MergeTreeData::DataPart::Checksums data_checksums;
writeBinary(checksums.files.size(), out);
@ -162,7 +172,6 @@ void Service::sendPartFromDisk(const MergeTreeData::DataPartPtr & part, WriteBuf
{
String file_name = it.first;
auto disk = part->volume->getDisk();
String path = part->getFullRelativePath() + file_name;
UInt64 size = disk->getFileSize(path);
@ -182,8 +191,7 @@ void Service::sendPartFromDisk(const MergeTreeData::DataPartPtr & part, WriteBuf
writePODBinary(hashing_out.getHash(), out);
if (file_name != "checksums.txt" &&
file_name != "columns.txt")
if (!file_names_without_checksums.count(file_name))
data_checksums.addFile(file_name, hashing_out.count(), hashing_out.getHash());
}
@ -230,7 +238,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart(
{
{"endpoint", getEndpointId(replica_path)},
{"part", part_name},
{"client_protocol_version", toString(REPLICATION_PROTOCOL_VERSION_WITH_PARTS_TYPE)},
{"client_protocol_version", toString(REPLICATION_PROTOCOL_VERSION_WITH_PARTS_DEFAULT_COMPRESSION)},
{"compress", "false"}
});
@ -308,7 +316,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToMemory(
new_data_part->minmax_idx.update(block, data.minmax_idx_columns);
new_data_part->partition.create(metadata_snapshot, block, 0);
MergedBlockOutputStream part_out(new_data_part, metadata_snapshot, block.getNamesAndTypesList(), {}, nullptr);
MergedBlockOutputStream part_out(new_data_part, metadata_snapshot, block.getNamesAndTypesList(), {}, CompressionCodecFactory::instance().get("NONE", {}));
part_out.writePrefix();
part_out.write(block);
part_out.writeSuffixAndFinalizePart(new_data_part);
@ -381,7 +389,8 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk(
ErrorCodes::CHECKSUM_DOESNT_MATCH);
if (file_name != "checksums.txt" &&
file_name != "columns.txt")
file_name != "columns.txt" &&
file_name != IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME)
checksums.addFile(file_name, file_size, expected_hash);
}

View File

@ -32,7 +32,7 @@ public:
private:
MergeTreeData::DataPartPtr findPart(const String & name);
void sendPartFromMemory(const MergeTreeData::DataPartPtr & part, WriteBuffer & out);
void sendPartFromDisk(const MergeTreeData::DataPartPtr & part, WriteBuffer & out);
void sendPartFromDisk(const MergeTreeData::DataPartPtr & part, WriteBuffer & out, bool send_default_compression_file);
private:
/// StorageReplicatedMergeTree::shutdown() waits for all parts exchange handlers to finish,

View File

@ -13,6 +13,8 @@
#include <Common/escapeForFileName.h>
#include <common/JSON.h>
#include <common/logger_useful.h>
#include <Compression/getCompressionCodecForFile.h>
#include <Parsers/queryToString.h>
namespace DB
{
@ -31,10 +33,6 @@ namespace ErrorCodes
extern const int NOT_IMPLEMENTED;
}
extern const char * DELETE_ON_DESTROY_MARKER_PATH;
static std::unique_ptr<ReadBufferFromFileBase> openForReading(const DiskPtr & disk, const String & path)
{
return disk->readFile(path, std::min(size_t(DBMS_DEFAULT_BUFFER_SIZE), disk->getFileSize(path)));
@ -418,9 +416,10 @@ void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checks
loadRowsCount(); /// Must be called after loadIndexGranularity() as it uses the value of `index_granularity`.
loadPartitionAndMinMaxIndex();
loadTTLInfos();
if (check_consistency)
checkConsistency(require_columns_checksums);
loadDefaultCompressionCodec();
}
void IMergeTreeDataPart::loadIndexGranularity()
@ -474,6 +473,89 @@ void IMergeTreeDataPart::loadIndex()
}
}
NameSet IMergeTreeDataPart::getFileNamesWithoutChecksums() const
{
if (!isStoredOnDisk())
return {};
NameSet result = {"checksums.txt", "columns.txt"};
String default_codec_path = getFullRelativePath() + DEFAULT_COMPRESSION_CODEC_FILE_NAME;
if (volume->getDisk()->exists(default_codec_path))
result.emplace(DEFAULT_COMPRESSION_CODEC_FILE_NAME);
return result;
}
void IMergeTreeDataPart::loadDefaultCompressionCodec()
{
/// In memory parts doesn't have any compression
if (!isStoredOnDisk())
{
default_codec = CompressionCodecFactory::instance().get("NONE", {});
return;
}
String path = getFullRelativePath() + DEFAULT_COMPRESSION_CODEC_FILE_NAME;
if (!volume->getDisk()->exists(path))
{
default_codec = detectDefaultCompressionCodec();
}
else
{
auto file_buf = openForReading(volume->getDisk(), path);
String codec_line;
readEscapedStringUntilEOL(codec_line, *file_buf);
ReadBufferFromString buf(codec_line);
if (!checkString("CODEC", buf))
{
LOG_WARNING(storage.log, "Cannot parse default codec for part {} from file {}, content '{}'. Default compression codec will be deduced automatically, from data on disk", name, path, codec_line);
default_codec = detectDefaultCompressionCodec();
}
try
{
ParserCodec codec_parser;
auto codec_ast = parseQuery(codec_parser, codec_line.data() + buf.getPosition(), codec_line.data() + codec_line.length(), "codec parser", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
default_codec = CompressionCodecFactory::instance().get(codec_ast, {});
}
catch (const DB::Exception & ex)
{
LOG_WARNING(storage.log, "Cannot parse default codec for part {} from file {}, content '{}', error '{}'. Default compression codec will be deduced automatically, from data on disk.", name, path, codec_line, ex.what());
default_codec = detectDefaultCompressionCodec();
}
}
}
CompressionCodecPtr IMergeTreeDataPart::detectDefaultCompressionCodec() const
{
/// In memory parts doesn't have any compression
if (!isStoredOnDisk())
return CompressionCodecFactory::instance().get("NONE", {});
auto metadata_snapshot = storage.getInMemoryMetadataPtr();
const auto & storage_columns = metadata_snapshot->getColumns();
CompressionCodecPtr result = nullptr;
for (const auto & part_column : columns)
{
/// It was compressed with default codec and it's not empty
auto column_size = getColumnSize(part_column.name, *part_column.type);
if (column_size.data_compressed != 0 && !storage_columns.hasCompressionCodec(part_column.name))
{
result = getCompressionCodecForFile(volume->getDisk(), getFullRelativePath() + getFileNameForColumn(part_column) + ".bin");
break;
}
}
if (!result)
result = CompressionCodecFactory::instance().getDefaultCodec();
return result;
}
void IMergeTreeDataPart::loadPartitionAndMinMaxIndex()
{
if (storage.format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
@ -795,7 +877,9 @@ void IMergeTreeDataPart::remove() const
for (const auto & file : {"checksums.txt", "columns.txt"})
volume->getDisk()->remove(to + "/" + file);
volume->getDisk()->removeIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_PATH);
volume->getDisk()->removeIfExists(to + "/" + DEFAULT_COMPRESSION_CODEC_FILE_NAME);
volume->getDisk()->removeIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME);
volume->getDisk()->remove(to);
}
@ -850,7 +934,7 @@ void IMergeTreeDataPart::makeCloneInDetached(const String & prefix, const Storag
/// Backup is not recursive (max_level is 0), so do not copy inner directories
localBackup(volume->getDisk(), getFullRelativePath(), destination_path, 0);
volume->getDisk()->removeIfExists(destination_path + "/" + DELETE_ON_DESTROY_MARKER_PATH);
volume->getDisk()->removeIfExists(destination_path + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME);
}
void IMergeTreeDataPart::makeCloneOnDiskDetached(const ReservationPtr & reservation) const
@ -867,7 +951,7 @@ void IMergeTreeDataPart::makeCloneOnDiskDetached(const ReservationPtr & reservat
reserved_disk->createDirectory(path_to_clone);
volume->getDisk()->copy(getFullRelativePath(), reserved_disk, path_to_clone);
volume->getDisk()->removeIfExists(path_to_clone + "/" + DELETE_ON_DESTROY_MARKER_PATH);
volume->getDisk()->removeIfExists(path_to_clone + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME);
}
void IMergeTreeDataPart::checkConsistencyBase() const

View File

@ -288,6 +288,8 @@ public:
/// Columns with values, that all have been zeroed by expired ttl
NameSet expired_columns;
CompressionCodecPtr default_codec;
/// For data in RAM ('index')
UInt64 getIndexSizeInBytes() const;
UInt64 getIndexSizeInAllocatedBytes() const;
@ -330,7 +332,20 @@ public:
String getRelativePathForPrefix(const String & prefix) const;
/// Return set of metadat file names without checksums. For example,
/// columns.txt or checksums.txt itself.
NameSet getFileNamesWithoutChecksums() const;
/// File with compression codec name which was used to compress part columns
/// by default. Some columns may have their own compression codecs, but
/// default will be stored in this file.
static inline constexpr auto DEFAULT_COMPRESSION_CODEC_FILE_NAME = "default_compression_codec.txt";
static inline constexpr auto DELETE_ON_DESTROY_MARKER_FILE_NAME = "delete-on-destroy.txt";
protected:
/// Total size of all columns, calculated once in calcuateColumnSizesOnDisk
ColumnSize total_columns_size;
@ -380,6 +395,15 @@ private:
void loadTTLInfos();
void loadPartitionAndMinMaxIndex();
/// Load default compression codec from file default_compression_codec.txt
/// if it not exists tries to deduce codec from compressed column without
/// any specifial compression.
void loadDefaultCompressionCodec();
/// Found column without specific compression and return codec
/// for this column with default parameters.
CompressionCodecPtr detectDefaultCompressionCodec() const;
};
using MergeTreeDataPartState = IMergeTreeDataPart::State;

View File

@ -113,9 +113,6 @@ namespace ErrorCodes
}
const char * DELETE_ON_DESTROY_MARKER_PATH = "delete-on-destroy.txt";
static void checkSampleExpression(const StorageInMemoryMetadata & metadata, bool allow_sampling_expression_not_in_primary_key)
{
const auto & pk_sample_block = metadata.getPrimaryKey().sample_block;
@ -781,7 +778,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
bool broken = false;
String part_path = relative_data_path + "/" + part_name;
String marker_path = part_path + "/" + DELETE_ON_DESTROY_MARKER_PATH;
String marker_path = part_path + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME;
if (part_disk_ptr->exists(marker_path))
{
LOG_WARNING(log, "Detaching stale part {}{}, which should have been deleted after a move. That can only happen after unclean restart of ClickHouse after move of a part having an operation blocking that stale copy of part.", getFullPathOnDisk(part_disk_ptr), part_name);
@ -908,6 +905,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
for (auto & part : broken_parts_to_detach)
part->renameToDetached("");
/// Delete from the set of current parts those parts that are covered by another part (those parts that
/// were merged), but that for some reason are still not deleted from the filesystem.
/// Deletion of files will be performed later in the clearOldParts() method.
@ -958,6 +956,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
calculateColumnSizesImpl();
LOG_DEBUG(log, "Loaded data parts ({} items)", data_parts_indexes.size());
}
@ -2370,7 +2369,7 @@ void MergeTreeData::swapActivePart(MergeTreeData::DataPartPtr part_copy)
modifyPartState(part_it, DataPartState::Committed);
auto disk = original_active_part->volume->getDisk();
String marker_path = original_active_part->getFullRelativePath() + DELETE_ON_DESTROY_MARKER_PATH;
String marker_path = original_active_part->getFullRelativePath() + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME;
try
{
disk->createFile(marker_path);
@ -2441,14 +2440,6 @@ static void loadPartAndFixMetadataImpl(MergeTreeData::MutableDataPartPtr part)
part->modification_time = disk->getLastModified(full_part_path).epochTime();
}
MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartAndFixMetadata(const VolumePtr & volume, const String & relative_path) const
{
MutableDataPartPtr part = createPart(Poco::Path(relative_path).getFileName(), volume, relative_path);
loadPartAndFixMetadataImpl(part);
return part;
}
void MergeTreeData::calculateColumnSizesImpl()
{
column_sizes.clear();
@ -2919,6 +2910,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const
LOG_DEBUG(log, "Checking parts");
MutableDataPartsVector loaded_parts;
loaded_parts.reserve(renamed_parts.old_and_new_names.size());
for (const auto & part_names : renamed_parts.old_and_new_names)
{
LOG_DEBUG(log, "Checking part {}", part_names.second);
@ -3286,7 +3278,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPartOnSameDisk(
LOG_DEBUG(log, "Cloning part {} to {}", fullPath(disk, src_part_path), fullPath(disk, dst_part_path));
localBackup(disk, src_part_path, dst_part_path);
disk->removeIfExists(dst_part_path + "/" + DELETE_ON_DESTROY_MARKER_PATH);
disk->removeIfExists(dst_part_path + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME);
auto single_disk_volume = std::make_shared<SingleDiskVolume>(disk->getName(), disk);
auto dst_data_part = createPart(dst_part_name, dst_part_info, single_disk_volume, tmp_dst_part_name);
@ -3377,7 +3369,7 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher(MatcherFn m
else
localBackup(part->volume->getDisk(), part->getFullRelativePath(), backup_part_path);
part->volume->getDisk()->removeIfExists(backup_part_path + "/" + DELETE_ON_DESTROY_MARKER_PATH);
part->volume->getDisk()->removeIfExists(backup_part_path + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME);
part->is_frozen.store(true, std::memory_order_relaxed);
result.push_back(PartitionCommandResultInfo{

View File

@ -534,9 +534,6 @@ public:
return DB::extractKeyExpressionList(node);
}
/// Check that the part is not broken and calculate the checksums for it if they are not present.
MutableDataPartPtr loadPartAndFixMetadata(const VolumePtr & volume, const String & relative_path) const;
/** Create local backup (snapshot) for parts with specified prefix.
* Backup is created in directory clickhouse_dir/shadow/i/, where i - incremental number,
* or if 'with_name' is specified - backup is created in directory with specified name.

View File

@ -29,6 +29,7 @@
#include <Common/interpolate.h>
#include <Common/typeid_cast.h>
#include <Common/escapeForFileName.h>
#include <Parsers/queryToString.h>
#include <cmath>
#include <ctime>
@ -1122,7 +1123,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor
/// We will modify only some of the columns. Other columns and key values can be copied as-is.
auto indices_to_recalc = getIndicesToRecalculate(in, updated_header.getNamesAndTypesList(), metadata_snapshot, context);
NameSet files_to_skip = collectFilesToSkip(updated_header, indices_to_recalc, mrk_extension);
NameSet files_to_skip = collectFilesToSkip(source_part, updated_header, indices_to_recalc, mrk_extension);
NameToNameVector files_to_rename = collectFilesForRenames(source_part, for_file_renames, mrk_extension);
if (need_remove_expired_values)
@ -1184,7 +1185,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor
}
}
finalizeMutatedPart(source_part, new_data_part, need_remove_expired_values);
finalizeMutatedPart(source_part, new_data_part, need_remove_expired_values, compression_codec);
}
return new_data_part;
@ -1448,9 +1449,12 @@ NameToNameVector MergeTreeDataMergerMutator::collectFilesForRenames(
}
NameSet MergeTreeDataMergerMutator::collectFilesToSkip(
const Block & updated_header, const std::set<MergeTreeIndexPtr> & indices_to_recalc, const String & mrk_extension)
const MergeTreeDataPartPtr & source_part,
const Block & updated_header,
const std::set<MergeTreeIndexPtr> & indices_to_recalc,
const String & mrk_extension)
{
NameSet files_to_skip = {"checksums.txt", "columns.txt"};
NameSet files_to_skip = source_part->getFileNamesWithoutChecksums();
/// Skip updated files
for (const auto & entry : updated_header)
@ -1737,7 +1741,8 @@ void MergeTreeDataMergerMutator::mutateSomePartColumns(
void MergeTreeDataMergerMutator::finalizeMutatedPart(
const MergeTreeDataPartPtr & source_part,
MergeTreeData::MutableDataPartPtr new_data_part,
bool need_remove_expired_values)
bool need_remove_expired_values,
const CompressionCodecPtr & codec)
{
auto disk = new_data_part->volume->getDisk();
if (need_remove_expired_values)
@ -1756,6 +1761,10 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart(
new_data_part->checksums.write(*out_checksums);
} /// close fd
{
auto out = disk->writeFile(new_data_part->getFullRelativePath() + IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME, 4096);
DB::writeText(queryToString(codec->getFullCodecDesc()), *out);
}
{
/// Write a file with a description of columns.
@ -1770,6 +1779,7 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart(
new_data_part->modification_time = time(nullptr);
new_data_part->setBytesOnDisk(
MergeTreeData::DataPart::calculateTotalSizeOnDisk(new_data_part->volume->getDisk(), new_data_part->getFullRelativePath()));
new_data_part->default_codec = codec;
new_data_part->calculateColumnsSizesOnDisk();
}

View File

@ -156,7 +156,11 @@ private:
/// Files, that we don't need to remove and don't need to hardlink, for example columns.txt and checksums.txt.
/// Because we will generate new versions of them after we perform mutation.
static NameSet collectFilesToSkip(const Block & updated_header, const std::set<MergeTreeIndexPtr> & indices_to_recalc, const String & mrk_extension);
static NameSet collectFilesToSkip(
const MergeTreeDataPartPtr & source_part,
const Block & updated_header,
const std::set<MergeTreeIndexPtr> & indices_to_recalc,
const String & mrk_extension);
/// Get the columns list of the resulting part in the same order as storage_columns.
static NamesAndTypesList getColumnsForNewDataPart(
@ -209,7 +213,8 @@ private:
static void finalizeMutatedPart(
const MergeTreeDataPartPtr & source_part,
MergeTreeData::MutableDataPartPtr new_data_part,
bool need_remove_expired_values);
bool need_remove_expired_values,
const CompressionCodecPtr & codec);
public :
/** Is used to cancel all merges and mutations. On cancel() call all currently running actions will throw exception soon.

View File

@ -56,7 +56,7 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter(
const NamesAndTypesList & columns_list,
const StorageMetadataPtr & metadata_snapshot,
const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
const CompressionCodecPtr & default_codec,
const CompressionCodecPtr & default_codec_,
const MergeTreeWriterSettings & writer_settings,
const MergeTreeIndexGranularity & computed_index_granularity) const
{
@ -71,7 +71,7 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter(
return std::make_unique<MergeTreeDataPartWriterCompact>(
shared_from_this(), ordered_columns_list, metadata_snapshot,
indices_to_recalc, index_granularity_info.marks_file_extension,
default_codec, writer_settings, computed_index_granularity);
default_codec_, writer_settings, computed_index_granularity);
}

View File

@ -24,6 +24,7 @@ MergeTreeDataPartInMemory::MergeTreeDataPartInMemory(
const std::optional<String> & relative_path_)
: IMergeTreeDataPart(storage_, name_, volume_, relative_path_, Type::IN_MEMORY)
{
default_codec = CompressionCodecFactory::instance().get("NONE", {});
}
MergeTreeDataPartInMemory::MergeTreeDataPartInMemory(
@ -34,6 +35,7 @@ MergeTreeDataPartInMemory::MergeTreeDataPartInMemory(
const std::optional<String> & relative_path_)
: IMergeTreeDataPart(storage_, name_, info_, volume_, relative_path_, Type::IN_MEMORY)
{
default_codec = CompressionCodecFactory::instance().get("NONE", {});
}
IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartInMemory::getReader(
@ -129,5 +131,4 @@ DataPartInMemoryPtr asInMemoryPart(const MergeTreeDataPartPtr & part)
{
return std::dynamic_pointer_cast<const MergeTreeDataPartInMemory>(part);
}
}

View File

@ -57,14 +57,14 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartWide::getWriter(
const NamesAndTypesList & columns_list,
const StorageMetadataPtr & metadata_snapshot,
const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
const CompressionCodecPtr & default_codec,
const CompressionCodecPtr & default_codec_,
const MergeTreeWriterSettings & writer_settings,
const MergeTreeIndexGranularity & computed_index_granularity) const
{
return std::make_unique<MergeTreeDataPartWriterWide>(
shared_from_this(), columns_list, metadata_snapshot, indices_to_recalc,
index_granularity_info.marks_file_extension,
default_codec, writer_settings, computed_index_granularity);
default_codec_, writer_settings, computed_index_granularity);
}

View File

@ -151,7 +151,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeWriteAheadLog::restore(const Stor
if (action_type == ActionType::ADD_PART)
{
MergedBlockOutputStream part_out(part, metadata_snapshot, block.getNamesAndTypesList(), {}, nullptr);
MergedBlockOutputStream part_out(part, metadata_snapshot, block.getNamesAndTypesList(), {}, CompressionCodecFactory::instance().get("NONE", {}));
part->minmax_idx.update(block, storage.minmax_idx_columns);
part->partition.create(metadata_snapshot, block, 0);

View File

@ -1,6 +1,7 @@
#include <Storages/MergeTree/MergedBlockOutputStream.h>
#include <Interpreters/Context.h>
#include <Poco/File.h>
#include <Parsers/queryToString.h>
namespace DB
@ -18,14 +19,14 @@ MergedBlockOutputStream::MergedBlockOutputStream(
const StorageMetadataPtr & metadata_snapshot_,
const NamesAndTypesList & columns_list_,
const MergeTreeIndices & skip_indices,
CompressionCodecPtr default_codec,
CompressionCodecPtr default_codec_,
bool blocks_are_granules_size)
: MergedBlockOutputStream(
data_part,
metadata_snapshot_,
columns_list_,
skip_indices,
default_codec,
default_codec_,
{},
data_part->storage.global_context.getSettings().min_bytes_to_use_direct_io,
blocks_are_granules_size)
@ -37,12 +38,13 @@ MergedBlockOutputStream::MergedBlockOutputStream(
const StorageMetadataPtr & metadata_snapshot_,
const NamesAndTypesList & columns_list_,
const MergeTreeIndices & skip_indices,
CompressionCodecPtr default_codec,
CompressionCodecPtr default_codec_,
const MergeTreeData::DataPart::ColumnToSize & merged_column_to_size,
size_t aio_threshold,
bool blocks_are_granules_size)
: IMergedBlockOutputStream(data_part, metadata_snapshot_)
, columns_list(columns_list_)
, default_codec(default_codec_)
{
MergeTreeWriterSettings writer_settings(
storage.global_context.getSettings(),
@ -120,6 +122,8 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart(
new_part->setBytesOnDisk(checksums.getTotalSizeOnDisk());
new_part->index_granularity = writer->getIndexGranularity();
new_part->calculateColumnsSizesOnDisk();
if (default_codec != nullptr)
new_part->default_codec = default_codec;
}
void MergedBlockOutputStream::finalizePartOnDisk(
@ -162,6 +166,17 @@ void MergedBlockOutputStream::finalizePartOnDisk(
part_columns.writeText(*out);
}
if (default_codec != nullptr)
{
auto out = volume->getDisk()->writeFile(part_path + IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME, 4096);
DB::writeText(queryToString(default_codec->getFullCodecDesc()), *out);
}
else
{
throw Exception("Compression codec have to be specified for part on disk, empty for" + new_part->name
+ ". It is a bug.", ErrorCodes::LOGICAL_ERROR);
}
{
/// Write file with checksums.
auto out = volume->getDisk()->writeFile(part_path + "checksums.txt", 4096);

View File

@ -18,7 +18,7 @@ public:
const StorageMetadataPtr & metadata_snapshot_,
const NamesAndTypesList & columns_list_,
const MergeTreeIndices & skip_indices,
CompressionCodecPtr default_codec,
CompressionCodecPtr default_codec_,
bool blocks_are_granules_size = false);
MergedBlockOutputStream(
@ -26,7 +26,7 @@ public:
const StorageMetadataPtr & metadata_snapshot_,
const NamesAndTypesList & columns_list_,
const MergeTreeIndices & skip_indices,
CompressionCodecPtr default_codec,
CompressionCodecPtr default_codec_,
const MergeTreeData::DataPart::ColumnToSize & merged_column_to_size,
size_t aio_threshold,
bool blocks_are_granules_size = false);
@ -64,6 +64,7 @@ private:
NamesAndTypesList columns_list;
IMergeTreeDataPart::MinMaxIndex minmax_idx;
size_t rows_count = 0;
CompressionCodecPtr default_codec;
};
}

Some files were not shown because too many files have changed in this diff Show More