thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 23:52:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #49635 from ClickHouse/remove-commented-code

Browse Source 
Remove commented code
This commit is contained in:
Alexey Milovidov 2023-05-08 06:06:12 +03:00 committed by GitHub
commit ad025fb4f6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 84 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

142
src/Processors/Transforms/WindowTransform.cpp
View File

@ -19,6 +19,30 @@
#include <DataTypes/DataTypeDateTime64.h>
/// See https://fmt.dev/latest/api.html#formatting-user-defined-types
template <>
struct fmt::formatter<DB::RowNumber>
{
static constexpr auto parse(format_parse_context & ctx)
{
const auto * it = ctx.begin();
const auto * end = ctx.end();
/// Only support {}.
if (it != end && *it != '}')
throw fmt::format_error("Invalid format");
return it;
}
template <typename FormatContext>
auto format(const DB::RowNumber & x, FormatContext & ctx)
{
return fmt::format_to(ctx.out(), "{}:{}", x.block, x.row);
}
};
namespace DB
{
@ -34,7 +58,7 @@ namespace ErrorCodes
// Interface for true window functions. It's not much of an interface, they just
// accept the guts of WindowTransform and do 'something'. Given a small number of
// true window functions, and the fact that the WindowTransform internals are
// pretty much well defined in domain terms (e.g. frame boundaries), this is
// pretty much well-defined in domain terms (e.g. frame boundaries), this is
// somewhat acceptable.
class IWindowFunction
{
@ -323,8 +347,6 @@ void WindowTransform::advancePartitionEnd()
const RowNumber end = blocksEnd();
// fmt::print(stderr, "end {}, partition_end {}\n", end, partition_end);
// If we're at the total end of data, we must end the partition. This is one
// of the few places in calculations where we need special handling for end
// of data, other places will work as usual based on
@ -383,9 +405,6 @@ void WindowTransform::advancePartitionEnd()
const auto block_rows = blockRowsNumber(partition_end);
for (; partition_end.row < block_rows; ++partition_end.row)
{
// fmt::print(stderr, "compare reference '{}' to compared '{}'\n",
// prev_frame_start, partition_end);
size_t i = 0;
for (; i < partition_by_columns; ++i)
{
@ -394,9 +413,6 @@ void WindowTransform::advancePartitionEnd()
const auto * compared_column
= inputAt(partition_end)[partition_by_indices[i]].get();
// fmt::print(stderr, "reference '{}', compared '{}'\n",
// (*reference_column)[prev_frame_start.row],
// (*compared_column)[partition_end.row]);
if (compared_column->compareAt(partition_end.row,
prev_frame_start.row, *reference_column,
1 /* nan_direction_hint */) != 0)
@ -421,26 +437,26 @@ void WindowTransform::advancePartitionEnd()
assert(!partition_ended && partition_end == blocksEnd());
}
auto WindowTransform::moveRowNumberNoCheck(const RowNumber & _x, int64_t offset) const
auto WindowTransform::moveRowNumberNoCheck(const RowNumber & original_row_number, Int64 offset) const
{
RowNumber x = _x;
RowNumber moved_row_number = original_row_number;
if (offset > 0 && x != blocksEnd())
if (offset > 0 && moved_row_number != blocksEnd())
{
for (;;)
{
assertValid(x);
assertValid(moved_row_number);
assert(offset >= 0);
const auto block_rows = blockRowsNumber(x);
x.row += offset;
if (x.row >= block_rows)
const auto block_rows = blockRowsNumber(moved_row_number);
moved_row_number.row += offset;
if (moved_row_number.row >= block_rows)
{
offset = x.row - block_rows;
x.row = 0;
x.block++;
offset = moved_row_number.row - block_rows;
moved_row_number.row = 0;
++moved_row_number.block;
if (x == blocksEnd())
if (moved_row_number == blocksEnd())
{
break;
}
@ -456,56 +472,55 @@ auto WindowTransform::moveRowNumberNoCheck(const RowNumber & _x, int64_t offset)
{
for (;;)
{
assertValid(x);
assertValid(moved_row_number);
assert(offset <= 0);
// abs(offset) is less than INT64_MAX, as checked in the parser, so
// this negation should always work.
assert(offset >= -INT64_MAX);
if (x.row >= static_cast<uint64_t>(-offset))
if (moved_row_number.row >= static_cast<UInt64>(-offset))
{
x.row -= -offset;
moved_row_number.row -= -offset;
offset = 0;
break;
}
// Move to the first row in current block. Note that the offset is
// negative.
offset += x.row;
x.row = 0;
offset += moved_row_number.row;
moved_row_number.row = 0;
// Move to the last row of the previous block, if we are not at the
// first one. Offset also is incremented by one, because we pass over
// the first row of this block.
if (x.block == first_block_number)
if (moved_row_number.block == first_block_number)
{
break;
}
--x.block;
--moved_row_number.block;
offset += 1;
x.row = blockRowsNumber(x) - 1;
moved_row_number.row = blockRowsNumber(moved_row_number) - 1;
}
}
return std::tuple<RowNumber, int64_t>{x, offset};
return std::tuple<RowNumber, Int64>{moved_row_number, offset};
}
auto WindowTransform::moveRowNumber(const RowNumber & _x, int64_t offset) const
auto WindowTransform::moveRowNumber(const RowNumber & original_row_number, Int64 offset) const
{
auto [x, o] = moveRowNumberNoCheck(_x, offset);
auto [moved_row_number, offset_after_move] = moveRowNumberNoCheck(original_row_number, offset);
#ifndef NDEBUG
// Check that it was reversible.
auto [xx, oo] = moveRowNumberNoCheck(x, -(offset - o));
/// Check that it was reversible. If we move back, we get the original row number with zero offset.
const auto [original_row_number_to_validate, offset_after_move_back]
= moveRowNumberNoCheck(moved_row_number, -(offset - offset_after_move));
// fmt::print(stderr, "{} -> {}, result {}, {}, new offset {}, twice {}, {}\n",
// _x, offset, x, o, -(offset - o), xx, oo);
assert(xx == _x);
assert(oo == 0);
assert(original_row_number_to_validate == original_row_number);
assert(0 == offset_after_move_back);
#endif
return std::tuple<RowNumber, int64_t>{x, o};
return std::tuple<RowNumber, Int64>{moved_row_number, offset_after_move};
}
@ -520,9 +535,6 @@ void WindowTransform::advanceFrameStartRowsOffset()
assertValid(frame_start);
// fmt::print(stderr, "frame start {} left {} partition start {}\n",
// frame_start, offset_left, partition_start);
if (frame_start <= partition_start)
{
// Got to the beginning of partition and can't go further back.
@ -685,8 +697,6 @@ bool WindowTransform::arePeers(const RowNumber & x, const RowNumber & y) const
void WindowTransform::advanceFrameEndCurrentRow()
{
// fmt::print(stderr, "starting from frame_end {}\n", frame_end);
// We only process one block here, and frame_end must be already in it: if
// we didn't find the end in the previous block, frame_end is now the first
// row of the current block. We need this knowledge to write a simpler loop
@ -708,7 +718,7 @@ void WindowTransform::advanceFrameEndCurrentRow()
// We advance until the partition end. It's either in the current block or
// in the next one, which is also the past-the-end block. Figure out how
// many rows we have to process.
uint64_t rows_end;
UInt64 rows_end;
if (partition_end.row == 0)
{
assert(partition_end == blocksEnd());
@ -722,14 +732,11 @@ void WindowTransform::advanceFrameEndCurrentRow()
// Equality would mean "no data to process", for which we checked above.
assert(frame_end.row < rows_end);
// fmt::print(stderr, "first row {} last {}\n", frame_end.row, rows_end);
// Advance frame_end while it is still peers with the current row.
for (; frame_end.row < rows_end; ++frame_end.row)
{
if (!arePeers(current_row, frame_end))
{
// fmt::print(stderr, "{} and {} don't match\n", reference, frame_end);
frame_ended = true;
return;
}
@ -852,8 +859,6 @@ void WindowTransform::advanceFrameEnd()
break;
}
// fmt::print(stderr, "frame_end {} -> {}\n", frame_end_before, frame_end);
// We might not have advanced the frame end if we found out we reached the
// end of input or the partition, or if we still don't know the frame start.
if (frame_end_before == frame_end)
@ -865,9 +870,6 @@ void WindowTransform::advanceFrameEnd()
// Update the aggregation states after the frame has changed.
void WindowTransform::updateAggregationState()
{
// fmt::print(stderr, "update agg states [{}, {}) -> [{}, {})\n",
// prev_frame_start, prev_frame_end, frame_start, frame_end);
// Assert that the frame boundaries are known, have proper order wrt each
// other, and have not gone back wrt the previous frame.
assert(frame_started);
@ -915,7 +917,6 @@ void WindowTransform::updateAggregationState()
if (reset_aggregation)
{
// fmt::print(stderr, "(2) reset aggregation\n");
a->destroy(buf);
a->create(buf);
}
@ -991,9 +992,6 @@ void WindowTransform::writeOutCurrentRow()
a->insertMergeResultInto(buf, *result_column, arena.get());
}
}
// fmt::print(stderr, "wrote out aggregation state for current row '{}'\n",
// current_row);
}
static void assertSameColumns(const Columns & left_all,
@ -1030,10 +1028,6 @@ static void assertSameColumns(const Columns & left_all,
void WindowTransform::appendChunk(Chunk & chunk)
{
// fmt::print(stderr, "new chunk, {} rows, finished={}\n", chunk.getNumRows(),
// input_is_finished);
// fmt::print(stderr, "chunk structure '{}'\n", chunk.dumpStructure());
// First, prepare the new input block and add it to the queue. We might not
// have it if it's end of data, though.
if (!input_is_finished)
@ -1093,9 +1087,6 @@ void WindowTransform::appendChunk(Chunk & chunk)
for (;;)
{
advancePartitionEnd();
// fmt::print(stderr, "partition [{}, {}), {}\n",
// partition_start, partition_end, partition_ended);
// Either we ran out of data or we found the end of partition (maybe
// both, but this only happens at the total end of data).
assert(partition_ended || partition_end == blocksEnd());
@ -1109,10 +1100,6 @@ void WindowTransform::appendChunk(Chunk & chunk)
// which is precisely the definition of `partition_end`.
while (current_row < partition_end)
{
// fmt::print(stderr, "(1) row {} frame [{}, {}) {}, {}\n",
// current_row, frame_start, frame_end,
// frame_started, frame_ended);
// We now know that the current row is valid, so we can update the
// peer group start.
if (!arePeers(peer_group_start, current_row))
@ -1152,10 +1139,6 @@ void WindowTransform::appendChunk(Chunk & chunk)
return;
}
// fmt::print(stderr, "(2) row {} frame [{}, {}) {}, {}\n",
// current_row, frame_start, frame_end,
// frame_started, frame_ended);
// The frame can be empty sometimes, e.g. the boundaries coincide
// or the start is after the partition end. But hopefully start is
// not after end.
@ -1236,8 +1219,6 @@ void WindowTransform::appendChunk(Chunk & chunk)
peer_group_start_row_number = 1;
peer_group_number = 1;
// fmt::print(stderr, "reinitialize agg data at start of {}\n",
// partition_start);
// Reinitialize the aggregate function states because the new partition
// has started.
for (auto & ws : workspaces)
@ -1278,10 +1259,6 @@ void WindowTransform::appendChunk(Chunk & chunk)
IProcessor::Status WindowTransform::prepare()
{
// fmt::print(stderr, "prepare, next output {}, not ready row {}, first block {}, hold {} blocks\n",
// next_output_block_number, first_not_ready_row, first_block_number,
// blocks.size());
if (output.isFinished() || isCancelled())
{
// The consumer asked us not to continue (or we decided it ourselves),
@ -1325,10 +1302,6 @@ IProcessor::Status WindowTransform::prepare()
}
output_data.chunk.setColumns(columns, block.rows);
// fmt::print(stderr, "output block {} as chunk '{}'\n",
// next_output_block_number,
// output_data.chunk.dumpStructure());
++next_output_block_number;
output.pushData(std::move(output_data));
@ -1428,9 +1401,6 @@ void WindowTransform::work()
std::min(prev_frame_start.block, current_row.block));
if (first_block_number < first_used_block)
{
// fmt::print(stderr, "will drop blocks from {} to {}\n", first_block_number,
// first_used_block);
blocks.erase(blocks.begin(),
blocks.begin() + (first_used_block - first_block_number));
first_block_number = first_used_block;
@ -2196,7 +2166,7 @@ struct WindowFunctionLagLeadInFrame final : public WindowFunction
IColumn & to = *current_block.output_columns[function_index];
const auto & workspace = transform->workspaces[function_index];
int64_t offset = 1;
Int64 offset = 1;
if (argument_types.size() > 1)
{
offset = (*current_block.input_columns[
@ -2286,7 +2256,7 @@ struct WindowFunctionNthValue final : public WindowFunction
IColumn & to = *current_block.output_columns[function_index];
const auto & workspace = transform->workspaces[function_index];
int64_t offset = (*current_block.input_columns[
Int64 offset = (*current_block.input_columns[
workspace.argument_column_indices[1]])[
transform->current_row.row].get<Int64>();

96
src/Processors/Transforms/WindowTransform.h
View File

@ -8,6 +8,10 @@
#include <deque>
/// See https://stackoverflow.com/questions/72533435/error-zero-as-null-pointer-constant-while-comparing-template-class-using-spaces
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
namespace DB
{
@ -34,7 +38,7 @@ struct WindowFunctionWorkspace
// Argument columns. Be careful, this is a per-block cache.
std::vector<const IColumn *> argument_columns;
uint64_t cached_block_number = std::numeric_limits<uint64_t>::max();
UInt64 cached_block_number = std::numeric_limits<UInt64>::max();
};
struct WindowTransformBlock
@ -48,28 +52,14 @@ struct WindowTransformBlock
struct RowNumber
{
uint64_t block = 0;
uint64_t row = 0;
UInt64 block = 0;
UInt64 row = 0;
bool operator < (const RowNumber & other) const
{
return block < other.block
|| (block == other.block && row < other.row);
}
bool operator == (const RowNumber & other) const
{
return block == other.block && row == other.row;
}
bool operator <= (const RowNumber & other) const
{
return *this < other || *this == other;
}
auto operator <=>(const RowNumber &) const = default;
};
/*
* Computes several window functions that share the same window. The input must
/* Computes several window functions that share the same window. The input must
* be sorted by PARTITION BY (in any order), then by ORDER BY.
* We need to track the following pointers:
* 1) boundaries of partition -- rows that compare equal w/PARTITION BY.
@ -103,19 +93,16 @@ public:
static Block transformHeader(Block header, const ExpressionActionsPtr & expression);
/*
* (former) Implementation of ISimpleTransform.
/* (former) Implementation of ISimpleTransform.
*/
void appendChunk(Chunk & chunk) /*override*/;
/*
* Implementation of IProcessor;
/* Implementation of IProcessor;
*/
Status prepare() override;
void work() override;
/*
* Implementation details.
/* Implementation details.
*/
void advancePartitionEnd();
@ -146,14 +133,14 @@ public:
return const_cast<WindowTransform *>(this)->inputAt(x);
}
auto & blockAt(const uint64_t block_number)
auto & blockAt(const UInt64 block_number)
{
assert(block_number >= first_block_number);
assert(block_number - first_block_number < blocks.size());
return blocks[block_number - first_block_number];
}
const auto & blockAt(const uint64_t block_number) const
const auto & blockAt(const UInt64 block_number) const
{
return const_cast<WindowTransform *>(this)->blockAt(block_number);
}
@ -188,7 +175,7 @@ public:
const auto block_rows = blockAt(x).rows;
assert(x.row < block_rows);
x.row++;
++x.row;
if (x.row < block_rows)
{
return;
@ -237,20 +224,16 @@ public:
return result;
}
auto moveRowNumber(const RowNumber & _x, int64_t offset) const;
auto moveRowNumberNoCheck(const RowNumber & _x, int64_t offset) const;
auto moveRowNumber(const RowNumber & original_row_number, Int64 offset) const;
auto moveRowNumberNoCheck(const RowNumber & original_row_number, Int64 offset) const;
void assertValid(const RowNumber & x) const
{
assert(x.block >= first_block_number);
if (x.block == first_block_number + blocks.size())
{
assert(x.row == 0);
}
else
{
assert(x.row < blockRowsNumber(x));
}
}
RowNumber blocksEnd() const
@ -263,8 +246,7 @@ public:
return RowNumber{first_block_number, 0};
}
/*
* Data (formerly) inherited from ISimpleTransform, needed for the
/* Data (formerly) inherited from ISimpleTransform, needed for the
* implementation of the IProcessor interface.
*/
InputPort & input;
@ -276,8 +258,7 @@ public:
bool has_output = false;
Port::Data output_data;
/*
* Data for window transform itself.
/* Data for window transform itself.
*/
Block input_header;
@ -300,9 +281,9 @@ public:
// have an always-incrementing index. The index of the first block is in
// `first_block_number`.
std::deque<WindowTransformBlock> blocks;
uint64_t first_block_number = 0;
UInt64 first_block_number = 0;
// The next block we are going to pass to the consumer.
uint64_t next_output_block_number = 0;
UInt64 next_output_block_number = 0;
// The first row for which we still haven't calculated the window functions.
// Used to determine which resulting blocks we can pass to the consumer.
RowNumber first_not_ready_row;
@ -326,9 +307,9 @@ public:
RowNumber peer_group_start;
// Row and group numbers in partition for calculating rank() and friends.
uint64_t current_row_number = 1;
uint64_t peer_group_start_row_number = 1;
uint64_t peer_group_number = 1;
UInt64 current_row_number = 1;
UInt64 peer_group_start_row_number = 1;
UInt64 peer_group_number = 1;
// The frame is [frame_start, frame_end) if frame_ended && frame_started,
// and unknown otherwise. Note that when we move to the next row, both the
@ -353,34 +334,13 @@ public:
// Comparison function for RANGE OFFSET frames. We choose the appropriate
// overload once, based on the type of the ORDER BY column. Choosing it for
// each row would be slow.
int (* compare_values_with_offset) (
std::function<int(
const IColumn * compared_column, size_t compared_row,
const IColumn * reference_column, size_t reference_row,
const Field & offset,
bool offset_is_preceding);
bool offset_is_preceding)> compare_values_with_offset;
};
}
/// See https://fmt.dev/latest/api.html#formatting-user-defined-types
template <>
struct fmt::formatter<DB::RowNumber>
{
static constexpr auto parse(format_parse_context & ctx)
{
const auto * it = ctx.begin();
const auto * end = ctx.end();
/// Only support {}.
if (it != end && *it != '}')
throw fmt::format_error("invalid format");
return it;
}
template <typename FormatContext>
auto format(const DB::RowNumber & x, FormatContext & ctx)
{
return fmt::format_to(ctx.out(), "{}:{}", x.block, x.row);
}
};
#pragma clang diagnostic pop