mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-26 09:32:01 +00:00
Merge pull request #60428 from ClickHouse/revert-58140-storage-merge-tree-index
Revert "Add table function `mergeTreeIndex`"
This commit is contained in:
Alexander Tokmakov
GitHub
commit
710cdeb76a
@ -1,83 +0,0 @@
|
||||
---
|
||||
slug: /en/sql-reference/table-functions/mergeTreeIndex
|
||||
sidebar_position: 77
|
||||
sidebar_label: mergeTreeIndex
|
||||
---
|
||||
|
||||
# mergeTreeIndex
|
||||
|
||||
Represents the contents of index and marks files of MergeTree tables. It can be used for introspection
|
||||
|
||||
``` sql
|
||||
mergeTreeIndex(database, table, [with_marks = true])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `database`- The database name to read index and marks from.
|
||||
- `table`- The table name to read index and marks from.
|
||||
- `with_marks` - Whether include columns with marks to the result.
|
||||
|
||||
**Returned Value**
|
||||
|
||||
A table object with columns with values of primary index of source table, columns with values of marks (if enabled) for all possible files in data parts of source table and virtual columns:
|
||||
|
||||
- `part_name` - The name of data part.
|
||||
- `mark_number` - The number of current mark in data part.
|
||||
- `rows_in_granule` - The number of rows in current granule.
|
||||
|
||||
Marks column may contain `(NULL, NULL)` value in case when column is absent in data part or marks for one of its substreams are not written (e.g. in compact parts).
|
||||
|
||||
## Usage Example
|
||||
|
||||
```sql
|
||||
CREATE TABLE test_table
|
||||
(
|
||||
`id` UInt64,
|
||||
`n` UInt64,
|
||||
`arr` Array(UInt64)
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY id
|
||||
SETTINGS index_granularity = 3, min_bytes_for_wide_part = 0, min_rows_for_wide_part = 8;
|
||||
|
||||
INSERT INTO test_table SELECT number, number, range(number % 5) FROM numbers(5);
|
||||
|
||||
INSERT INTO test_table SELECT number, number, range(number % 5) FROM numbers(10, 10);
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT * FROM mergeTreeIndex(currentDatabase(), test_table, with_marks = true);
|
||||
```
|
||||
|
||||
```text
|
||||
┌─part_name─┬─mark_number─┬─rows_in_granule─┬─id─┬─id.mark─┬─n.mark──┬─arr.size0.mark─┬─arr.mark─┐
|
||||
│ all_1_1_0 │ 0 │ 3 │ 0 │ (0,0) │ (42,0) │ (NULL,NULL) │ (84,0) │
|
||||
│ all_1_1_0 │ 1 │ 2 │ 3 │ (133,0) │ (172,0) │ (NULL,NULL) │ (211,0) │
|
||||
│ all_1_1_0 │ 2 │ 0 │ 4 │ (271,0) │ (271,0) │ (NULL,NULL) │ (271,0) │
|
||||
└───────────┴─────────────┴─────────────────┴────┴─────────┴─────────┴────────────────┴──────────┘
|
||||
┌─part_name─┬─mark_number─┬─rows_in_granule─┬─id─┬─id.mark─┬─n.mark─┬─arr.size0.mark─┬─arr.mark─┐
|
||||
│ all_2_2_0 │ 0 │ 3 │ 10 │ (0,0) │ (0,0) │ (0,0) │ (0,0) │
|
||||
│ all_2_2_0 │ 1 │ 3 │ 13 │ (0,24) │ (0,24) │ (0,24) │ (0,24) │
|
||||
│ all_2_2_0 │ 2 │ 3 │ 16 │ (0,48) │ (0,48) │ (0,48) │ (0,80) │
|
||||
│ all_2_2_0 │ 3 │ 1 │ 19 │ (0,72) │ (0,72) │ (0,72) │ (0,128) │
|
||||
│ all_2_2_0 │ 4 │ 0 │ 19 │ (0,80) │ (0,80) │ (0,80) │ (0,160) │
|
||||
└───────────┴─────────────┴─────────────────┴────┴─────────┴────────┴────────────────┴──────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
DESCRIBE mergeTreeIndex(currentDatabase(), test_table, with_marks = true) SETTINGS describe_compact_output = 1;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─name────────────┬─type─────────────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ part_name │ String │
|
||||
│ mark_number │ UInt64 │
|
||||
│ rows_in_granule │ UInt64 │
|
||||
│ id │ UInt64 │
|
||||
│ id.mark │ Tuple(offset_in_compressed_file Nullable(UInt64), offset_in_decompressed_block Nullable(UInt64)) │
|
||||
│ n.mark │ Tuple(offset_in_compressed_file Nullable(UInt64), offset_in_decompressed_block Nullable(UInt64)) │
|
||||
│ arr.size0.mark │ Tuple(offset_in_compressed_file Nullable(UInt64), offset_in_decompressed_block Nullable(UInt64)) │
|
||||
│ arr.mark │ Tuple(offset_in_compressed_file Nullable(UInt64), offset_in_decompressed_block Nullable(UInt64)) │
|
||||
└─────────────────┴──────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
@ -5,7 +5,6 @@
|
||||
#include <Storages/checkAndGetLiteralArgument.h>
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <Parsers/queryToString.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -62,31 +61,8 @@ namespace
|
||||
auto value = literal_value->as<ASTLiteral>()->value;
|
||||
return std::pair{key, Field(value)};
|
||||
}
|
||||
|
||||
std::pair<String, Field> getKeyValueFromAST(ASTPtr ast, ContextPtr context)
|
||||
{
|
||||
auto res = getKeyValueFromAST(ast, true, context);
|
||||
|
||||
if (!res || !std::holds_alternative<Field>(res->second))
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Failed to get key value from ast '{}'", queryToString(ast));
|
||||
|
||||
return {res->first, std::get<Field>(res->second)};
|
||||
}
|
||||
}
|
||||
|
||||
std::map<String, Field> getParamsMapFromAST(ASTs asts, ContextPtr context)
|
||||
{
|
||||
std::map<String, Field> params;
|
||||
for (const auto & ast : asts)
|
||||
{
|
||||
auto [key, value] = getKeyValueFromAST(ast, context);
|
||||
bool inserted = params.emplace(key, value).second;
|
||||
if (!inserted)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Duplicated key '{}' in params", key);
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(
|
||||
ASTs asts, ContextPtr context, bool throw_unknown_collection, std::vector<std::pair<std::string, ASTPtr>> * complex_args)
|
||||
|
||||
@ -21,16 +21,10 @@ namespace DB
|
||||
/// Table engines have collection name as first argument of ast and other arguments are key-value overrides.
|
||||
MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(
|
||||
ASTs asts, ContextPtr context, bool throw_unknown_collection = true, std::vector<std::pair<std::string, ASTPtr>> * complex_args = nullptr);
|
||||
|
||||
/// Helper function to get named collection for dictionary source.
|
||||
/// Dictionaries have collection name as name argument of dict configuration and other arguments are overrides.
|
||||
MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context);
|
||||
|
||||
/// Parses asts as key value pairs and returns a map of them.
|
||||
/// If key or value cannot be parsed as literal or interpreted
|
||||
/// as constant expression throws an exception.
|
||||
std::map<String, Field> getParamsMapFromAST(ASTs asts, ContextPtr context);
|
||||
|
||||
HTTPHeaderEntries getHeadersFromNamedCollection(const NamedCollection & collection);
|
||||
|
||||
struct ExternalDatabaseEqualKeysSet
|
||||
|
||||
@ -1,310 +0,0 @@
|
||||
|
||||
#include <Storages/StorageMergeTreeIndex.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Columns/ColumnNullable.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/NestedUtils.h>
|
||||
#include <Parsers/ASTSelectQuery.h>
|
||||
#include <Storages/ColumnsDescription.h>
|
||||
#include <Storages/MergeTree/LoadedMergeTreeDataPartInfoForReader.h>
|
||||
#include <Storages/MergeTree/MergeTreeDataPartCompact.h>
|
||||
#include <Storages/MergeTree/MergeTreeMarksLoader.h>
|
||||
#include <Storages/VirtualColumnUtils.h>
|
||||
#include <Access/Common/AccessFlags.h>
|
||||
#include <Common/HashTable/HashSet.h>
|
||||
#include <Common/escapeForFileName.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int BAD_ARGUMENTS;
|
||||
extern const int NO_SUCH_COLUMN_IN_TABLE;
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
class MergeTreeIndexSource : public ISource, WithContext
|
||||
{
|
||||
public:
|
||||
MergeTreeIndexSource(
|
||||
Block header_,
|
||||
Block index_header_,
|
||||
MergeTreeData::DataPartsVector data_parts_,
|
||||
ContextPtr context_,
|
||||
bool with_marks_)
|
||||
: ISource(header_)
|
||||
, WithContext(context_)
|
||||
, header(std::move(header_))
|
||||
, index_header(std::move(index_header_))
|
||||
, data_parts(std::move(data_parts_))
|
||||
, with_marks(with_marks_)
|
||||
{
|
||||
}
|
||||
|
||||
String getName() const override { return "MergeTreeIndex"; }
|
||||
|
||||
protected:
|
||||
Chunk generate() override
|
||||
{
|
||||
if (part_index >= data_parts.size())
|
||||
return {};
|
||||
|
||||
const auto & part = data_parts[part_index];
|
||||
const auto & index_granularity = part->index_granularity;
|
||||
|
||||
std::shared_ptr<MergeTreeMarksLoader> marks_loader;
|
||||
if (with_marks && isCompactPart(part))
|
||||
marks_loader = createMarksLoader(part, MergeTreeDataPartCompact::DATA_FILE_NAME, part->getColumns().size());
|
||||
|
||||
size_t num_columns = header.columns();
|
||||
size_t num_rows = index_granularity.getMarksCount();
|
||||
|
||||
const auto & part_name_column = StorageMergeTreeIndex::part_name_column;
|
||||
const auto & mark_number_column = StorageMergeTreeIndex::mark_number_column;
|
||||
const auto & rows_in_granule_column = StorageMergeTreeIndex::rows_in_granule_column;
|
||||
|
||||
const auto & index = part->getIndex();
|
||||
Columns result_columns(num_columns);
|
||||
for (size_t pos = 0; pos < num_columns; ++pos)
|
||||
{
|
||||
const auto & column_name = header.getByPosition(pos).name;
|
||||
const auto & column_type = header.getByPosition(pos).type;
|
||||
|
||||
if (index_header.has(column_name))
|
||||
{
|
||||
size_t index_position = index_header.getPositionByName(column_name);
|
||||
result_columns[pos] = index[index_position];
|
||||
}
|
||||
else if (column_name == part_name_column.name)
|
||||
{
|
||||
auto column = column_type->createColumnConst(num_rows, part->name);
|
||||
result_columns[pos] = column->convertToFullColumnIfConst();
|
||||
}
|
||||
else if (column_name == mark_number_column.name)
|
||||
{
|
||||
auto column = column_type->createColumn();
|
||||
auto & data = assert_cast<ColumnUInt64 &>(*column).getData();
|
||||
|
||||
data.resize(num_rows);
|
||||
std::iota(data.begin(), data.end(), 0);
|
||||
|
||||
result_columns[pos] = std::move(column);
|
||||
}
|
||||
else if (column_name == rows_in_granule_column.name)
|
||||
{
|
||||
auto column = column_type->createColumn();
|
||||
auto & data = assert_cast<ColumnUInt64 &>(*column).getData();
|
||||
|
||||
data.resize(num_rows);
|
||||
for (size_t i = 0; i < num_rows; ++i)
|
||||
data[i] = index_granularity.getMarkRows(i);
|
||||
|
||||
result_columns[pos] = std::move(column);
|
||||
}
|
||||
else if (auto [first, second] = Nested::splitName(column_name, true); with_marks && second == "mark")
|
||||
{
|
||||
result_columns[pos] = fillMarks(part, marks_loader, *column_type, first);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw Exception(ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, "No such column {}", column_name);
|
||||
}
|
||||
}
|
||||
|
||||
++part_index;
|
||||
return Chunk(std::move(result_columns), num_rows);
|
||||
}
|
||||
|
||||
private:
|
||||
std::shared_ptr<MergeTreeMarksLoader> createMarksLoader(const MergeTreeDataPartPtr & part, const String & prefix_name, size_t num_columns)
|
||||
{
|
||||
auto info_for_read = std::make_shared<LoadedMergeTreeDataPartInfoForReader>(part, std::make_shared<AlterConversions>());
|
||||
auto local_context = getContext();
|
||||
|
||||
return std::make_shared<MergeTreeMarksLoader>(
|
||||
info_for_read,
|
||||
local_context->getMarkCache().get(),
|
||||
info_for_read->getIndexGranularityInfo().getMarksFilePath(prefix_name),
|
||||
info_for_read->getMarksCount(),
|
||||
info_for_read->getIndexGranularityInfo(),
|
||||
/*save_marks_in_cache=*/ false,
|
||||
local_context->getReadSettings(),
|
||||
/*load_marks_threadpool=*/ nullptr,
|
||||
num_columns);
|
||||
}
|
||||
|
||||
ColumnPtr fillMarks(
|
||||
MergeTreeDataPartPtr part,
|
||||
std::shared_ptr<MergeTreeMarksLoader> marks_loader,
|
||||
const IDataType & data_type,
|
||||
const String & column_name)
|
||||
{
|
||||
size_t col_idx = 0;
|
||||
bool has_marks_in_part = false;
|
||||
size_t num_rows = part->index_granularity.getMarksCount();
|
||||
|
||||
if (isWidePart(part))
|
||||
{
|
||||
if (auto stream_name = part->getStreamNameOrHash(column_name, part->checksums))
|
||||
{
|
||||
col_idx = 0;
|
||||
has_marks_in_part = true;
|
||||
marks_loader = createMarksLoader(part, *stream_name, /*num_columns=*/ 1);
|
||||
}
|
||||
}
|
||||
else if (isCompactPart(part))
|
||||
{
|
||||
auto unescaped_name = unescapeForFileName(column_name);
|
||||
if (auto col_idx_opt = part->getColumnPosition(unescaped_name))
|
||||
{
|
||||
col_idx = *col_idx_opt;
|
||||
has_marks_in_part = true;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Parts with type {} are not supported", part->getTypeName());
|
||||
}
|
||||
|
||||
if (!has_marks_in_part)
|
||||
{
|
||||
auto column = data_type.createColumnConstWithDefaultValue(num_rows);
|
||||
return column->convertToFullColumnIfConst();
|
||||
}
|
||||
|
||||
auto compressed = ColumnUInt64::create(num_rows);
|
||||
auto uncompressed = ColumnUInt64::create(num_rows);
|
||||
|
||||
auto & compressed_data = compressed->getData();
|
||||
auto & uncompressed_data = uncompressed->getData();
|
||||
|
||||
for (size_t i = 0; i < num_rows; ++i)
|
||||
{
|
||||
auto mark = marks_loader->getMark(i, col_idx);
|
||||
|
||||
compressed_data[i] = mark.offset_in_compressed_file;
|
||||
uncompressed_data[i] = mark.offset_in_decompressed_block;
|
||||
}
|
||||
|
||||
auto compressed_nullable = ColumnNullable::create(std::move(compressed), ColumnUInt8::create(num_rows, 0));
|
||||
auto uncompressed_nullable = ColumnNullable::create(std::move(uncompressed), ColumnUInt8::create(num_rows, 0));
|
||||
|
||||
return ColumnTuple::create(Columns{std::move(compressed_nullable), std::move(uncompressed_nullable)});
|
||||
}
|
||||
|
||||
Block header;
|
||||
Block index_header;
|
||||
MergeTreeData::DataPartsVector data_parts;
|
||||
bool with_marks;
|
||||
|
||||
size_t part_index = 0;
|
||||
};
|
||||
|
||||
const ColumnWithTypeAndName StorageMergeTreeIndex::part_name_column{std::make_shared<DataTypeString>(), "part_name"};
|
||||
const ColumnWithTypeAndName StorageMergeTreeIndex::mark_number_column{std::make_shared<DataTypeUInt64>(), "mark_number"};
|
||||
const ColumnWithTypeAndName StorageMergeTreeIndex::rows_in_granule_column{std::make_shared<DataTypeUInt64>(), "rows_in_granule"};
|
||||
const Block StorageMergeTreeIndex::virtuals_sample_block{part_name_column, mark_number_column, rows_in_granule_column};
|
||||
|
||||
StorageMergeTreeIndex::StorageMergeTreeIndex(
|
||||
const StorageID & table_id_,
|
||||
const StoragePtr & source_table_,
|
||||
const ColumnsDescription & columns,
|
||||
bool with_marks_)
|
||||
: IStorage(table_id_)
|
||||
, source_table(source_table_)
|
||||
, with_marks(with_marks_)
|
||||
, log(&Poco::Logger::get("StorageMergeTreeIndex"))
|
||||
{
|
||||
const auto * merge_tree = dynamic_cast<const MergeTreeData *>(source_table.get());
|
||||
if (!merge_tree)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Storage MergeTreeIndex expected MergeTree table, got: {}", source_table->getName());
|
||||
|
||||
data_parts = merge_tree->getDataPartsVectorForInternalUsage();
|
||||
key_sample_block = merge_tree->getInMemoryMetadataPtr()->getPrimaryKey().sample_block;
|
||||
|
||||
StorageInMemoryMetadata storage_metadata;
|
||||
storage_metadata.setColumns(columns);
|
||||
setInMemoryMetadata(storage_metadata);
|
||||
}
|
||||
|
||||
Pipe StorageMergeTreeIndex::read(
|
||||
const Names & column_names,
|
||||
const StorageSnapshotPtr & storage_snapshot,
|
||||
SelectQueryInfo & query_info,
|
||||
ContextPtr context,
|
||||
QueryProcessingStage::Enum,
|
||||
size_t /*max_block_size*/,
|
||||
size_t /*num_streams*/)
|
||||
{
|
||||
const auto & storage_columns = source_table->getInMemoryMetadataPtr()->getColumns();
|
||||
Names columns_from_storage;
|
||||
|
||||
for (const auto & column_name : column_names)
|
||||
{
|
||||
if (storage_columns.hasColumnOrSubcolumn(GetColumnsOptions::All, column_name))
|
||||
{
|
||||
columns_from_storage.push_back(column_name);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (with_marks)
|
||||
{
|
||||
auto [first, second] = Nested::splitName(column_name, true);
|
||||
auto unescaped_name = unescapeForFileName(first);
|
||||
|
||||
if (second == "mark" && storage_columns.hasColumnOrSubcolumn(GetColumnsOptions::All, unescapeForFileName(unescaped_name)))
|
||||
{
|
||||
columns_from_storage.push_back(unescaped_name);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
context->checkAccess(AccessType::SELECT, source_table->getStorageID(), columns_from_storage);
|
||||
|
||||
auto header = storage_snapshot->getSampleBlockForColumns(column_names);
|
||||
auto filtered_parts = getFilteredDataParts(query_info, context);
|
||||
|
||||
LOG_DEBUG(log, "Reading index{}from {} parts of table {}",
|
||||
with_marks ? " with marks " : " ",
|
||||
filtered_parts.size(),
|
||||
source_table->getStorageID().getNameForLogs());
|
||||
|
||||
return Pipe(std::make_shared<MergeTreeIndexSource>(std::move(header), key_sample_block, std::move(filtered_parts), context, with_marks));
|
||||
}
|
||||
|
||||
MergeTreeData::DataPartsVector StorageMergeTreeIndex::getFilteredDataParts(SelectQueryInfo & query_info, const ContextPtr & context) const
|
||||
{
|
||||
const auto * select_query = query_info.query->as<ASTSelectQuery>();
|
||||
if (!select_query || !select_query->where())
|
||||
return data_parts;
|
||||
|
||||
auto all_part_names = ColumnString::create();
|
||||
for (const auto & part : data_parts)
|
||||
all_part_names->insert(part->name);
|
||||
|
||||
Block filtered_block{{std::move(all_part_names), std::make_shared<DataTypeString>(), part_name_column.name}};
|
||||
VirtualColumnUtils::filterBlockWithQuery(query_info.query, filtered_block, context);
|
||||
|
||||
if (!filtered_block.rows())
|
||||
return {};
|
||||
|
||||
auto part_names = filtered_block.getByPosition(0).column;
|
||||
const auto & part_names_str = assert_cast<const ColumnString &>(*part_names);
|
||||
|
||||
HashSet<StringRef> part_names_set;
|
||||
for (size_t i = 0; i < part_names_str.size(); ++i)
|
||||
part_names_set.insert(part_names_str.getDataAt(i));
|
||||
|
||||
MergeTreeData::DataPartsVector filtered_parts;
|
||||
for (const auto & part : data_parts)
|
||||
if (part_names_set.has(part->name))
|
||||
filtered_parts.push_back(part);
|
||||
|
||||
return filtered_parts;
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,46 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <QueryPipeline/Pipe.h>
|
||||
#include <Storages/MergeTree/MergeTreeData.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// Internal temporary storage for table function mergeTreeIndex(...)
|
||||
class StorageMergeTreeIndex final : public IStorage
|
||||
{
|
||||
public:
|
||||
static const ColumnWithTypeAndName part_name_column;
|
||||
static const ColumnWithTypeAndName mark_number_column;
|
||||
static const ColumnWithTypeAndName rows_in_granule_column;
|
||||
static const Block virtuals_sample_block;
|
||||
|
||||
StorageMergeTreeIndex(
|
||||
const StorageID & table_id_,
|
||||
const StoragePtr & source_table_,
|
||||
const ColumnsDescription & columns,
|
||||
bool with_marks_);
|
||||
|
||||
Pipe read(
|
||||
const Names & column_names,
|
||||
const StorageSnapshotPtr & storage_snapshot,
|
||||
SelectQueryInfo & query_info,
|
||||
ContextPtr context,
|
||||
QueryProcessingStage::Enum processing_stage,
|
||||
size_t max_block_size,
|
||||
size_t num_streams) override;
|
||||
|
||||
String getName() const override { return "MergeTreeIndex"; }
|
||||
|
||||
private:
|
||||
MergeTreeData::DataPartsVector getFilteredDataParts(SelectQueryInfo & query_info, const ContextPtr & context) const;
|
||||
|
||||
StoragePtr source_table;
|
||||
bool with_marks;
|
||||
|
||||
MergeTreeData::DataPartsVector data_parts;
|
||||
Block key_sample_block;
|
||||
Poco::Logger * log;
|
||||
};
|
||||
|
||||
}
|
||||
@ -1,204 +0,0 @@
|
||||
#include <Storages/StorageMergeTreeIndex.h>
|
||||
#include <TableFunctions/ITableFunction.h>
|
||||
#include <Interpreters/DatabaseCatalog.h>
|
||||
#include <Interpreters/evaluateConstantExpression.h>
|
||||
#include <Storages/checkAndGetLiteralArgument.h>
|
||||
#include <TableFunctions/TableFunctionFactory.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <DataTypes/NestedUtils.h>
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
#include <Storages/NamedCollectionsHelpers.h>
|
||||
#include <Common/escapeForFileName.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
class TableFunctionMergeTreeIndex : public ITableFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "mergeTreeIndex";
|
||||
std::string getName() const override { return name; }
|
||||
|
||||
void parseArguments(const ASTPtr & ast_function, ContextPtr context) override;
|
||||
ColumnsDescription getActualTableStructure(ContextPtr context, bool is_insert_query) const override;
|
||||
|
||||
private:
|
||||
StoragePtr executeImpl(
|
||||
const ASTPtr & ast_function,
|
||||
ContextPtr context,
|
||||
const std::string & table_name,
|
||||
ColumnsDescription cached_columns,
|
||||
bool is_insert_query) const override;
|
||||
|
||||
const char * getStorageTypeName() const override { return "MergeTreeIndex"; }
|
||||
|
||||
StorageID source_table_id{StorageID::createEmpty()};
|
||||
bool with_marks = false;
|
||||
};
|
||||
|
||||
void TableFunctionMergeTreeIndex::parseArguments(const ASTPtr & ast_function, ContextPtr context)
|
||||
{
|
||||
ASTs & args_func = ast_function->children;
|
||||
if (args_func.size() != 1)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Table function ({}) must have arguments.", quoteString(getName()));
|
||||
|
||||
ASTs & args = args_func.at(0)->children;
|
||||
if (args.size() < 2 || args.size() > 3)
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
|
||||
"Table function '{}' must have 2 or 3 arguments, got: {}", getName(), args.size());
|
||||
|
||||
args[0] = evaluateConstantExpressionForDatabaseName(args[0], context);
|
||||
args[1] = evaluateConstantExpressionOrIdentifierAsLiteral(args[1], context);
|
||||
|
||||
auto database = checkAndGetLiteralArgument<String>(args[0], "database");
|
||||
auto table = checkAndGetLiteralArgument<String>(args[1], "table");
|
||||
|
||||
ASTs rest_args(args.begin() + 2, args.end());
|
||||
if (!rest_args.empty())
|
||||
{
|
||||
auto params = getParamsMapFromAST(rest_args, context);
|
||||
auto param = params.extract("with_marks");
|
||||
|
||||
if (!param.empty())
|
||||
{
|
||||
auto & value = param.mapped();
|
||||
if (value.getType() != Field::Types::Bool && value.getType() != Field::Types::UInt64)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||
"Table function '{}' expected bool flag for 'with_marks' argument", getName());
|
||||
|
||||
if (value.getType() == Field::Types::Bool)
|
||||
with_marks = value.get<bool>();
|
||||
else
|
||||
with_marks = value.get<UInt64>();
|
||||
}
|
||||
|
||||
if (!params.empty())
|
||||
{
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||
"Unexpected arguments '{}' for table function '{}'",
|
||||
fmt::join(params | boost::adaptors::map_keys, ","), getName());
|
||||
}
|
||||
}
|
||||
|
||||
source_table_id = StorageID{database, table};
|
||||
}
|
||||
|
||||
static NameSet getAllPossibleStreamNames(
|
||||
const NameAndTypePair & column,
|
||||
const MergeTreeDataPartsVector & data_parts)
|
||||
{
|
||||
NameSet all_streams;
|
||||
|
||||
/// Add the stream with the name of column
|
||||
/// because it may be abcent in serialization streams (e.g. for Tuple type)
|
||||
/// but in compact parts we write only marks for whole columns, not subsubcolumns.
|
||||
auto main_stream_name = escapeForFileName(column.name);
|
||||
all_streams.insert(Nested::concatenateName(main_stream_name, "mark"));
|
||||
|
||||
auto callback = [&](const auto & substream_path)
|
||||
{
|
||||
auto stream_name = ISerialization::getFileNameForStream(column, substream_path);
|
||||
all_streams.insert(Nested::concatenateName(stream_name, "mark"));
|
||||
};
|
||||
|
||||
auto serialization = IDataType::getSerialization(column);
|
||||
serialization->enumerateStreams(callback);
|
||||
|
||||
if (!column.type->supportsSparseSerialization())
|
||||
return all_streams;
|
||||
|
||||
/// If there is at least one part with sparse serialization
|
||||
/// add columns with marks of its substreams to the table.
|
||||
for (const auto & part : data_parts)
|
||||
{
|
||||
serialization = part->tryGetSerialization(column.name);
|
||||
if (serialization && serialization->getKind() == ISerialization::Kind::SPARSE)
|
||||
{
|
||||
serialization->enumerateStreams(callback);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return all_streams;
|
||||
}
|
||||
|
||||
ColumnsDescription TableFunctionMergeTreeIndex::getActualTableStructure(ContextPtr context, bool /*is_insert_query*/) const
|
||||
{
|
||||
auto source_table = DatabaseCatalog::instance().getTable(source_table_id, context);
|
||||
auto metadata_snapshot = source_table->getInMemoryMetadataPtr();
|
||||
|
||||
ColumnsDescription columns;
|
||||
for (const auto & column : StorageMergeTreeIndex::virtuals_sample_block)
|
||||
columns.add({column.name, column.type});
|
||||
|
||||
for (const auto & column : metadata_snapshot->getPrimaryKey().sample_block)
|
||||
columns.add({column.name, column.type});
|
||||
|
||||
if (with_marks)
|
||||
{
|
||||
auto element_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>());
|
||||
auto mark_type = std::make_shared<DataTypeTuple>(
|
||||
DataTypes{element_type, element_type},
|
||||
Names{"offset_in_compressed_file", "offset_in_decompressed_block"});
|
||||
|
||||
const auto * merge_tree = dynamic_cast<const MergeTreeData *>(source_table.get());
|
||||
if (!merge_tree)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Table function mergeTreeIndex expected MergeTree table, got: {}", source_table->getName());
|
||||
|
||||
auto data_parts = merge_tree->getDataPartsVectorForInternalUsage();
|
||||
auto columns_list = Nested::convertToSubcolumns(metadata_snapshot->getColumns().getAllPhysical());
|
||||
|
||||
for (const auto & column : columns_list)
|
||||
{
|
||||
auto all_streams = getAllPossibleStreamNames(column, data_parts);
|
||||
for (const auto & stream_name : all_streams)
|
||||
{
|
||||
/// There may be shared substreams of columns (e.g. for Nested type)
|
||||
if (!columns.has(stream_name))
|
||||
columns.add({stream_name, mark_type});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return columns;
|
||||
}
|
||||
|
||||
StoragePtr TableFunctionMergeTreeIndex::executeImpl(
|
||||
const ASTPtr & /*ast_function*/,
|
||||
ContextPtr context,
|
||||
const std::string & table_name,
|
||||
ColumnsDescription /*cached_columns*/,
|
||||
bool is_insert_query) const
|
||||
{
|
||||
auto source_table = DatabaseCatalog::instance().getTable(source_table_id, context);
|
||||
auto columns = getActualTableStructure(context, is_insert_query);
|
||||
|
||||
StorageID storage_id(getDatabaseName(), table_name);
|
||||
auto res = std::make_shared<StorageMergeTreeIndex>(std::move(storage_id), std::move(source_table), std::move(columns), with_marks);
|
||||
|
||||
res->startup();
|
||||
return res;
|
||||
}
|
||||
|
||||
void registerTableFunctionMergeTreeIndex(TableFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<TableFunctionMergeTreeIndex>(
|
||||
{
|
||||
.documentation =
|
||||
{
|
||||
.description = "Represents the contents of index and marks files of MergeTree tables. It can be used for introspection",
|
||||
.examples = {{"mergeTreeIndex", "SELECT * FROM mergeTreeIndex(currentDatabase(), mt_table, with_marks = true)", ""}},
|
||||
.categories = {"Other"},
|
||||
},
|
||||
.allow_readonly = true,
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
@ -23,7 +23,6 @@ void registerTableFunctions()
|
||||
registerTableFunctionGenerate(factory);
|
||||
registerTableFunctionMongoDB(factory);
|
||||
registerTableFunctionRedis(factory);
|
||||
registerTableFunctionMergeTreeIndex(factory);
|
||||
#if USE_RAPIDJSON || USE_SIMDJSON
|
||||
registerTableFunctionFuzzJSON(factory);
|
||||
#endif
|
||||
|
||||
@ -20,7 +20,6 @@ void registerTableFunctionInput(TableFunctionFactory & factory);
|
||||
void registerTableFunctionGenerate(TableFunctionFactory & factory);
|
||||
void registerTableFunctionMongoDB(TableFunctionFactory & factory);
|
||||
void registerTableFunctionRedis(TableFunctionFactory & factory);
|
||||
void registerTableFunctionMergeTreeIndex(TableFunctionFactory & factory);
|
||||
#if USE_RAPIDJSON || USE_SIMDJSON
|
||||
void registerTableFunctionFuzzJSON(TableFunctionFactory & factory);
|
||||
#endif
|
||||
|
||||
@ -1,84 +0,0 @@
|
||||
0 0 v0
|
||||
0 5 v25
|
||||
1 1 v1
|
||||
1 6 v36
|
||||
2 2 v4
|
||||
2 7 v49
|
||||
3 3 v9
|
||||
3 8 v64
|
||||
4 4 v16
|
||||
4 9 v81
|
||||
0 10 v100
|
||||
0 15 v225
|
||||
1 11 v121
|
||||
1 16 v256
|
||||
2 12 v144
|
||||
2 17 v289
|
||||
3 13 v169
|
||||
3 18 v324
|
||||
4 14 v196
|
||||
4 19 v361
|
||||
┌─part_name─┬─mark_number─┬─rows_in_granule─┬─a─┬──b─┐
|
||||
│ all_1_1_0 │ 0 │ 3 │ 0 │ 0 │
|
||||
│ all_1_1_0 │ 1 │ 3 │ 1 │ 6 │
|
||||
│ all_1_1_0 │ 2 │ 3 │ 3 │ 3 │
|
||||
│ all_1_1_0 │ 3 │ 1 │ 4 │ 9 │
|
||||
│ all_1_1_0 │ 4 │ 0 │ 4 │ 9 │
|
||||
│ all_2_2_0 │ 0 │ 3 │ 0 │ 10 │
|
||||
│ all_2_2_0 │ 1 │ 3 │ 1 │ 16 │
|
||||
│ all_2_2_0 │ 2 │ 3 │ 3 │ 13 │
|
||||
│ all_2_2_0 │ 3 │ 1 │ 4 │ 19 │
|
||||
│ all_2_2_0 │ 4 │ 0 │ 4 │ 19 │
|
||||
└───────────┴─────────────┴─────────────────┴───┴────┘
|
||||
┌─part_name─┬─mark_number─┬─rows_in_granule─┬─a─┬──b─┬─a.mark─┬─b.mark─┬─s.mark─┐
|
||||
│ all_1_1_0 │ 0 │ 3 │ 0 │ 0 │ (0,0) │ (0,0) │ (0,0) │
|
||||
│ all_1_1_0 │ 1 │ 3 │ 1 │ 6 │ (0,24) │ (0,24) │ (0,10) │
|
||||
│ all_1_1_0 │ 2 │ 3 │ 3 │ 3 │ (0,48) │ (0,48) │ (0,21) │
|
||||
│ all_1_1_0 │ 3 │ 1 │ 4 │ 9 │ (0,72) │ (0,72) │ (0,32) │
|
||||
│ all_1_1_0 │ 4 │ 0 │ 4 │ 9 │ (0,80) │ (0,80) │ (0,36) │
|
||||
│ all_2_2_0 │ 0 │ 3 │ 0 │ 10 │ (0,0) │ (0,0) │ (0,0) │
|
||||
│ all_2_2_0 │ 1 │ 3 │ 1 │ 16 │ (0,24) │ (0,24) │ (0,15) │
|
||||
│ all_2_2_0 │ 2 │ 3 │ 3 │ 13 │ (0,48) │ (0,48) │ (0,30) │
|
||||
│ all_2_2_0 │ 3 │ 1 │ 4 │ 19 │ (0,72) │ (0,72) │ (0,45) │
|
||||
│ all_2_2_0 │ 4 │ 0 │ 4 │ 19 │ (0,80) │ (0,80) │ (0,50) │
|
||||
└───────────┴─────────────┴─────────────────┴───┴────┴────────┴────────┴────────┘
|
||||
0 0 v0
|
||||
0 4 v16
|
||||
0 8 v64
|
||||
1 1 v1
|
||||
1 5 v25
|
||||
1 9 v81
|
||||
2 2 v4
|
||||
2 6 v36
|
||||
3 3 v9
|
||||
3 7 v49
|
||||
0 12 v144
|
||||
0 16 v256
|
||||
1 13 v169
|
||||
1 17 v289
|
||||
2 10 v100
|
||||
2 14 v196
|
||||
2 18 v324
|
||||
3 11 v121
|
||||
3 15 v225
|
||||
3 19 v361
|
||||
┌─part_name─┬─mark_number─┬─rows_in_granule─┬─a─┬──b─┐
|
||||
│ all_1_1_0 │ 0 │ 3 │ 0 │ 0 │
|
||||
│ all_1_1_0 │ 1 │ 3 │ 1 │ 1 │
|
||||
│ all_1_1_0 │ 2 │ 4 │ 2 │ 2 │
|
||||
│ all_1_1_0 │ 3 │ 0 │ 3 │ 7 │
|
||||
│ all_2_2_0 │ 0 │ 3 │ 0 │ 12 │
|
||||
│ all_2_2_0 │ 1 │ 3 │ 1 │ 17 │
|
||||
│ all_2_2_0 │ 2 │ 4 │ 2 │ 18 │
|
||||
│ all_2_2_0 │ 3 │ 0 │ 3 │ 19 │
|
||||
└───────────┴─────────────┴─────────────────┴───┴────┘
|
||||
┌─part_name─┬─mark_number─┬─rows_in_granule─┬─a─┬──b─┬─a.mark──┬─b.mark──┬─s.mark──┐
|
||||
│ all_1_1_0 │ 0 │ 3 │ 0 │ 0 │ (0,0) │ (35,0) │ (77,0) │
|
||||
│ all_1_1_0 │ 1 │ 3 │ 1 │ 1 │ (114,0) │ (153,0) │ (197,0) │
|
||||
│ all_1_1_0 │ 2 │ 4 │ 2 │ 2 │ (234,0) │ (281,0) │ (329,0) │
|
||||
│ all_1_1_0 │ 3 │ 0 │ 3 │ 7 │ (369,0) │ (369,0) │ (369,0) │
|
||||
│ all_2_2_0 │ 0 │ 3 │ 0 │ 12 │ (0,0) │ (38,0) │ (82,0) │
|
||||
│ all_2_2_0 │ 1 │ 3 │ 1 │ 17 │ (124,0) │ (168,0) │ (212,0) │
|
||||
│ all_2_2_0 │ 2 │ 4 │ 2 │ 18 │ (254,0) │ (297,0) │ (345,0) │
|
||||
│ all_2_2_0 │ 3 │ 0 │ 3 │ 19 │ (392,0) │ (392,0) │ (392,0) │
|
||||
└───────────┴─────────────┴─────────────────┴───┴────┴─────────┴─────────┴─────────┘
|
||||
@ -1,37 +0,0 @@
|
||||
DROP TABLE IF EXISTS t_merge_tree_index;
|
||||
|
||||
CREATE TABLE t_merge_tree_index (a UInt64, b UInt64, s String)
|
||||
ENGINE = MergeTree ORDER BY (a, b)
|
||||
SETTINGS
|
||||
index_granularity = 3,
|
||||
min_bytes_for_wide_part = 0,
|
||||
ratio_of_defaults_for_sparse_serialization = 1.0;
|
||||
|
||||
SYSTEM STOP MERGES t_merge_tree_index;
|
||||
|
||||
INSERT INTO t_merge_tree_index SELECT number % 5, number, 'v' || toString(number * number) FROM numbers(10);
|
||||
INSERT INTO t_merge_tree_index SELECT number % 5, number, 'v' || toString(number * number) FROM numbers(10, 10);
|
||||
|
||||
SELECT * FROM t_merge_tree_index ORDER BY _part, a, b;
|
||||
SELECT * FROM mergeTreeIndex(currentDatabase(), t_merge_tree_index) ORDER BY part_name, mark_number FORMAT PrettyCompactNoEscapesMonoBlock;
|
||||
SELECT * FROM mergeTreeIndex(currentDatabase(), t_merge_tree_index, with_marks = true) ORDER BY part_name, mark_number FORMAT PrettyCompactNoEscapesMonoBlock;
|
||||
|
||||
DROP TABLE t_merge_tree_index;
|
||||
|
||||
CREATE TABLE t_merge_tree_index (a UInt64, b UInt64, s String)
|
||||
ENGINE = MergeTree ORDER BY (a, b)
|
||||
SETTINGS
|
||||
index_granularity = 3,
|
||||
min_bytes_for_wide_part = '1G',
|
||||
ratio_of_defaults_for_sparse_serialization = 1.0;
|
||||
|
||||
SYSTEM STOP MERGES t_merge_tree_index;
|
||||
|
||||
INSERT INTO t_merge_tree_index SELECT number % 4, number, 'v' || toString(number * number) FROM numbers(10);
|
||||
INSERT INTO t_merge_tree_index SELECT number % 4, number, 'v' || toString(number * number) FROM numbers(10, 10);
|
||||
|
||||
SELECT * FROM t_merge_tree_index ORDER BY _part, a, b;
|
||||
SELECT * FROM mergeTreeIndex(currentDatabase(), t_merge_tree_index) ORDER BY part_name, mark_number FORMAT PrettyCompactNoEscapesMonoBlock;
|
||||
SELECT * FROM mergeTreeIndex(currentDatabase(), t_merge_tree_index, with_marks = true) ORDER BY part_name, mark_number FORMAT PrettyCompactNoEscapesMonoBlock;
|
||||
|
||||
DROP TABLE t_merge_tree_index;
|
||||
@ -1,51 +0,0 @@
|
||||
┌─part_name─┬─mark_number─┬─rows_in_granule─┬─a─┬─b─┬─modulo(sipHash64(sp), 100)─┐
|
||||
│ all_1_1_0 │ 0 │ 3 │ 0 │ 0 │ 19 │
|
||||
│ all_1_1_0 │ 1 │ 3 │ 1 │ 6 │ 19 │
|
||||
│ all_1_1_0 │ 2 │ 3 │ 3 │ 3 │ 19 │
|
||||
│ all_1_1_0 │ 3 │ 1 │ 4 │ 9 │ 19 │
|
||||
│ all_1_1_0 │ 4 │ 0 │ 4 │ 9 │ 19 │
|
||||
│ all_2_2_0 │ 0 │ 3 │ 0 │ 0 │ 96 │
|
||||
│ all_2_2_0 │ 1 │ 2 │ 3 │ 3 │ 96 │
|
||||
│ all_2_2_0 │ 2 │ 0 │ 4 │ 4 │ 96 │
|
||||
│ all_3_3_0 │ 0 │ 3 │ 0 │ 0 │ 96 │
|
||||
│ all_3_3_0 │ 1 │ 3 │ 1 │ 6 │ 96 │
|
||||
│ all_3_3_0 │ 2 │ 3 │ 3 │ 3 │ 96 │
|
||||
│ all_3_3_0 │ 3 │ 1 │ 4 │ 9 │ 96 │
|
||||
│ all_3_3_0 │ 4 │ 0 │ 4 │ 9 │ 96 │
|
||||
└───────────┴─────────────┴─────────────────┴───┴───┴────────────────────────────┘
|
||||
┌─part_name─┬─mark_number─┬─rows_in_granule─┬─a─┬─b─┬─modulo(sipHash64(sp), 100)─┬─a.mark──┬─b.mark──┬─c.mark──────┬─sp.sparse.idx.mark─┬─sp.mark─┬─arr.size0.mark─┬─arr.dict.mark─┬─arr.mark─┬─n.size0.mark─┬─n%2Ec1.mark─┬─n%2Ec2.mark─┬─t%2Ec2.mark─┬─t%2Ec1.mark─┬─t.mark──────┬─column%2Ewith%2Edots.mark─┐
|
||||
│ all_1_1_0 │ 0 │ 3 │ 0 │ 0 │ 19 │ (0,0) │ (0,0) │ (NULL,NULL) │ (0,0) │ (0,0) │ (0,0) │ (0,8) │ (0,0) │ (0,0) │ (0,0) │ (0,0) │ (0,0) │ (0,0) │ (NULL,NULL) │ (0,0) │
|
||||
│ all_1_1_0 │ 1 │ 3 │ 1 │ 6 │ 19 │ (0,24) │ (0,24) │ (NULL,NULL) │ (0,9) │ (0,0) │ (0,24) │ (0,8) │ (0,22) │ (0,24) │ (0,36) │ (0,72) │ (0,24) │ (0,24) │ (NULL,NULL) │ (0,24) │
|
||||
│ all_1_1_0 │ 2 │ 3 │ 3 │ 3 │ 19 │ (0,48) │ (0,48) │ (NULL,NULL) │ (0,18) │ (0,0) │ (0,48) │ (0,8) │ (0,44) │ (0,48) │ (0,72) │ (0,144) │ (0,48) │ (0,48) │ (NULL,NULL) │ (0,48) │
|
||||
│ all_1_1_0 │ 3 │ 1 │ 4 │ 9 │ 19 │ (0,72) │ (0,72) │ (NULL,NULL) │ (0,27) │ (0,0) │ (0,72) │ (0,8) │ (0,66) │ (0,72) │ (0,108) │ (0,216) │ (0,72) │ (0,72) │ (NULL,NULL) │ (0,72) │
|
||||
│ all_1_1_0 │ 4 │ 0 │ 4 │ 9 │ 19 │ (0,80) │ (0,80) │ (NULL,NULL) │ (0,36) │ (0,0) │ (0,80) │ (0,25) │ (0,84) │ (0,80) │ (0,120) │ (0,240) │ (0,80) │ (0,80) │ (NULL,NULL) │ (0,80) │
|
||||
│ all_2_2_0 │ 0 │ 3 │ 0 │ 0 │ 96 │ (0,0) │ (42,0) │ (84,0) │ (NULL,NULL) │ (126,0) │ (NULL,NULL) │ (NULL,NULL) │ (165,0) │ (NULL,NULL) │ (232,0) │ (286,0) │ (NULL,NULL) │ (NULL,NULL) │ (342,0) │ (391,0) │
|
||||
│ all_2_2_0 │ 1 │ 2 │ 3 │ 3 │ 96 │ (433,0) │ (472,0) │ (511,0) │ (NULL,NULL) │ (550,0) │ (NULL,NULL) │ (NULL,NULL) │ (589,0) │ (NULL,NULL) │ (659,0) │ (717,0) │ (NULL,NULL) │ (NULL,NULL) │ (773,0) │ (817,0) │
|
||||
│ all_2_2_0 │ 2 │ 0 │ 4 │ 4 │ 96 │ (856,0) │ (856,0) │ (856,0) │ (NULL,NULL) │ (856,0) │ (NULL,NULL) │ (NULL,NULL) │ (856,0) │ (NULL,NULL) │ (856,0) │ (856,0) │ (NULL,NULL) │ (NULL,NULL) │ (856,0) │ (856,0) │
|
||||
│ all_3_3_0 │ 0 │ 3 │ 0 │ 0 │ 96 │ (0,0) │ (0,0) │ (0,0) │ (NULL,NULL) │ (0,0) │ (0,0) │ (0,8) │ (0,0) │ (0,0) │ (0,0) │ (0,0) │ (0,0) │ (0,0) │ (NULL,NULL) │ (0,0) │
|
||||
│ all_3_3_0 │ 1 │ 3 │ 1 │ 6 │ 96 │ (0,24) │ (0,24) │ (0,24) │ (NULL,NULL) │ (0,24) │ (0,24) │ (0,8) │ (0,22) │ (0,24) │ (0,36) │ (0,72) │ (0,24) │ (0,24) │ (NULL,NULL) │ (0,24) │
|
||||
│ all_3_3_0 │ 2 │ 3 │ 3 │ 3 │ 96 │ (0,48) │ (0,48) │ (0,48) │ (NULL,NULL) │ (0,48) │ (0,48) │ (0,8) │ (0,44) │ (0,48) │ (0,72) │ (0,144) │ (0,48) │ (0,48) │ (NULL,NULL) │ (0,48) │
|
||||
│ all_3_3_0 │ 3 │ 1 │ 4 │ 9 │ 96 │ (0,72) │ (0,72) │ (0,72) │ (NULL,NULL) │ (0,72) │ (0,72) │ (0,8) │ (0,66) │ (0,72) │ (0,108) │ (0,216) │ (0,72) │ (0,72) │ (NULL,NULL) │ (0,72) │
|
||||
│ all_3_3_0 │ 4 │ 0 │ 4 │ 9 │ 96 │ (0,80) │ (0,80) │ (0,80) │ (NULL,NULL) │ (0,80) │ (0,80) │ (0,25) │ (0,84) │ (0,80) │ (0,120) │ (0,240) │ (0,80) │ (0,80) │ (NULL,NULL) │ (0,80) │
|
||||
└───────────┴─────────────┴─────────────────┴───┴───┴────────────────────────────┴─────────┴─────────┴─────────────┴────────────────────┴─────────┴────────────────┴───────────────┴──────────┴──────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴───────────────────────────┘
|
||||
part_name String
|
||||
mark_number UInt64
|
||||
rows_in_granule UInt64
|
||||
a UInt64
|
||||
b UInt64
|
||||
modulo(sipHash64(sp), 100) UInt8
|
||||
a.mark Tuple(offset_in_compressed_file Nullable(UInt64), offset_in_decompressed_block Nullable(UInt64))
|
||||
b.mark Tuple(offset_in_compressed_file Nullable(UInt64), offset_in_decompressed_block Nullable(UInt64))
|
||||
c.mark Tuple(offset_in_compressed_file Nullable(UInt64), offset_in_decompressed_block Nullable(UInt64))
|
||||
sp.sparse.idx.mark Tuple(offset_in_compressed_file Nullable(UInt64), offset_in_decompressed_block Nullable(UInt64))
|
||||
sp.mark Tuple(offset_in_compressed_file Nullable(UInt64), offset_in_decompressed_block Nullable(UInt64))
|
||||
arr.size0.mark Tuple(offset_in_compressed_file Nullable(UInt64), offset_in_decompressed_block Nullable(UInt64))
|
||||
arr.dict.mark Tuple(offset_in_compressed_file Nullable(UInt64), offset_in_decompressed_block Nullable(UInt64))
|
||||
arr.mark Tuple(offset_in_compressed_file Nullable(UInt64), offset_in_decompressed_block Nullable(UInt64))
|
||||
n.size0.mark Tuple(offset_in_compressed_file Nullable(UInt64), offset_in_decompressed_block Nullable(UInt64))
|
||||
n%2Ec1.mark Tuple(offset_in_compressed_file Nullable(UInt64), offset_in_decompressed_block Nullable(UInt64))
|
||||
n%2Ec2.mark Tuple(offset_in_compressed_file Nullable(UInt64), offset_in_decompressed_block Nullable(UInt64))
|
||||
t%2Ec2.mark Tuple(offset_in_compressed_file Nullable(UInt64), offset_in_decompressed_block Nullable(UInt64))
|
||||
t%2Ec1.mark Tuple(offset_in_compressed_file Nullable(UInt64), offset_in_decompressed_block Nullable(UInt64))
|
||||
t.mark Tuple(offset_in_compressed_file Nullable(UInt64), offset_in_decompressed_block Nullable(UInt64))
|
||||
column%2Ewith%2Edots.mark Tuple(offset_in_compressed_file Nullable(UInt64), offset_in_decompressed_block Nullable(UInt64))
|
||||
@ -1,38 +0,0 @@
|
||||
DROP TABLE IF EXISTS t_merge_tree_index;
|
||||
|
||||
SET print_pretty_type_names = 0;
|
||||
|
||||
CREATE TABLE t_merge_tree_index
|
||||
(
|
||||
`a` UInt64,
|
||||
`b` UInt64,
|
||||
`sp` UInt64,
|
||||
`arr` Array(LowCardinality(String)),
|
||||
`n` Nested(c1 String, c2 UInt64),
|
||||
`t` Tuple(c1 UInt64, c2 UInt64),
|
||||
`column.with.dots` UInt64
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (a, b, sipHash64(sp) % 100)
|
||||
SETTINGS
|
||||
index_granularity = 3,
|
||||
min_bytes_for_wide_part = 0,
|
||||
min_rows_for_wide_part = 6,
|
||||
ratio_of_defaults_for_sparse_serialization = 0.9;
|
||||
|
||||
SYSTEM STOP MERGES t_merge_tree_index;
|
||||
|
||||
INSERT INTO t_merge_tree_index SELECT number % 5, number, 0, ['foo', 'bar'], ['aaa', 'bbb', 'ccc'], [11, 22, 33], (number, number), number FROM numbers(10);
|
||||
|
||||
ALTER TABLE t_merge_tree_index ADD COLUMN c UInt64 AFTER b;
|
||||
|
||||
INSERT INTO t_merge_tree_index SELECT number % 5, number, number, 10, ['foo', 'bar'], ['aaa', 'bbb', 'ccc'], [11, 22, 33], (number, number), number FROM numbers(5);
|
||||
INSERT INTO t_merge_tree_index SELECT number % 5, number, number, 10, ['foo', 'bar'], ['aaa', 'bbb', 'ccc'], [11, 22, 33], (number, number), number FROM numbers(10);
|
||||
|
||||
SELECT * FROM mergeTreeIndex(currentDatabase(), t_merge_tree_index) ORDER BY part_name, mark_number FORMAT PrettyCompactNoEscapesMonoBlock;
|
||||
SELECT * FROM mergeTreeIndex(currentDatabase(), t_merge_tree_index, with_marks = true) ORDER BY part_name, mark_number FORMAT PrettyCompactNoEscapesMonoBlock;
|
||||
|
||||
SET describe_compact_output = 1;
|
||||
DESCRIBE mergeTreeIndex(currentDatabase(), t_merge_tree_index, with_marks = true);
|
||||
|
||||
DROP TABLE t_merge_tree_index;
|
||||
@ -1,10 +0,0 @@
|
||||
ACCESS_DENIED
|
||||
ACCESS_DENIED
|
||||
ACCESS_DENIED
|
||||
OK
|
||||
ACCESS_DENIED
|
||||
ACCESS_DENIED
|
||||
ACCESS_DENIED
|
||||
ACCESS_DENIED
|
||||
OK
|
||||
OK
|
||||
@ -1,50 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
user_name="${CLICKHOUSE_DATABASE}_test_user_02947"
|
||||
|
||||
$CLICKHOUSE_CLIENT -n -q "
|
||||
DROP TABLE IF EXISTS t_merge_tree_index;
|
||||
DROP USER IF EXISTS $user_name;
|
||||
|
||||
CREATE TABLE t_merge_tree_index
|
||||
(
|
||||
a UInt64,
|
||||
b UInt64,
|
||||
arr Array(LowCardinality(String)),
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (a, b)
|
||||
SETTINGS
|
||||
index_granularity = 3,
|
||||
min_bytes_for_wide_part = 0,
|
||||
min_rows_for_wide_part = 6,
|
||||
ratio_of_defaults_for_sparse_serialization = 0.9;
|
||||
|
||||
INSERT INTO t_merge_tree_index (a) VALUES (1);
|
||||
|
||||
CREATE USER $user_name IDENTIFIED WITH plaintext_password BY 'password';
|
||||
REVOKE SELECT ON $CLICKHOUSE_DATABASE.t_merge_tree_index FROM $user_name;
|
||||
GRANT SELECT (b) ON $CLICKHOUSE_DATABASE.t_merge_tree_index TO $user_name;
|
||||
"
|
||||
|
||||
$CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "SELECT a FROM t_merge_tree_index" 2>&1 | grep -m1 -o "ACCESS_DENIED" || echo "OK"
|
||||
$CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "SELECT arr FROM t_merge_tree_index" 2>&1 | grep -m1 -o "ACCESS_DENIED" || echo "OK"
|
||||
$CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "SELECT arr.size0 FROM t_merge_tree_index" 2>&1 | grep -m1 -o "ACCESS_DENIED" || echo "OK"
|
||||
$CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "SELECT b FROM t_merge_tree_index" 2>&1 | grep -m1 -o "ACCESS_DENIED" || echo "OK"
|
||||
|
||||
$CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "SELECT a FROM mergeTreeIndex(currentDatabase(), t_merge_tree_index, with_marks = true)" 2>&1 | grep -m1 -o "ACCESS_DENIED" || echo "OK"
|
||||
$CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "SELECT a.mark FROM mergeTreeIndex(currentDatabase(), t_merge_tree_index, with_marks = true)" 2>&1 | grep -m1 -o "ACCESS_DENIED" || echo "OK"
|
||||
$CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "SELECT arr.mark FROM mergeTreeIndex(currentDatabase(), t_merge_tree_index, with_marks = true)" 2>&1 | grep -m1 -o "ACCESS_DENIED" || echo "OK"
|
||||
$CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "SELECT arr.size0.mark FROM mergeTreeIndex(currentDatabase(), t_merge_tree_index, with_marks = true)" 2>&1 | grep -m1 -o "ACCESS_DENIED" || echo "OK"
|
||||
|
||||
$CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "SELECT b FROM mergeTreeIndex(currentDatabase(), t_merge_tree_index, with_marks = true)" 2>&1 | grep -m1 -o "ACCESS_DENIED" || echo "OK"
|
||||
$CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "SELECT b.mark FROM mergeTreeIndex(currentDatabase(), t_merge_tree_index, with_marks = true)" 2>&1 | grep -m1 -o "ACCESS_DENIED" || echo "OK"
|
||||
|
||||
$CLICKHOUSE_CLIENT -n -q "
|
||||
DROP TABLE IF EXISTS t_merge_tree_index;
|
||||
DROP USER IF EXISTS $user_name;
|
||||
"
|
||||
@ -1892,7 +1892,6 @@ mdadm
|
||||
meanZTest
|
||||
meanztest
|
||||
mebibytes
|
||||
mergeTreeIndex
|
||||
mergeable
|
||||
mergetree
|
||||
messageID
|
||||
@ -2420,7 +2419,6 @@ subranges
|
||||
subreddits
|
||||
subseconds
|
||||
subsequence
|
||||
substreams
|
||||
substring
|
||||
substringIndex
|
||||
substringIndexUTF
|
||||
|
||||
Loading…
Reference in New Issue
Block a user