mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-26 01:22:04 +00:00
212 lines
8.0 KiB
Bash
Executable File
212 lines
8.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Tags: no-fasttest
|
|
|
|
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|
# shellcheck source=../shell_config.sh
|
|
. "$CURDIR"/../shell_config.sh
|
|
|
|
PAR_PATH="$CURDIR"/data_parquet/native_parquet_reader.parquet
|
|
# the content of parquet file can be generated by following codes
|
|
# <<EndOfCodes
|
|
#
|
|
# #include <Columns/ColumnNullable.h>
|
|
# #include <Columns/ColumnString.h>
|
|
# #include <Common/tests/gtest_global_register.h>
|
|
# #include <Core/Block.h>
|
|
# #include <DataTypes/DataTypeDateTime64.h>
|
|
# #include <DataTypes/DataTypeLowCardinality.h>
|
|
# #include <DataTypes/DataTypeNullable.h>
|
|
# #include <DataTypes/DataTypeString.h>
|
|
# #include <DataTypes/DataTypesDecimal.h>
|
|
# #include <IO/WriteBufferFromFile.h>
|
|
# #include <Processors/Formats/IOutputFormat.h>
|
|
# #include <Processors/Formats/Impl/Parquet/ParquetRecordReader.h>
|
|
#
|
|
# namespace
|
|
# {
|
|
#
|
|
# using namespace DB;
|
|
#
|
|
# const UInt32 ROW_NUM = 2000;
|
|
# const UInt32 MIN_STRING_LEN = 3;
|
|
# const UInt32 MAX_STRING_LEN = 5;
|
|
#
|
|
# const UInt32 PLAIN_ENCODING_CARDINALITY = ROW_NUM * 2;
|
|
# const UInt32 MIX_ENCODING_CARDINALITY = 800;
|
|
# const UInt32 DICT_ENCODING_CARDINALITY = 20;
|
|
#
|
|
# UInt16 nextNum()
|
|
# {
|
|
# static UInt16 idx = 0;
|
|
# static UInt16 nums[] = {0, 21845, 43690};
|
|
# static size_t nums_len = sizeof(nums) / sizeof(nums[0]);
|
|
# return nums[(idx++) % nums_len]++;
|
|
# }
|
|
#
|
|
# template <typename NumericDataType>
|
|
# void generateValues(MutableColumnPtr & col, size_t num)
|
|
# {
|
|
# using FieldType = typename NumericDataType::FieldType;
|
|
#
|
|
# const size_t next_num_bytes = sizeof(nextNum());
|
|
# char bytewise_val[sizeof(FieldType)];
|
|
#
|
|
# while (col->size() < num)
|
|
# {
|
|
# for (auto bytes = 0; bytes < sizeof(FieldType); bytes += next_num_bytes)
|
|
# {
|
|
# auto tmp = nextNum();
|
|
# memcpy(bytewise_val + bytes, &tmp, std::min(next_num_bytes, sizeof(FieldType) - bytes));
|
|
# }
|
|
# if (is_decimal<FieldType>)
|
|
# {
|
|
# // clean highest 3 bits, make sure the result doest not exceed the limits of the decimal type
|
|
# if (bytewise_val[sizeof(FieldType) - 1] > 0)
|
|
# bytewise_val[sizeof(FieldType) - 1] &= 0x0f;
|
|
# else
|
|
# bytewise_val[sizeof(FieldType) - 1] |= 0xf0;
|
|
# }
|
|
# FieldType val;
|
|
# memcpy(&val, &bytewise_val, sizeof(FieldType));
|
|
# col->insert(val);
|
|
# }
|
|
# }
|
|
#
|
|
# template <>
|
|
# void generateValues<DataTypeString>(MutableColumnPtr & col, size_t num)
|
|
# {
|
|
# std::string str;
|
|
# while (col->size() < num)
|
|
# {
|
|
# auto len = MIN_STRING_LEN + nextNum() % (MAX_STRING_LEN - MIN_STRING_LEN);
|
|
# str.clear();
|
|
# for (size_t i = 0; i < len; i++)
|
|
# {
|
|
# str.push_back('a' + nextNum() % ('z' - 'a'));
|
|
# }
|
|
# col->insert(str);
|
|
# }
|
|
# }
|
|
#
|
|
# template <typename DataType>
|
|
# ColumnWithTypeAndName generateColumn(
|
|
# std::shared_ptr<DataType> ch_type,
|
|
# size_t cardinality,
|
|
# const std::string & col_name,
|
|
# const std::set<size_t> & null_indice)
|
|
# {
|
|
# DataTypePtr col_type = ch_type;
|
|
# if (!null_indice.empty())
|
|
# {
|
|
# col_type = std::make_shared<DataTypeNullable>(ch_type);
|
|
# }
|
|
#
|
|
# auto values = ch_type->createColumn();
|
|
# values->reserve(cardinality);
|
|
# generateValues<DataType>(values, cardinality);
|
|
#
|
|
# auto col = col_type->createColumn();
|
|
# col->reserve(ROW_NUM);
|
|
# for (size_t i = 0; i < ROW_NUM; i++)
|
|
# {
|
|
# if (!null_indice.empty() && null_indice.contains(i))
|
|
# {
|
|
# col->insert(Null());
|
|
# }
|
|
# else
|
|
# {
|
|
# col->insert(values->operator[](nextNum() % cardinality));
|
|
# }
|
|
# }
|
|
# return {std::move(col), col_type, col_name};
|
|
# }
|
|
#
|
|
# Block generateBlock()
|
|
# {
|
|
# ColumnsWithTypeAndName cols;
|
|
#
|
|
# // test Int32 type
|
|
# std::set<size_t> null_indice{512, 1001, 211, 392, 553, 1725};
|
|
# // Nullability is expressed by definition level, and encoded by bit packed with smallest group size of 8
|
|
# // when null value appeared. Here we make a big bit packed group with more than 1000 values.
|
|
# for (size_t i = 0; i < 170; i++)
|
|
# {
|
|
# null_indice.emplace(622 + i * 6);
|
|
# }
|
|
# cols.emplace_back(generateColumn(
|
|
# std::make_shared<DataTypeInt32>(), PLAIN_ENCODING_CARDINALITY, "plain_encoding_i32", null_indice));
|
|
# null_indice = {917, 482, 283, 580, 1926, 1667, 1971};
|
|
# cols.emplace_back(generateColumn(
|
|
# std::make_shared<DataTypeInt32>(), DICT_ENCODING_CARDINALITY, "dict_encoding_i32", null_indice));
|
|
#
|
|
# // test string type
|
|
# null_indice = {818, 928, 1958, 1141, 1553, 1407, 690, 1769};
|
|
# cols.emplace_back(generateColumn(
|
|
# std::make_shared<DataTypeString>(), PLAIN_ENCODING_CARDINALITY, "plain_encoding_str", null_indice));
|
|
# null_indice = {1441, 1747, 216, 1209, 89, 52, 536, 625};
|
|
# cols.emplace_back(generateColumn(
|
|
# std::make_shared<DataTypeString>(), MIX_ENCODING_CARDINALITY, "mix_encoding_str", null_indice));
|
|
# null_indice = {1478, 1862, 894, 1314, 1844, 243, 869, 551};
|
|
# cols.emplace_back(generateColumn(
|
|
# std::make_shared<DataTypeString>(), DICT_ENCODING_CARDINALITY, "dict_encoding_str", null_indice));
|
|
#
|
|
# // test DateTime64 type
|
|
# auto dt_type = std::make_shared<DataTypeDateTime64>(ParquetRecordReader::default_datetime64_scale);
|
|
# null_indice = {1078, 112, 1981, 795, 371, 1176, 1526, 11};
|
|
# cols.emplace_back(generateColumn(dt_type, PLAIN_ENCODING_CARDINALITY, "plain_encoding_dt64", null_indice));
|
|
# null_indice = {1734, 1153, 1893, 1205, 644, 1670, 1482, 1479};
|
|
# cols.emplace_back(generateColumn(dt_type, DICT_ENCODING_CARDINALITY, "dict_encoding_dt64", null_indice));
|
|
#
|
|
# // test Decimal128 type
|
|
# auto d128_type = std::make_shared<DataTypeDecimal128>(DecimalUtils::max_precision<Decimal128>, 3);
|
|
# null_indice = {852, 1448, 1569, 896, 1866, 1655, 100, 418};
|
|
# cols.emplace_back(generateColumn(d128_type, PLAIN_ENCODING_CARDINALITY, "plain_encoding_decimal128", null_indice));
|
|
#
|
|
# return {cols};
|
|
# }
|
|
#
|
|
# void dumpBlock(const Block & block)
|
|
# {
|
|
# WriteBufferFromFile output_buf("/tmp/ut-out.csv");
|
|
# auto out = getContext().context->getOutputFormat("CSVWithNames", output_buf, block);
|
|
# out->write(block);
|
|
# out->finalize();
|
|
# std::cerr << block.dumpStructure() << std::endl << std::endl;
|
|
# }
|
|
#
|
|
# }
|
|
#
|
|
# EndOfCodes
|
|
#
|
|
# How to generate the parquet file:
|
|
# 1. Use above C++ codes.
|
|
# Put above codes in src/Common/tests/gtest_main.cpp, add following two inlines in main function:
|
|
# tryRegisterFormats();
|
|
# dumpBlock(generateBlock());
|
|
# 2. Genetate /tmp/ut-out.csv.
|
|
# After compiled, run any test, such as "./src/unit_tests_dbms --gtest_filter=IColumn.dumpStructure",
|
|
# 3. Generate the parquet file by following spark sql
|
|
# create temporary view tv using csv options('path' '/tmp/ut-out.csv', 'header' 'true', 'nullValue' '\\N');
|
|
# insert overwrite directory "/tmp/test-parquet" using Parquet
|
|
# options('parquet.dictionary.page.size' '500')
|
|
# select /*+ COALESCE(1) */ cast(plain_encoding_i32 as int), cast(dict_encoding_i32 as int),
|
|
# plain_encoding_str, mix_encoding_str, dict_encoding_str,
|
|
# cast(plain_encoding_dt64 as timestamp), cast(dict_encoding_dt64 as timestamp),
|
|
# cast(plain_encoding_decimal128 as decimal(38, 3))
|
|
# from tv;
|
|
#
|
|
|
|
CH_SCHEMA="\
|
|
plain_encoding_i32 Nullable(Int32), \
|
|
dict_encoding_i32 Nullable(Int32), \
|
|
plain_encoding_str Nullable(String), \
|
|
mix_encoding_str Nullable(String), \
|
|
dict_encoding_str LowCardinality(Nullable(String)), \
|
|
plain_encoding_dt64 Nullable(DateTime64(9, \\'UTC\\')), \
|
|
dict_encoding_dt64 Nullable(DateTime64(9, \\'UTC\\')), \
|
|
plain_encoding_decimal128 Nullable(Decimal(38, 3))"
|
|
QUERY="SELECT * from file('$PAR_PATH', 'Parquet', '$CH_SCHEMA')"
|
|
|
|
# there may be more than on group in parquet files, unstable results may generated by multithreads
|
|
$CLICKHOUSE_LOCAL --max_threads 1 --max_parsing_threads 1 --input_format_parquet_use_native_reader true --query "$QUERY"
|