mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-09 17:14:47 +00:00
Merge remote-tracking branch 'origin/master' into pr-local-plan
This commit is contained in:
Igor Nikonov
commit
217e50d12e
3
.github/workflows/backport_branches.yml
vendored
3
.github/workflows/backport_branches.yml
vendored
@ -272,7 +272,4 @@ jobs:
|
||||
cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
|
||||
${{ toJson(needs) }}
|
||||
EOF
|
||||
echo "::group::Workflow results"
|
||||
python3 -m json.tool "$WORKFLOW_RESULT_FILE"
|
||||
echo "::endgroup::"
|
||||
python3 ./tests/ci/ci_buddy.py --check-wf-status
|
||||
|
||||
3
.github/workflows/master.yml
vendored
3
.github/workflows/master.yml
vendored
@ -138,7 +138,4 @@ jobs:
|
||||
cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
|
||||
${{ toJson(needs) }}
|
||||
EOF
|
||||
echo "::group::Workflow results"
|
||||
python3 -m json.tool "$WORKFLOW_RESULT_FILE"
|
||||
echo "::endgroup::"
|
||||
python3 ./tests/ci/ci_buddy.py --check-wf-status
|
||||
|
||||
3
.github/workflows/merge_queue.yml
vendored
3
.github/workflows/merge_queue.yml
vendored
@ -111,7 +111,4 @@ jobs:
|
||||
cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
|
||||
${{ toJson(needs) }}
|
||||
EOF
|
||||
echo "::group::Workflow results"
|
||||
python3 -m json.tool "$WORKFLOW_RESULT_FILE"
|
||||
echo "::endgroup::"
|
||||
python3 ./tests/ci/ci_buddy.py --check-wf-status
|
||||
|
||||
3
.github/workflows/nightly.yml
vendored
3
.github/workflows/nightly.yml
vendored
@ -57,7 +57,4 @@ jobs:
|
||||
cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
|
||||
${{ toJson(needs) }}
|
||||
EOF
|
||||
echo "::group::Workflow results"
|
||||
python3 -m json.tool "$WORKFLOW_RESULT_FILE"
|
||||
echo "::endgroup::"
|
||||
python3 ./tests/ci/ci_buddy.py --check-wf-status
|
||||
|
||||
3
.github/workflows/pull_request.yml
vendored
3
.github/workflows/pull_request.yml
vendored
@ -171,9 +171,6 @@ jobs:
|
||||
cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
|
||||
${{ toJson(needs) }}
|
||||
EOF
|
||||
echo "::group::Workflow results"
|
||||
python3 -m json.tool "$WORKFLOW_RESULT_FILE"
|
||||
echo "::endgroup::"
|
||||
python3 ./tests/ci/ci_buddy.py --check-wf-status
|
||||
|
||||
################################# Stage Final #################################
|
||||
|
||||
4
.github/workflows/release_branches.yml
vendored
4
.github/workflows/release_branches.yml
vendored
@ -492,7 +492,5 @@ jobs:
|
||||
cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
|
||||
${{ toJson(needs) }}
|
||||
EOF
|
||||
echo "::group::Workflow results"
|
||||
python3 -m json.tool "$WORKFLOW_RESULT_FILE"
|
||||
echo "::endgroup::"
|
||||
|
||||
python3 ./tests/ci/ci_buddy.py --check-wf-status
|
||||
|
||||
@ -2,11 +2,11 @@
|
||||
|
||||
# NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION,
|
||||
# only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
|
||||
SET(VERSION_REVISION 54488)
|
||||
SET(VERSION_REVISION 54489)
|
||||
SET(VERSION_MAJOR 24)
|
||||
SET(VERSION_MINOR 7)
|
||||
SET(VERSION_MINOR 8)
|
||||
SET(VERSION_PATCH 1)
|
||||
SET(VERSION_GITHASH aa023477a9265e403982fca5ee29a714db5133d9)
|
||||
SET(VERSION_DESCRIBE v24.7.1.1-testing)
|
||||
SET(VERSION_STRING 24.7.1.1)
|
||||
SET(VERSION_GITHASH 3f8b27d7accd2b5ec4afe7d0dd459115323304af)
|
||||
SET(VERSION_DESCRIBE v24.8.1.1-testing)
|
||||
SET(VERSION_STRING 24.8.1.1)
|
||||
# end of autochange
|
||||
|
||||
2
contrib/libunwind
vendored
2
contrib/libunwind
vendored
@ -1 +1 @@
|
||||
Subproject commit 8f28e64d15819d2d096badd598c7d85bebddb1f2
|
||||
Subproject commit fe854449e24bedfa26e38465b84374312dbd587f
|
||||
@ -191,8 +191,8 @@ else
|
||||
ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
|
||||
SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
|
||||
|
||||
clickhouse-client --query "INSERT INTO test.hits SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0"
|
||||
clickhouse-client --query "INSERT INTO test.visits SELECT * FROM datasets.visits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0"
|
||||
clickhouse-client --query "INSERT INTO test.hits SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16"
|
||||
clickhouse-client --query "INSERT INTO test.visits SELECT * FROM datasets.visits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16"
|
||||
clickhouse-client --query "DROP TABLE datasets.visits_v1 SYNC"
|
||||
clickhouse-client --query "DROP TABLE datasets.hits_v1 SYNC"
|
||||
else
|
||||
@ -200,7 +200,7 @@ else
|
||||
clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
|
||||
fi
|
||||
clickhouse-client --query "CREATE TABLE test.hits_s3 (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
|
||||
clickhouse-client --query "INSERT INTO test.hits_s3 SELECT * FROM test.hits SETTINGS enable_filesystem_cache_on_write_operations=0"
|
||||
clickhouse-client --query "INSERT INTO test.hits_s3 SELECT * FROM test.hits SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16"
|
||||
fi
|
||||
|
||||
clickhouse-client --query "SHOW TABLES FROM test"
|
||||
|
||||
@ -209,9 +209,9 @@ clickhouse-client --query "CREATE TABLE test.visits (CounterID UInt32, StartDat
|
||||
ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
|
||||
SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='$TEMP_POLICY'"
|
||||
|
||||
clickhouse-client --query "INSERT INTO test.hits_s3 SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0"
|
||||
clickhouse-client --query "INSERT INTO test.hits SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0"
|
||||
clickhouse-client --query "INSERT INTO test.visits SELECT * FROM datasets.visits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0"
|
||||
clickhouse-client --query "INSERT INTO test.hits_s3 SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16"
|
||||
clickhouse-client --query "INSERT INTO test.hits SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16"
|
||||
clickhouse-client --query "INSERT INTO test.visits SELECT * FROM datasets.visits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16"
|
||||
|
||||
clickhouse-client --query "DROP TABLE datasets.visits_v1 SYNC"
|
||||
clickhouse-client --query "DROP TABLE datasets.hits_v1 SYNC"
|
||||
|
||||
@ -57,7 +57,8 @@ static struct InitFiu
|
||||
PAUSEABLE_ONCE(finish_clean_quorum_failed_parts) \
|
||||
PAUSEABLE(dummy_pausable_failpoint) \
|
||||
ONCE(execute_query_calling_empty_set_result_func_on_exception) \
|
||||
ONCE(receive_timeout_on_table_status_response)
|
||||
ONCE(receive_timeout_on_table_status_response) \
|
||||
REGULAR(keepermap_fail_drop_data) \
|
||||
|
||||
|
||||
namespace FailPoints
|
||||
|
||||
@ -1144,7 +1144,7 @@ std::optional<UInt64> MergeTreeData::totalRowsByPartitionPredicateImpl(
|
||||
auto metadata_snapshot = getInMemoryMetadataPtr();
|
||||
auto virtual_columns_block = getBlockWithVirtualsForFilter(metadata_snapshot, {parts[0]});
|
||||
|
||||
auto filter_dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), nullptr);
|
||||
auto filter_dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), nullptr, /*allow_non_deterministic_functions=*/ false);
|
||||
if (!filter_dag)
|
||||
return {};
|
||||
|
||||
|
||||
@ -37,6 +37,7 @@
|
||||
|
||||
#include <Common/Base64.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/FailPoint.h>
|
||||
#include <Common/ZooKeeper/IKeeper.h>
|
||||
#include <Common/ZooKeeper/KeeperException.h>
|
||||
#include <Common/ZooKeeper/Types.h>
|
||||
@ -64,6 +65,11 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace FailPoints
|
||||
{
|
||||
extern const char keepermap_fail_drop_data[];
|
||||
}
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
@ -411,18 +417,16 @@ StorageKeeperMap::StorageKeeperMap(
|
||||
|
||||
auto code = client->tryCreate(zk_table_path, "", zkutil::CreateMode::Persistent);
|
||||
|
||||
// tables_path was removed with drop
|
||||
if (code == Coordination::Error::ZNONODE)
|
||||
{
|
||||
LOG_INFO(log, "Metadata nodes were removed by another server, will retry");
|
||||
continue;
|
||||
}
|
||||
else if (code != Coordination::Error::ZOK)
|
||||
{
|
||||
throw zkutil::KeeperException(code, "Failed to create table on path {} because a table with same UUID already exists", zk_root_path);
|
||||
}
|
||||
/// A table on the same Keeper path already exists, we just appended our table id to subscribe as a new replica
|
||||
/// We still don't know if the table matches the expected metadata so table_is_valid is not changed
|
||||
/// It will be checked lazily on the first operation
|
||||
if (code == Coordination::Error::ZOK)
|
||||
return;
|
||||
|
||||
return;
|
||||
if (code != Coordination::Error::ZNONODE)
|
||||
throw zkutil::KeeperException(code, "Failed to create table on path {} because a table with same UUID already exists", zk_root_path);
|
||||
|
||||
/// ZNONODE means we dropped zk_tables_path but didn't finish drop completely
|
||||
}
|
||||
|
||||
if (client->exists(zk_dropped_path))
|
||||
@ -473,6 +477,7 @@ StorageKeeperMap::StorageKeeperMap(
|
||||
|
||||
|
||||
table_is_valid = true;
|
||||
/// we are the first table created for the specified Keeper path, i.e. we are the first replica
|
||||
return;
|
||||
}
|
||||
|
||||
@ -561,6 +566,10 @@ void StorageKeeperMap::truncate(const ASTPtr &, const StorageMetadataPtr &, Cont
|
||||
|
||||
bool StorageKeeperMap::dropTable(zkutil::ZooKeeperPtr zookeeper, const zkutil::EphemeralNodeHolder::Ptr & metadata_drop_lock)
|
||||
{
|
||||
fiu_do_on(FailPoints::keepermap_fail_drop_data,
|
||||
{
|
||||
throw zkutil::KeeperException(Coordination::Error::ZOPERATIONTIMEOUT, "Manually triggered operation timeout");
|
||||
});
|
||||
zookeeper->removeChildrenRecursive(zk_data_path);
|
||||
|
||||
bool completely_removed = false;
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
// autogenerated by tests/ci/version_helper.py
|
||||
const char * auto_contributors[] {
|
||||
"0x01f",
|
||||
"0xflotus",
|
||||
"13DaGGeR",
|
||||
"1lann",
|
||||
@ -167,6 +168,7 @@ const char * auto_contributors[] {
|
||||
"AnneClickHouse",
|
||||
"Anselmo D. Adams",
|
||||
"Anthony N. Simon",
|
||||
"AntiTopQuark",
|
||||
"Anton Ivashkin",
|
||||
"Anton Kobzev",
|
||||
"Anton Kozlov",
|
||||
@ -299,6 +301,7 @@ const char * auto_contributors[] {
|
||||
"Dan Wu",
|
||||
"DanRoscigno",
|
||||
"Dani Pozo",
|
||||
"Daniel Anugerah",
|
||||
"Daniel Bershatsky",
|
||||
"Daniel Byta",
|
||||
"Daniel Dao",
|
||||
@ -370,6 +373,7 @@ const char * auto_contributors[] {
|
||||
"Elena",
|
||||
"Elena Baskakova",
|
||||
"Elena Torró",
|
||||
"Elena Torró Martínez",
|
||||
"Elghazal Ahmed",
|
||||
"Eliot Hautefeuille",
|
||||
"Elizaveta Mironyuk",
|
||||
@ -415,6 +419,7 @@ const char * auto_contributors[] {
|
||||
"FgoDt",
|
||||
"Filatenkov Artur",
|
||||
"Filipe Caixeta",
|
||||
"Filipp Bakanov",
|
||||
"Filipp Ozinov",
|
||||
"Filippov Denis",
|
||||
"Fille",
|
||||
@ -451,6 +456,7 @@ const char * auto_contributors[] {
|
||||
"Gleb Novikov",
|
||||
"Gleb-Tretyakov",
|
||||
"GoGoWen2021",
|
||||
"Gosha Letov",
|
||||
"Gregory",
|
||||
"Grigorii Sokolik",
|
||||
"Grigory",
|
||||
@ -461,6 +467,7 @@ const char * auto_contributors[] {
|
||||
"Guillaume Tassery",
|
||||
"Guo Wangyang",
|
||||
"Guo Wei (William)",
|
||||
"Guspan Tanadi",
|
||||
"Haavard Kvaalen",
|
||||
"Habibullah Oladepo",
|
||||
"HaiBo Li",
|
||||
@ -474,6 +481,7 @@ const char * auto_contributors[] {
|
||||
"HarryLeeIBM",
|
||||
"Hasitha Kanchana",
|
||||
"Hasnat",
|
||||
"Haydn",
|
||||
"Heena Bansal",
|
||||
"HeenaBansal2009",
|
||||
"Hendrik M",
|
||||
@ -606,6 +614,7 @@ const char * auto_contributors[] {
|
||||
"Kevin Chiang",
|
||||
"Kevin Michel",
|
||||
"Kevin Mingtarja",
|
||||
"Kevin Song",
|
||||
"Kevin Zhang",
|
||||
"KevinyhZou",
|
||||
"KinderRiven",
|
||||
@ -661,6 +670,7 @@ const char * auto_contributors[] {
|
||||
"Lewinma",
|
||||
"Li Shuai",
|
||||
"Li Yin",
|
||||
"Linh Giang",
|
||||
"Lino Uruñuela",
|
||||
"Lirikl",
|
||||
"Liu Cong",
|
||||
@ -690,6 +700,7 @@ const char * auto_contributors[] {
|
||||
"Maksim Alekseev",
|
||||
"Maksim Buren",
|
||||
"Maksim Fedotov",
|
||||
"Maksim Galkin",
|
||||
"Maksim Kita",
|
||||
"Maksym Sobolyev",
|
||||
"Mal Curtis",
|
||||
@ -724,6 +735,7 @@ const char * auto_contributors[] {
|
||||
"Max Akhmedov",
|
||||
"Max Bruce",
|
||||
"Max K",
|
||||
"Max K.",
|
||||
"Max Kainov",
|
||||
"Max Vetrov",
|
||||
"MaxTheHuman",
|
||||
@ -811,6 +823,7 @@ const char * auto_contributors[] {
|
||||
"Nataly Merezhuk",
|
||||
"Natalya Chizhonkova",
|
||||
"Natasha Murashkina",
|
||||
"Nathan Clevenger",
|
||||
"NeZeD [Mac Pro]",
|
||||
"Neeke Gao",
|
||||
"Neng Liu",
|
||||
@ -946,6 +959,7 @@ const char * auto_contributors[] {
|
||||
"Robert Coelho",
|
||||
"Robert Hodges",
|
||||
"Robert Schulze",
|
||||
"Rodolphe Dugé de Bernonville",
|
||||
"RogerYK",
|
||||
"Rohit Agarwal",
|
||||
"Romain Neutron",
|
||||
@ -1107,6 +1121,7 @@ const char * auto_contributors[] {
|
||||
"Timur Solodovnikov",
|
||||
"TiunovNN",
|
||||
"Tobias Adamson",
|
||||
"Tobias Florek",
|
||||
"Tobias Lins",
|
||||
"Tom Bombadil",
|
||||
"Tom Risse",
|
||||
@ -1231,11 +1246,13 @@ const char * auto_contributors[] {
|
||||
"Yingchun Lai",
|
||||
"Yingfan Chen",
|
||||
"Yinzheng-Sun",
|
||||
"Yinzuo Jiang",
|
||||
"Yiğit Konur",
|
||||
"Yohann Jardin",
|
||||
"Yong Wang",
|
||||
"Yong-Hao Zou",
|
||||
"Youenn Lebras",
|
||||
"Your Name",
|
||||
"Yu, Peng",
|
||||
"Yuko Takagi",
|
||||
"Yuntao Wu",
|
||||
@ -1250,6 +1267,7 @@ const char * auto_contributors[] {
|
||||
"Yury Stankevich",
|
||||
"Yusuke Tanaka",
|
||||
"Zach Naimon",
|
||||
"Zawa-II",
|
||||
"Zheng Miao",
|
||||
"ZhiHong Zhang",
|
||||
"ZhiYong Wang",
|
||||
@ -1380,6 +1398,7 @@ const char * auto_contributors[] {
|
||||
"conicliu",
|
||||
"copperybean",
|
||||
"coraxster",
|
||||
"cw5121",
|
||||
"cwkyaoyao",
|
||||
"d.v.semenov",
|
||||
"dalei2019",
|
||||
@ -1460,12 +1479,14 @@ const char * auto_contributors[] {
|
||||
"fuzzERot",
|
||||
"fyu",
|
||||
"g-arslan",
|
||||
"gabrielmcg44",
|
||||
"ggerogery",
|
||||
"giordyb",
|
||||
"glockbender",
|
||||
"glushkovds",
|
||||
"grantovsky",
|
||||
"gulige",
|
||||
"gun9nir",
|
||||
"guoleiyi",
|
||||
"guomaolin",
|
||||
"guov100",
|
||||
@ -1527,6 +1548,7 @@ const char * auto_contributors[] {
|
||||
"jferroal",
|
||||
"jiahui-97",
|
||||
"jianmei zhang",
|
||||
"jiaosenvip",
|
||||
"jinjunzh",
|
||||
"jiyoungyoooo",
|
||||
"jktng",
|
||||
@ -1541,6 +1563,7 @@ const char * auto_contributors[] {
|
||||
"jun won",
|
||||
"jus1096",
|
||||
"justindeguzman",
|
||||
"jwoodhead",
|
||||
"jyz0309",
|
||||
"karnevil13",
|
||||
"kashwy",
|
||||
@ -1633,10 +1656,12 @@ const char * auto_contributors[] {
|
||||
"mateng0915",
|
||||
"mateng915",
|
||||
"mauidude",
|
||||
"max-vostrikov",
|
||||
"maxim",
|
||||
"maxim-babenko",
|
||||
"maxkuzn",
|
||||
"maxulan",
|
||||
"maxvostrikov",
|
||||
"mayamika",
|
||||
"mehanizm",
|
||||
"melin",
|
||||
@ -1677,6 +1702,7 @@ const char * auto_contributors[] {
|
||||
"nathanbegbie",
|
||||
"nauta",
|
||||
"nautaa",
|
||||
"nauu",
|
||||
"ndchikin",
|
||||
"nellicus",
|
||||
"nemonlou",
|
||||
@ -1975,6 +2001,7 @@ const char * auto_contributors[] {
|
||||
"张健",
|
||||
"张风啸",
|
||||
"徐炘",
|
||||
"忒休斯~Theseus",
|
||||
"曲正鹏",
|
||||
"木木夕120",
|
||||
"未来星___费",
|
||||
|
||||
@ -271,7 +271,8 @@ bool isDeterministicInScopeOfQuery(const ActionsDAG::Node * node)
|
||||
static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
|
||||
const ActionsDAG::Node * node,
|
||||
const Block * allowed_inputs,
|
||||
ActionsDAG::Nodes & additional_nodes)
|
||||
ActionsDAG::Nodes & additional_nodes,
|
||||
bool allow_non_deterministic_functions)
|
||||
{
|
||||
if (node->type == ActionsDAG::ActionType::FUNCTION)
|
||||
{
|
||||
@ -280,8 +281,14 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
|
||||
auto & node_copy = additional_nodes.emplace_back(*node);
|
||||
node_copy.children.clear();
|
||||
for (const auto * child : node->children)
|
||||
if (const auto * child_copy = splitFilterNodeForAllowedInputs(child, allowed_inputs, additional_nodes))
|
||||
if (const auto * child_copy = splitFilterNodeForAllowedInputs(child, allowed_inputs, additional_nodes, allow_non_deterministic_functions))
|
||||
node_copy.children.push_back(child_copy);
|
||||
/// Expression like (now_allowed AND allowed) is not allowed if allow_non_deterministic_functions = true. This is important for
|
||||
/// trivial count optimization, otherwise we can get incorrect results. For example, if the query is
|
||||
/// SELECT count() FROM table WHERE _partition_id = '0' AND rowNumberInBlock() = 1, we cannot apply
|
||||
/// trivial count.
|
||||
else if (!allow_non_deterministic_functions)
|
||||
return nullptr;
|
||||
|
||||
if (node_copy.children.empty())
|
||||
return nullptr;
|
||||
@ -307,7 +314,7 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
|
||||
{
|
||||
auto & node_copy = additional_nodes.emplace_back(*node);
|
||||
for (auto & child : node_copy.children)
|
||||
if (child = splitFilterNodeForAllowedInputs(child, allowed_inputs, additional_nodes); !child)
|
||||
if (child = splitFilterNodeForAllowedInputs(child, allowed_inputs, additional_nodes, allow_non_deterministic_functions); !child)
|
||||
return nullptr;
|
||||
|
||||
return &node_copy;
|
||||
@ -321,7 +328,7 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
|
||||
auto index_hint_dag = index_hint->getActions()->clone();
|
||||
ActionsDAG::NodeRawConstPtrs atoms;
|
||||
for (const auto & output : index_hint_dag->getOutputs())
|
||||
if (const auto * child_copy = splitFilterNodeForAllowedInputs(output, allowed_inputs, additional_nodes))
|
||||
if (const auto * child_copy = splitFilterNodeForAllowedInputs(output, allowed_inputs, additional_nodes, allow_non_deterministic_functions))
|
||||
atoms.push_back(child_copy);
|
||||
|
||||
if (!atoms.empty())
|
||||
@ -355,13 +362,13 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
|
||||
return node;
|
||||
}
|
||||
|
||||
ActionsDAGPtr splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs)
|
||||
ActionsDAGPtr splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs, bool allow_non_deterministic_functions)
|
||||
{
|
||||
if (!predicate)
|
||||
return nullptr;
|
||||
|
||||
ActionsDAG::Nodes additional_nodes;
|
||||
const auto * res = splitFilterNodeForAllowedInputs(predicate, allowed_inputs, additional_nodes);
|
||||
const auto * res = splitFilterNodeForAllowedInputs(predicate, allowed_inputs, additional_nodes, allow_non_deterministic_functions);
|
||||
if (!res)
|
||||
return nullptr;
|
||||
|
||||
@ -370,7 +377,7 @@ ActionsDAGPtr splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate,
|
||||
|
||||
void filterBlockWithPredicate(const ActionsDAG::Node * predicate, Block & block, ContextPtr context)
|
||||
{
|
||||
auto dag = splitFilterDagForAllowedInputs(predicate, &block);
|
||||
auto dag = splitFilterDagForAllowedInputs(predicate, &block, /*allow_non_deterministic_functions=*/ false);
|
||||
if (dag)
|
||||
filterBlockWithDAG(dag, block, context);
|
||||
}
|
||||
|
||||
@ -32,7 +32,15 @@ void buildSetsForDAG(const ActionsDAG & dag, const ContextPtr & context);
|
||||
bool isDeterministicInScopeOfQuery(const ActionsDAG::Node * node);
|
||||
|
||||
/// Extract a part of predicate that can be evaluated using only columns from input_names.
|
||||
ActionsDAGPtr splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs);
|
||||
/// When allow_non_deterministic_functions is true then even if the predicate contains non-deterministic
|
||||
/// functions, we still allow to extract a part of the predicate, otherwise we return nullptr.
|
||||
/// allow_non_deterministic_functions must be false when we are going to use the result to filter parts in
|
||||
/// MergeTreeData::totalRowsByPartitionPredicateImp. For example, if the query is
|
||||
/// `SELECT count() FROM table WHERE _partition_id = '0' AND rowNumberInBlock() = 1`
|
||||
/// The predicate will be `_partition_id = '0' AND rowNumberInBlock() = 1`, and `rowNumberInBlock()` is
|
||||
/// non-deterministic. If we still extract the part `_partition_id = '0'` for filtering parts, then trivial
|
||||
/// count optimization will be mistakenly applied to the query.
|
||||
ActionsDAGPtr splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs, bool allow_non_deterministic_functions = true);
|
||||
|
||||
/// Extract from the input stream a set of `name` column values
|
||||
template <typename T>
|
||||
|
||||
@ -31,6 +31,7 @@ class CIBuddy:
|
||||
self.sha = pr_info.sha[:10]
|
||||
|
||||
def check_workflow(self):
|
||||
GHActions.print_workflow_results()
|
||||
res = GHActions.get_workflow_job_result(GHActions.ActionsNames.RunConfig)
|
||||
if res != GHActions.ActionStatuses.SUCCESS:
|
||||
self.post_job_error("Workflow Configuration Failed", critical=True)
|
||||
|
||||
@ -92,15 +92,30 @@ class GHActions:
|
||||
PENDING = "pending"
|
||||
SUCCESS = "success"
|
||||
|
||||
@staticmethod
|
||||
def get_workflow_job_result(wf_job_name: str) -> Optional[str]:
|
||||
@classmethod
|
||||
def _get_workflow_results(cls):
|
||||
if not Path(Envs.WORKFLOW_RESULT_FILE).exists():
|
||||
print(
|
||||
f"ERROR: Failed to get workflow results from file [{Envs.WORKFLOW_RESULT_FILE}]"
|
||||
)
|
||||
return None
|
||||
return {}
|
||||
with open(Envs.WORKFLOW_RESULT_FILE, "r", encoding="utf-8") as json_file:
|
||||
res = json.load(json_file)
|
||||
try:
|
||||
res = json.load(json_file)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"ERROR: json decoder exception {e}")
|
||||
return {}
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
def print_workflow_results(cls):
|
||||
res = cls._get_workflow_results()
|
||||
results = [f"{job}: {data['result']}" for job, data in res.items()]
|
||||
cls.print_in_group("Workflow results", results)
|
||||
|
||||
@classmethod
|
||||
def get_workflow_job_result(cls, wf_job_name: str) -> Optional[str]:
|
||||
res = cls._get_workflow_results()
|
||||
if wf_job_name in res:
|
||||
return res[wf_job_name]["result"] # type: ignore
|
||||
else:
|
||||
|
||||
@ -197,6 +197,10 @@ def get_instance_id():
|
||||
return _query_imds("latest/meta-data/instance-id")
|
||||
|
||||
|
||||
def get_instance_lifecycle():
|
||||
return _query_imds("latest/meta-data/instance-life-cycle")
|
||||
|
||||
|
||||
def prepare_tests_results_for_clickhouse(
|
||||
pr_info: PRInfo,
|
||||
test_results: TestResults,
|
||||
@ -233,7 +237,7 @@ def prepare_tests_results_for_clickhouse(
|
||||
"head_ref": head_ref,
|
||||
"head_repo": head_repo,
|
||||
"task_url": pr_info.task_url,
|
||||
"instance_type": get_instance_type(),
|
||||
"instance_type": ",".join([get_instance_type(), get_instance_lifecycle()]),
|
||||
"instance_id": get_instance_id(),
|
||||
}
|
||||
|
||||
|
||||
1
tests/ci/worker/.gitignore
vendored
1
tests/ci/worker/.gitignore
vendored
@ -1 +0,0 @@
|
||||
generated_*init_runner.sh
|
||||
@ -1,87 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 ENVIRONMENT" >&2
|
||||
echo "Valid values for ENVIRONMENT: staging, production" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
case "$1" in
|
||||
staging|production)
|
||||
ENVIRONMENT="$1" ;;
|
||||
--help)
|
||||
usage ;;
|
||||
*)
|
||||
echo "Invalid argument" >&2
|
||||
usage ;;
|
||||
esac
|
||||
|
||||
cd "$(dirname "$0")" || exit 1
|
||||
SOURCE_SCRIPT='init_runner.sh'
|
||||
|
||||
check_response() {
|
||||
# Are we even in the interactive shell?
|
||||
[ -t 1 ] || return 1
|
||||
local request
|
||||
request="$1"
|
||||
read -rp "$request (y/N): " response
|
||||
case "$response" in
|
||||
[Yy])
|
||||
return 0
|
||||
# Your code to continue goes here
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
check_dirty() {
|
||||
if [ -n "$(git status --porcelain=v2 "$SOURCE_SCRIPT")" ]; then
|
||||
echo "The $SOURCE_SCRIPT has uncommited changes, won't deploy it" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
GIT_HASH=$(git log -1 --format=format:%H)
|
||||
|
||||
header() {
|
||||
cat << EOF
|
||||
#!/usr/bin/env bash
|
||||
|
||||
echo 'The $ENVIRONMENT script is generated from $SOURCE_SCRIPT, commit $GIT_HASH'
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
body() {
|
||||
local first_line
|
||||
first_line=$(sed -n '/^# THE SCRIPT START$/{=;q;}' "$SOURCE_SCRIPT")
|
||||
if [ -z "$first_line" ]; then
|
||||
echo "The pattern '# THE SCRIPT START' is not found in $SOURCE_SCRIPT" >&2
|
||||
exit 1
|
||||
fi
|
||||
tail "+$first_line" "$SOURCE_SCRIPT"
|
||||
}
|
||||
|
||||
GENERATED_FILE="generated_${ENVIRONMENT}_${SOURCE_SCRIPT}"
|
||||
|
||||
{ header && body; } > "$GENERATED_FILE"
|
||||
|
||||
echo "The file $GENERATED_FILE is generated"
|
||||
|
||||
if check_response "Display the content of $GENERATED_FILE?"; then
|
||||
if [ -z "$PAGER" ]; then
|
||||
less "$GENERATED_FILE"
|
||||
else
|
||||
$PAGER "$GENERATED_FILE"
|
||||
fi
|
||||
fi
|
||||
|
||||
check_dirty
|
||||
|
||||
S3_OBJECT=${S3_OBJECT:-s3://github-runners-data/cloud-init/${ENVIRONMENT}.sh}
|
||||
if check_response "Deploy the generated script to $S3_OBJECT?"; then
|
||||
aws s3 mv "$GENERATED_FILE" "$S3_OBJECT"
|
||||
fi
|
||||
@ -1,406 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
cat > /dev/null << 'EOF'
|
||||
The following content is embedded into the s3 object via the script
|
||||
deploy-runner-init.sh {staging,production}
|
||||
with additional helping information
|
||||
|
||||
In the `user data` you should define as the following text
|
||||
between `### COPY BELOW` and `### COPY ABOVE`
|
||||
|
||||
### COPY BELOW
|
||||
Content-Type: multipart/mixed; boundary="//"
|
||||
MIME-Version: 1.0
|
||||
|
||||
--//
|
||||
Content-Type: text/cloud-config; charset="us-ascii"
|
||||
MIME-Version: 1.0
|
||||
Content-Transfer-Encoding: 7bit
|
||||
Content-Disposition: attachment; filename="cloud-config.txt"
|
||||
|
||||
#cloud-config
|
||||
cloud_final_modules:
|
||||
- [scripts-user, always]
|
||||
|
||||
--//
|
||||
Content-Type: text/x-shellscript; charset="us-ascii"
|
||||
MIME-Version: 1.0
|
||||
Content-Transfer-Encoding: 7bit
|
||||
Content-Disposition: attachment; filename="userdata.txt"
|
||||
|
||||
#!/bin/bash
|
||||
INSTANCE_ID=$(ec2metadata --instance-id)
|
||||
INIT_ENVIRONMENT=$(/usr/local/bin/aws ec2 describe-tags --filters "Name=resource-id,Values=$INSTANCE_ID" --query "Tags[?Key=='github:init-environment'].Value" --output text)
|
||||
echo "Downloading and using $INIT_ENVIRONMENT cloud-init.sh"
|
||||
aws s3 cp "s3://github-runners-data/cloud-init/${INIT_ENVIRONMENT:-production}.sh" /tmp/cloud-init.sh
|
||||
chmod 0700 /tmp/cloud-init.sh
|
||||
exec bash /tmp/cloud-init.sh
|
||||
--//
|
||||
### COPY ABOVE
|
||||
EOF
|
||||
|
||||
# THE SCRIPT START
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
####################################
|
||||
# IMPORTANT! #
|
||||
# EC2 instance should have #
|
||||
# `github:runner-type` tag #
|
||||
# set accordingly to a runner role #
|
||||
####################################
|
||||
|
||||
echo "Running init v1.1"
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
export RUNNER_HOME=/home/ubuntu/actions-runner
|
||||
|
||||
export RUNNER_ORG="ClickHouse"
|
||||
export RUNNER_URL="https://github.com/${RUNNER_ORG}"
|
||||
# Funny fact, but metadata service has fixed IP
|
||||
INSTANCE_ID=$(ec2metadata --instance-id)
|
||||
export INSTANCE_ID
|
||||
|
||||
bash /usr/local/share/scripts/init-network.sh
|
||||
|
||||
# combine labels
|
||||
RUNNER_TYPE=$(/usr/local/bin/aws ec2 describe-tags --filters "Name=resource-id,Values=$INSTANCE_ID" --query "Tags[?Key=='github:runner-type'].Value" --output text)
|
||||
LABELS="self-hosted,Linux,$(uname -m),$RUNNER_TYPE"
|
||||
export LABELS
|
||||
echo "Instance Labels: $LABELS"
|
||||
|
||||
LIFE_CYCLE=$(curl -s --fail http://169.254.169.254/latest/meta-data/instance-life-cycle)
|
||||
export LIFE_CYCLE
|
||||
echo "Instance lifecycle: $LIFE_CYCLE"
|
||||
|
||||
INSTANCE_TYPE=$(ec2metadata --instance-type)
|
||||
echo "Instance type: $INSTANCE_TYPE"
|
||||
|
||||
# Refresh CloudWatch agent config
|
||||
aws ssm get-parameter --region us-east-1 --name AmazonCloudWatch-github-runners --query 'Parameter.Value' --output text > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json
|
||||
systemctl restart amazon-cloudwatch-agent.service
|
||||
|
||||
# Refresh teams ssh keys
|
||||
TEAM_KEYS_URL=$(aws ssm get-parameter --region us-east-1 --name team-keys-url --query 'Parameter.Value' --output=text)
|
||||
curl -s "${TEAM_KEYS_URL}" > /home/ubuntu/.ssh/authorized_keys2
|
||||
chown ubuntu: /home/ubuntu/.ssh -R
|
||||
|
||||
|
||||
# Create a pre-run script that will provide diagnostics info
|
||||
mkdir -p /tmp/actions-hooks
|
||||
cat > /tmp/actions-hooks/common.sh << 'EOF'
|
||||
#!/bin/bash
|
||||
EOF
|
||||
|
||||
terminate_delayed() {
|
||||
# The function for post hook to gracefully finish the job and then tear down
|
||||
# The very specific sleep time is used later to determine in the main loop if
|
||||
# the instance is tearing down
|
||||
# IF `sleep` IS CHANGED, CHANGE ANOTHER VALUE IN `pgrep`
|
||||
sleep=13.14159265358979323846
|
||||
echo "Going to terminate the runner's instance in $sleep seconds"
|
||||
# We execute it with `at` to not have it as an orphan process, but launched independently
|
||||
# GH Runners kill all remain processes
|
||||
echo "sleep '$sleep'; aws ec2 terminate-instances --instance-ids $INSTANCE_ID" | at now || \
|
||||
aws ec2 terminate-instances --instance-ids "$INSTANCE_ID" # workaround for complete out of space or non-installed `at`
|
||||
exit 0
|
||||
}
|
||||
|
||||
detect_delayed_termination() {
|
||||
# The function look for very specific sleep with pi
|
||||
if pgrep 'sleep 13.14159265358979323846'; then
|
||||
echo 'The instance has delayed termination, sleep the same time to wait if it goes down'
|
||||
sleep 14
|
||||
fi
|
||||
}
|
||||
|
||||
declare -f terminate_delayed >> /tmp/actions-hooks/common.sh
|
||||
|
||||
terminate_and_exit() {
|
||||
# Terminate instance and exit from the script instantly
|
||||
echo "Going to terminate the runner's instance"
|
||||
aws ec2 terminate-instances --instance-ids "$INSTANCE_ID"
|
||||
exit 0
|
||||
}
|
||||
|
||||
terminate_decrease_and_exit() {
|
||||
# Terminate instance and exit from the script instantly
|
||||
echo "Going to terminate the runner's instance and decrease asg capacity"
|
||||
aws autoscaling terminate-instance-in-auto-scaling-group --instance-id "$INSTANCE_ID" --should-decrement-desired-capacity
|
||||
exit 0
|
||||
}
|
||||
|
||||
declare -f terminate_and_exit >> /tmp/actions-hooks/common.sh
|
||||
|
||||
check_spot_instance_is_old() {
|
||||
if [ "$LIFE_CYCLE" == "spot" ]; then
|
||||
local UPTIME
|
||||
UPTIME=$(< /proc/uptime)
|
||||
UPTIME=${UPTIME%%.*}
|
||||
if (( 3600 < UPTIME )); then
|
||||
echo "The spot instance has uptime $UPTIME, it's time to shut it down"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
check_proceed_spot_termination() {
|
||||
# The function checks and proceeds spot instance termination if exists
|
||||
# The event for spot instance termination
|
||||
local FORCE
|
||||
FORCE=${1:-}
|
||||
if TERMINATION_DATA=$(curl -s --fail http://169.254.169.254/latest/meta-data/spot/instance-action); then
|
||||
# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-instance-termination-notices.html#instance-action-metadata
|
||||
_action=$(jq '.action' -r <<< "$TERMINATION_DATA")
|
||||
_time=$(jq '.time | fromdate' <<< "$TERMINATION_DATA")
|
||||
_until_action=$((_time - $(date +%s)))
|
||||
echo "Received the '$_action' event that will be effective in $_until_action seconds"
|
||||
if (( _until_action <= 30 )) || [ "$FORCE" == "force" ]; then
|
||||
echo "The action $_action will be done in $_until_action, killing the runner and exit"
|
||||
local runner_pid
|
||||
runner_pid=$(pgrep Runner.Listener)
|
||||
if [ -n "$runner_pid" ]; then
|
||||
# Kill the runner to not allow it cancelling the job
|
||||
# shellcheck disable=SC2046
|
||||
kill -9 "$runner_pid" $(list_children "$runner_pid")
|
||||
fi
|
||||
sudo -u ubuntu ./config.sh remove --token "$(get_runner_token)"
|
||||
terminate_and_exit
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
no_terminating_metadata() {
|
||||
# The function check that instance could continue work
|
||||
# Returns 1 if any of termination events are received
|
||||
|
||||
# The event for rebalance recommendation. Not strict, so we have some room to make a decision here
|
||||
if curl -s --fail http://169.254.169.254/latest/meta-data/events/recommendations/rebalance; then
|
||||
echo 'Received recommendation to rebalance, checking the uptime'
|
||||
local UPTIME
|
||||
UPTIME=$(< /proc/uptime)
|
||||
UPTIME=${UPTIME%%.*}
|
||||
# We don't shutdown the instances younger than 30m
|
||||
if (( 1800 < UPTIME )); then
|
||||
# To not shutdown everything at once, use the 66% to survive
|
||||
if (( $((RANDOM % 3)) == 0 )); then
|
||||
echo 'The instance is older than 30m and won the roulette'
|
||||
return 1
|
||||
fi
|
||||
echo 'The instance is older than 30m, but is not chosen for rebalance'
|
||||
else
|
||||
echo 'The instance is younger than 30m, do not shut it down'
|
||||
fi
|
||||
fi
|
||||
|
||||
# Checks if the ASG in a lifecycle hook state
|
||||
local ASG_STATUS
|
||||
ASG_STATUS=$(curl -s http://169.254.169.254/latest/meta-data/autoscaling/target-lifecycle-state)
|
||||
if [ "$ASG_STATUS" == "Terminated" ]; then
|
||||
echo 'The instance in ASG status Terminating:Wait'
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
terminate_on_event() {
|
||||
# If there is a rebalance event, then the instance could die soon
|
||||
# Let's don't wait for it and terminate proactively
|
||||
if curl -s --fail http://169.254.169.254/latest/meta-data/events/recommendations/rebalance; then
|
||||
terminate_and_exit
|
||||
fi
|
||||
|
||||
# Here we check if the autoscaling group marked the instance for termination, and it's wait for the job to finish
|
||||
ASG_STATUS=$(curl -s http://169.254.169.254/latest/meta-data/autoscaling/target-lifecycle-state)
|
||||
if [ "$ASG_STATUS" == "Terminated" ]; then
|
||||
INSTANCE_ID=$(ec2metadata --instance-id)
|
||||
ASG_NAME=$(aws ec2 describe-tags --filters "Name=resource-id,Values=$INSTANCE_ID" --query "Tags[?Key=='aws:autoscaling:groupName'].Value" --output text)
|
||||
LIFECYCLE_HOOKS=$(aws autoscaling describe-lifecycle-hooks --auto-scaling-group-name "$ASG_NAME" --query "LifecycleHooks[].LifecycleHookName" --output text)
|
||||
for LCH in $LIFECYCLE_HOOKS; do
|
||||
aws autoscaling complete-lifecycle-action --lifecycle-action-result CONTINUE \
|
||||
--lifecycle-hook-name "$LCH" --auto-scaling-group-name "$ASG_NAME" \
|
||||
--instance-id "$INSTANCE_ID"
|
||||
true # autoformat issue
|
||||
done
|
||||
echo 'The runner is marked as "Terminated" by the autoscaling group, we are terminating'
|
||||
terminate_and_exit
|
||||
fi
|
||||
}
|
||||
|
||||
cat > /tmp/actions-hooks/pre-run.sh << EOF
|
||||
#!/bin/bash
|
||||
set -uo pipefail
|
||||
|
||||
echo "Runner's public DNS: $(ec2metadata --public-hostname)"
|
||||
echo "Runner's labels: ${LABELS}"
|
||||
echo "Runner's instance type: $(ec2metadata --instance-type)"
|
||||
EOF
|
||||
|
||||
# Create a post-run script that will restart docker daemon before the job started
|
||||
cat > /tmp/actions-hooks/post-run.sh << 'EOF'
|
||||
#!/bin/bash
|
||||
set -xuo pipefail
|
||||
|
||||
source /tmp/actions-hooks/common.sh
|
||||
|
||||
# Free KiB, free percents
|
||||
ROOT_STAT=($(df / | awk '/\// {print $4 " " int($4/$2 * 100)}'))
|
||||
if [[ ${ROOT_STAT[0]} -lt 3000000 ]] || [[ ${ROOT_STAT[1]} -lt 5 ]]; then
|
||||
echo "The runner has ${ROOT_STAT[0]}KiB and ${ROOT_STAT[1]}% of free space on /"
|
||||
terminate_delayed
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2046
|
||||
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
|
||||
# shellcheck disable=SC2046
|
||||
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
|
||||
|
||||
# If we have hanged containers after the previous commands, than we have a hanged one
|
||||
# and should restart the daemon
|
||||
if [ "$(docker ps --all --quiet)" ]; then
|
||||
# Systemd service of docker has StartLimitBurst=3 and StartLimitInterval=60s,
|
||||
# that's why we try restarting it for long
|
||||
for i in {1..25};
|
||||
do
|
||||
sudo systemctl restart docker && break || sleep 5
|
||||
done
|
||||
|
||||
for i in {1..10}
|
||||
do
|
||||
docker info && break || sleep 2
|
||||
done
|
||||
# Last chance, otherwise we have to terminate poor instance
|
||||
docker info 1>/dev/null || { echo Docker unable to start; terminate_delayed ; }
|
||||
fi
|
||||
EOF
|
||||
|
||||
get_runner_token() {
|
||||
/usr/local/bin/aws ssm get-parameter --name github_runner_registration_token --with-decryption --output text --query Parameter.Value
|
||||
}
|
||||
|
||||
is_job_assigned() {
|
||||
local runner_pid
|
||||
runner_pid=$(pgrep Runner.Listener)
|
||||
if [ -z "$runner_pid" ]; then
|
||||
# if runner has finished, it's fine
|
||||
return 0
|
||||
fi
|
||||
local log_file
|
||||
log_file=$(lsof -p "$runner_pid" 2>/dev/null | grep -o "$RUNNER_HOME/_diag/Runner.*log")
|
||||
if [ -z "$log_file" ]; then
|
||||
# assume, the process is over or just started
|
||||
return 0
|
||||
fi
|
||||
# So far it's the only solid way to determine that the job is starting
|
||||
grep -q 'Terminal] .* Running job:' "$log_file" \
|
||||
&& return 0 \
|
||||
|| return 1
|
||||
}
|
||||
|
||||
list_children () {
|
||||
local children
|
||||
children=$(ps --ppid "$1" -o pid=)
|
||||
if [ -z "$children" ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
for pid in $children; do
|
||||
list_children "$pid"
|
||||
done
|
||||
echo "$children"
|
||||
}
|
||||
|
||||
# There's possibility that it fails because the runner's version is outdated,
|
||||
# so after the first failure we'll try to launch it with enabled autoupdate.
|
||||
#
|
||||
# We'll fail and terminate after 10 consequent failures.
|
||||
ATTEMPT=0
|
||||
# In `kill` 0 means "all processes in process group", -1 is "all but PID 1"
|
||||
# We use `-2` to get an error
|
||||
RUNNER_PID=-2
|
||||
|
||||
while true; do
|
||||
# Does not send signal, but checks that the process $RUNNER_PID is running
|
||||
if kill -0 -- $RUNNER_PID; then
|
||||
ATTEMPT=0
|
||||
echo "Runner is working with pid $RUNNER_PID, checking the metadata in background"
|
||||
check_proceed_spot_termination
|
||||
|
||||
if ! is_job_assigned; then
|
||||
RUNNER_AGE=$(( $(date +%s) - $(stat -c +%Y /proc/"$RUNNER_PID" 2>/dev/null || date +%s) ))
|
||||
echo "The runner is launched $RUNNER_AGE seconds ago and still hasn't received a job"
|
||||
if (( 60 < RUNNER_AGE )); then
|
||||
echo "Attempt to delete the runner for a graceful shutdown"
|
||||
sudo -u ubuntu ./config.sh remove --token "$(get_runner_token)" \
|
||||
|| continue
|
||||
echo "Runner didn't launch or have assigned jobs after ${RUNNER_AGE} seconds, shutting down"
|
||||
terminate_decrease_and_exit
|
||||
fi
|
||||
fi
|
||||
else
|
||||
if [ "$RUNNER_PID" != "-2" ]; then
|
||||
wait $RUNNER_PID \
|
||||
&& echo "Runner with PID $RUNNER_PID successfully finished" \
|
||||
|| echo "Attempt $((++ATTEMPT)) to start the runner"
|
||||
fi
|
||||
if (( ATTEMPT > 10 )); then
|
||||
echo "The runner has failed to start after $ATTEMPT attempt. Give up and terminate it"
|
||||
terminate_and_exit
|
||||
fi
|
||||
|
||||
cd $RUNNER_HOME || terminate_and_exit
|
||||
detect_delayed_termination
|
||||
# If runner is not active, check that it needs to terminate itself
|
||||
echo "Checking if the instance suppose to terminate"
|
||||
no_terminating_metadata || terminate_on_event
|
||||
check_spot_instance_is_old && terminate_and_exit
|
||||
check_proceed_spot_termination force
|
||||
|
||||
echo "Going to configure runner"
|
||||
token_args=(--token "$(get_runner_token)")
|
||||
config_args=(
|
||||
"${token_args[@]}" --url "$RUNNER_URL"
|
||||
--ephemeral --unattended --replace --runnergroup Default
|
||||
--labels "$LABELS" --work _work --name "$INSTANCE_ID"
|
||||
)
|
||||
if (( ATTEMPT > 1 )); then
|
||||
echo 'The runner failed to start at least once. Removing it and then configuring with autoupdate enabled.'
|
||||
sudo -u ubuntu ./config.sh remove "${token_args[@]}"
|
||||
sudo -u ubuntu ./config.sh "${config_args[@]}"
|
||||
else
|
||||
echo "Configure runner with disabled autoupdate"
|
||||
config_args+=("--disableupdate")
|
||||
sudo -u ubuntu ./config.sh "${config_args[@]}"
|
||||
fi
|
||||
|
||||
echo "Another one check to avoid race between runner and infrastructure"
|
||||
no_terminating_metadata || terminate_on_event
|
||||
check_spot_instance_is_old && terminate_and_exit
|
||||
check_proceed_spot_termination force
|
||||
|
||||
# There were some failures to start the Job because of trash in _work
|
||||
rm -rf _work
|
||||
|
||||
# https://github.com/actions/runner/issues/3266
|
||||
# We're unable to know if the runner is failed to start.
|
||||
echo 'Monkey-patching run helpers to get genuine exit code of the runner'
|
||||
for script in run.sh run-helper.sh.template; do
|
||||
# shellcheck disable=SC2016
|
||||
grep -q 'exit 0$' "$script" && \
|
||||
sed 's/exit 0/exit $returnCode/' -i "$script" && \
|
||||
echo "Script $script is patched"
|
||||
done
|
||||
|
||||
echo "Run"
|
||||
sudo -u ubuntu \
|
||||
ACTIONS_RUNNER_HOOK_JOB_STARTED=/tmp/actions-hooks/pre-run.sh \
|
||||
ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/tmp/actions-hooks/post-run.sh \
|
||||
./run.sh &
|
||||
RUNNER_PID=$!
|
||||
|
||||
sleep 10
|
||||
fi
|
||||
|
||||
sleep 5
|
||||
done
|
||||
|
||||
# vim:ts=4:sw=4
|
||||
@ -104,3 +104,24 @@ def test_keeper_map_without_zk(started_cluster):
|
||||
node.query("DETACH TABLE test_keeper_map_without_zk")
|
||||
|
||||
client.stop()
|
||||
|
||||
|
||||
def test_keeper_map_with_failed_drop(started_cluster):
|
||||
run_query(
|
||||
"CREATE TABLE test_keeper_map_with_failed_drop (key UInt64, value UInt64) ENGINE = KeeperMap('/test_keeper_map_with_failed_drop') PRIMARY KEY(key);"
|
||||
)
|
||||
|
||||
run_query("INSERT INTO test_keeper_map_with_failed_drop VALUES (1, 11)")
|
||||
run_query("SYSTEM ENABLE FAILPOINT keepermap_fail_drop_data")
|
||||
node.query("DROP TABLE test_keeper_map_with_failed_drop SYNC")
|
||||
|
||||
zk_client = get_genuine_zk()
|
||||
assert (
|
||||
zk_client.get("/test_keeper_map/test_keeper_map_with_failed_drop/data")
|
||||
is not None
|
||||
)
|
||||
|
||||
run_query("SYSTEM DISABLE FAILPOINT keepermap_fail_drop_data")
|
||||
run_query(
|
||||
"CREATE TABLE test_keeper_map_with_failed_drop_another (key UInt64, value UInt64) ENGINE = KeeperMap('/test_keeper_map_with_failed_drop') PRIMARY KEY(key);"
|
||||
)
|
||||
|
||||
@ -1,126 +0,0 @@
|
||||
12 4 21722 2209341 4 1415 2333 4 61 64 3
|
||||
21 1134 11363 58484 1106 1458 1592 136 26 62 32
|
||||
22 210 4504 5729 196 291 767 124 47 54 8
|
||||
26 196 1327684 5221 195 4140 5661 161 28 49 19
|
||||
28 5 2034378 7102 5 325 3255 2 53 60 4
|
||||
29 53 45041 45189 45 1580 211 31 55 84 18
|
||||
38 424 1600675 4653 424 562 5944 244 60 65 6
|
||||
45 17 62743 674873 17 6239 6494 17 65 76 8
|
||||
72 1862 1210073 6200 1677 2498 528 859 51 61 11
|
||||
79 2 2255228 2255293 2 5495 7057 2 65 65 1
|
||||
85 459 1051571 1829638 459 6402 7131 334 32 61 25
|
||||
86 10 1748130 1754217 10 4374 7003 10 56 59 4
|
||||
91 165 5718 5802 75 282 7113 112 41 63 22
|
||||
94 20 1231916 2050003 20 4802 4917 19 53 59 7
|
||||
99 2 3665 36667 2 497 697 2 70 71 2
|
||||
103 1 2446615 2446615 1 2498 2498 1 58 58 1
|
||||
106 72 6149 6699 67 527 826 40 61 61 1
|
||||
111 43 2273186 5272 43 492 4923 4 54 72 15
|
||||
120 3129 45117 6735 2868 1030 1625 561 59 64 6
|
||||
138 2 49243 49374 2 1428 1519 2 47 48 2
|
||||
143 100 23321 63639 100 1115 1624 88 51 51 1
|
||||
145 1 2447976 2447976 1 6173 6173 1 44 44 1
|
||||
153 16 13748 16881 16 1506 1636 16 54 68 9
|
||||
159 19952 1525336 7131 12957 1280 6163 2668 24 66 39
|
||||
171 5 15042 16698 5 1302 1608 5 65 65 1
|
||||
179 6264 1362341 2686 6244 2554 7132 2705 61 67 7
|
||||
192 1 1639623 1639623 1 3406 3406 1 32 32 1
|
||||
193 1 1429969 1429969 1 7131 7131 1 45 45 1
|
||||
207 12 23057 32500 12 1491 1726 12 32 46 7
|
||||
221 5081 1366870 6649 3432 4527 5226 687 24 69 39
|
||||
228 73 12281 17929 71 1328 2034 63 49 71 18
|
||||
229 2 1617478 1723791 2 4590 5578 2 41 42 2
|
||||
230 3916 1332729 6949 3668 1330 4703 845 62 65 4
|
||||
238 25 2624456 2625673 24 2535 6465 25 58 75 14
|
||||
241 154 2554929 2616444 154 2626 7131 148 34 57 17
|
||||
248 276 15529 30987 274 1040 1222 136 37 79 27
|
||||
254 3018 33966 6635 2837 1057 1622 539 24 60 33
|
||||
255 20 1581774 1811334 20 6068 6301 18 33 57 10
|
||||
256 5 5145 6841 5 367 376 5 58 58 1
|
||||
270 2 2195579 2262119 2 7102 7123 2 33 34 2
|
||||
281 32 2379460 616215 32 6042 6086 23 53 64 12
|
||||
282 7 1292651 24244 7 1607 2455 6 46 55 5
|
||||
286 123 1521935 5269 123 3793 3940 81 40 66 22
|
||||
291 21 2419080 3567 21 297 4731 21 54 55 2
|
||||
316 4 5221 5616 4 505 558 4 32 35 3
|
||||
319 232 56480 63033 230 1599 313 50 33 64 26
|
||||
327 15 51647 51894 14 1292 1585 14 47 57 7
|
||||
332 24 23484 54948 24 1609 1726 16 32 49 11
|
||||
333 1 14189 14189 1 1550 1550 1 63 63 1
|
||||
342 49 2579220 2622432 49 4626 6933 48 34 54 14
|
||||
344 1 6486 6486 1 509 509 1 24 24 1
|
||||
346 1987 53016 6735 1823 1334 174 294 26 62 32
|
||||
358 45 59058 60844 44 6746 722 40 57 84 15
|
||||
363 1198 1260033 2568811 1196 5710 5790 82 55 80 26
|
||||
384 150 2361175 476024 150 7008 7123 81 38 64 22
|
||||
387 277 5200 6553 252 243 521 130 65 65 1
|
||||
392 1877 1607428 2030850 1875 1416 7131 1379 54 66 13
|
||||
396 8181 1380803 6186 7920 545 798 1743 24 67 39
|
||||
398 3 5183 5213 2 291 352 3 53 59 3
|
||||
399 62 51494 59203 61 7073 754 42 55 78 18
|
||||
412 2141 1360120 2189792 2136 2491 5658 1371 71 75 5
|
||||
413 2 2036037 2064917 2 3963 4666 2 43 45 2
|
||||
431 33 2302331 2348449 33 4425 6516 32 69 69 1
|
||||
447 59 25125 33094 59 1176 1817 56 53 58 6
|
||||
456 1 53157 53157 1 1556 1556 1 26 26 1
|
||||
462 5 5456 6280 5 348 4337 5 28 40 5
|
||||
472 1 1443716 1443716 1 6122 6122 1 42 42 1
|
||||
491 34 1066102 1183673 34 6606 6822 32 46 67 15
|
||||
498 896 2230163 3054 895 537 7131 714 24 59 28
|
||||
504 108 12281 25180 108 1318 1784 94 55 66 12
|
||||
515 22 1588883 2640809 22 6554 6571 15 46 59 12
|
||||
518 1 37743 37743 1 1558 1558 1 72 72 1
|
||||
530 1 3033 3033 1 561 561 1 59 59 1
|
||||
532 26 5721 6355 25 549 665 14 44 50 7
|
||||
546 156 2577874 48517 156 1105 324 133 44 51 8
|
||||
554 12 1665194 2640066 12 1817 2951 12 57 57 1
|
||||
564 3865 2028049 2083433 3722 1115 985 2203 44 84 41
|
||||
566 4432 50605 57509 3217 1191 267 459 26 72 39
|
||||
567 8 5221 5893 7 333 558 8 27 35 4
|
||||
582 1172 1320619 2019743 1172 5819 7131 757 26 63 30
|
||||
584 43100 2500 5594 22561 134 4573 1660 48 84 37
|
||||
589 28 6046 6068 19 345 564 27 55 62 8
|
||||
595 139 1585165 1683606 138 2231 3598 132 54 84 28
|
||||
615 3 1056081 1116230 3 5794 5796 2 59 62 3
|
||||
619 7 1543114 5241 7 2442 3105 7 41 45 3
|
||||
634 2722 1221058 4999 2686 2426 7131 1735 54 60 7
|
||||
635 237 2119333 4667 237 561 5999 176 49 60 12
|
||||
644 5 1774169 2056171 5 5591 6091 4 33 39 3
|
||||
647 8 51632 64403 8 1457 1624 8 26 34 5
|
||||
651 1325 1620565 6281 1301 528 792 815 62 63 2
|
||||
665 13 4598 4789 13 511 558 11 39 46 7
|
||||
679 1560 1613200 25940 1552 1569 3118 781 49 84 35
|
||||
704 2 14226 15594 2 1086 1116 2 65 71 2
|
||||
715 25 1199352 3490 25 5036 5112 23 34 55 13
|
||||
716 1253 61989 6735 1050 1203 1625 397 52 65 14
|
||||
730 2584 5560 6170 634 2421 627 293 56 69 14
|
||||
736 8 1433153 4941 8 339 4594 8 28 36 5
|
||||
749 2 1326176 1339862 2 4339 6213 2 49 50 2
|
||||
753 1 53157 53157 1 1556 1556 1 26 26 1
|
||||
761 63 1443230 6881 63 3154 3204 26 56 73 14
|
||||
762 49 1449596 1968154 49 2437 3753 48 54 62 9
|
||||
775 35107 5330 769436 2471 447 6607 656 70 81 12
|
||||
789 1 1552458 1552458 1 2441 2441 1 62 62 1
|
||||
794 158 5585 6585 155 495 929 67 24 50 20
|
||||
839 9 29223 46530 9 1336 1465 9 52 52 1
|
||||
844 5 2377545 2377635 5 5129 6321 5 53 69 5
|
||||
846 50 2172273 2589295 50 1582 3053 48 64 68 5
|
||||
847 2577 56656 63658 1582 1444 838 474 26 63 33
|
||||
861 1333 5570 6909 839 457 489 37 33 70 34
|
||||
873 2360 1519811 50487 2248 1310 1784 316 60 68 9
|
||||
879 228 6704 6785 79 279 507 121 35 66 24
|
||||
889 5130 2070007 39692 5040 1151 6791 2606 44 66 23
|
||||
896 4 511246 859452 4 6554 6561 4 67 71 4
|
||||
912 146 1322641 2238040 146 1366 6354 143 59 59 1
|
||||
913 82 5495 6870 78 350 565 67 24 43 15
|
||||
921 763 1580790 416881 763 6191 7131 509 63 64 2
|
||||
925 318 2500952 5025 309 476 6114 182 32 56 21
|
||||
931 12 4277 4809 12 238 256 9 63 83 9
|
||||
942 954 1331 2228193 952 1121 5047 788 65 70 6
|
||||
948 14 1785593 2600431 14 6550 6598 13 34 49 9
|
||||
956 5 5755 6023 5 359 411 5 43 48 4
|
||||
963 4 3812 3835 4 444 537 4 47 53 4
|
||||
978 5 51632 58212 5 1127 1556 5 24 32 5
|
||||
980 53 47201 59744 53 1537 1625 36 41 49 9
|
||||
987 6033 2020131 763444 4306 256 792 1832 60 64 5
|
||||
993 4 1615159 1718339 4 1570 3093 4 62 63 2
|
||||
15
tests/queries/0_stateless/02995_index_1.reference
Normal file
15
tests/queries/0_stateless/02995_index_1.reference
Normal file
@ -0,0 +1,15 @@
|
||||
12 4 21722 2209341 4 1415 2333 4 61 64 3
|
||||
21 1134 11363 58484 1106 1458 1592 136 26 62 32
|
||||
22 210 4504 5729 196 291 767 124 47 54 8
|
||||
26 196 1327684 5221 195 4140 5661 161 28 49 19
|
||||
28 5 2034378 7102 5 325 3255 2 53 60 4
|
||||
29 53 45041 45189 45 1580 211 31 55 84 18
|
||||
38 424 1600675 4653 424 562 5944 244 60 65 6
|
||||
45 17 62743 674873 17 6239 6494 17 65 76 8
|
||||
72 1862 1210073 6200 1677 2498 528 859 51 61 11
|
||||
79 2 2255228 2255293 2 5495 7057 2 65 65 1
|
||||
85 459 1051571 1829638 459 6402 7131 334 32 61 25
|
||||
86 10 1748130 1754217 10 4374 7003 10 56 59 4
|
||||
91 165 5718 5802 75 282 7113 112 41 63 22
|
||||
94 20 1231916 2050003 20 4802 4917 19 53 59 7
|
||||
99 2 3665 36667 2 497 697 2 70 71 2
|
||||
@ -26,7 +26,7 @@ DETACH TABLE test;
|
||||
ATTACH TABLE test;
|
||||
"
|
||||
|
||||
for i in {1..1000}
|
||||
for i in {1..100}
|
||||
do
|
||||
echo "
|
||||
WITH ${i} AS try
|
||||
13
tests/queries/0_stateless/02995_index_10.reference
Normal file
13
tests/queries/0_stateless/02995_index_10.reference
Normal file
@ -0,0 +1,13 @@
|
||||
912 146 1322641 2238040 146 1366 6354 143 59 59 1
|
||||
913 82 5495 6870 78 350 565 67 24 43 15
|
||||
921 763 1580790 416881 763 6191 7131 509 63 64 2
|
||||
925 318 2500952 5025 309 476 6114 182 32 56 21
|
||||
931 12 4277 4809 12 238 256 9 63 83 9
|
||||
942 954 1331 2228193 952 1121 5047 788 65 70 6
|
||||
948 14 1785593 2600431 14 6550 6598 13 34 49 9
|
||||
956 5 5755 6023 5 359 411 5 43 48 4
|
||||
963 4 3812 3835 4 444 537 4 47 53 4
|
||||
978 5 51632 58212 5 1127 1556 5 24 32 5
|
||||
980 53 47201 59744 53 1537 1625 36 41 49 9
|
||||
987 6033 2020131 763444 4306 256 792 1832 60 64 5
|
||||
993 4 1615159 1718339 4 1570 3093 4 62 63 2
|
||||
44
tests/queries/0_stateless/02995_index_10.sh
Executable file
44
tests/queries/0_stateless/02995_index_10.sh
Executable file
@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiquery "
|
||||
|
||||
DROP TABLE IF EXISTS test;
|
||||
CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
DETACH TABLE test;
|
||||
ATTACH TABLE test;
|
||||
"
|
||||
|
||||
for i in {901..1000}
|
||||
do
|
||||
echo "
|
||||
WITH ${i} AS try
|
||||
SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
|
||||
WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
|
||||
AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
|
||||
AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
|
||||
AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
|
||||
AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
|
||||
AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
|
||||
HAVING count() > 0;
|
||||
"
|
||||
done | ${CLICKHOUSE_CLIENT} --multiquery
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
|
||||
13
tests/queries/0_stateless/02995_index_2.reference
Normal file
13
tests/queries/0_stateless/02995_index_2.reference
Normal file
@ -0,0 +1,13 @@
|
||||
103 1 2446615 2446615 1 2498 2498 1 58 58 1
|
||||
106 72 6149 6699 67 527 826 40 61 61 1
|
||||
111 43 2273186 5272 43 492 4923 4 54 72 15
|
||||
120 3129 45117 6735 2868 1030 1625 561 59 64 6
|
||||
138 2 49243 49374 2 1428 1519 2 47 48 2
|
||||
143 100 23321 63639 100 1115 1624 88 51 51 1
|
||||
145 1 2447976 2447976 1 6173 6173 1 44 44 1
|
||||
153 16 13748 16881 16 1506 1636 16 54 68 9
|
||||
159 19952 1525336 7131 12957 1280 6163 2668 24 66 39
|
||||
171 5 15042 16698 5 1302 1608 5 65 65 1
|
||||
179 6264 1362341 2686 6244 2554 7132 2705 61 67 7
|
||||
192 1 1639623 1639623 1 3406 3406 1 32 32 1
|
||||
193 1 1429969 1429969 1 7131 7131 1 45 45 1
|
||||
44
tests/queries/0_stateless/02995_index_2.sh
Executable file
44
tests/queries/0_stateless/02995_index_2.sh
Executable file
@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiquery "
|
||||
|
||||
DROP TABLE IF EXISTS test;
|
||||
CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
DETACH TABLE test;
|
||||
ATTACH TABLE test;
|
||||
"
|
||||
|
||||
for i in {101..200}
|
||||
do
|
||||
echo "
|
||||
WITH ${i} AS try
|
||||
SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
|
||||
WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
|
||||
AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
|
||||
AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
|
||||
AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
|
||||
AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
|
||||
AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
|
||||
HAVING count() > 0;
|
||||
"
|
||||
done | ${CLICKHOUSE_CLIENT} --multiquery
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
|
||||
16
tests/queries/0_stateless/02995_index_3.reference
Normal file
16
tests/queries/0_stateless/02995_index_3.reference
Normal file
@ -0,0 +1,16 @@
|
||||
207 12 23057 32500 12 1491 1726 12 32 46 7
|
||||
221 5081 1366870 6649 3432 4527 5226 687 24 69 39
|
||||
228 73 12281 17929 71 1328 2034 63 49 71 18
|
||||
229 2 1617478 1723791 2 4590 5578 2 41 42 2
|
||||
230 3916 1332729 6949 3668 1330 4703 845 62 65 4
|
||||
238 25 2624456 2625673 24 2535 6465 25 58 75 14
|
||||
241 154 2554929 2616444 154 2626 7131 148 34 57 17
|
||||
248 276 15529 30987 274 1040 1222 136 37 79 27
|
||||
254 3018 33966 6635 2837 1057 1622 539 24 60 33
|
||||
255 20 1581774 1811334 20 6068 6301 18 33 57 10
|
||||
256 5 5145 6841 5 367 376 5 58 58 1
|
||||
270 2 2195579 2262119 2 7102 7123 2 33 34 2
|
||||
281 32 2379460 616215 32 6042 6086 23 53 64 12
|
||||
282 7 1292651 24244 7 1607 2455 6 46 55 5
|
||||
286 123 1521935 5269 123 3793 3940 81 40 66 22
|
||||
291 21 2419080 3567 21 297 4731 21 54 55 2
|
||||
44
tests/queries/0_stateless/02995_index_3.sh
Executable file
44
tests/queries/0_stateless/02995_index_3.sh
Executable file
@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiquery "
|
||||
|
||||
DROP TABLE IF EXISTS test;
|
||||
CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
DETACH TABLE test;
|
||||
ATTACH TABLE test;
|
||||
"
|
||||
|
||||
for i in {201..300}
|
||||
do
|
||||
echo "
|
||||
WITH ${i} AS try
|
||||
SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
|
||||
WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
|
||||
AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
|
||||
AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
|
||||
AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
|
||||
AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
|
||||
AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
|
||||
HAVING count() > 0;
|
||||
"
|
||||
done | ${CLICKHOUSE_CLIENT} --multiquery
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
|
||||
16
tests/queries/0_stateless/02995_index_4.reference
Normal file
16
tests/queries/0_stateless/02995_index_4.reference
Normal file
@ -0,0 +1,16 @@
|
||||
316 4 5221 5616 4 505 558 4 32 35 3
|
||||
319 232 56480 63033 230 1599 313 50 33 64 26
|
||||
327 15 51647 51894 14 1292 1585 14 47 57 7
|
||||
332 24 23484 54948 24 1609 1726 16 32 49 11
|
||||
333 1 14189 14189 1 1550 1550 1 63 63 1
|
||||
342 49 2579220 2622432 49 4626 6933 48 34 54 14
|
||||
344 1 6486 6486 1 509 509 1 24 24 1
|
||||
346 1987 53016 6735 1823 1334 174 294 26 62 32
|
||||
358 45 59058 60844 44 6746 722 40 57 84 15
|
||||
363 1198 1260033 2568811 1196 5710 5790 82 55 80 26
|
||||
384 150 2361175 476024 150 7008 7123 81 38 64 22
|
||||
387 277 5200 6553 252 243 521 130 65 65 1
|
||||
392 1877 1607428 2030850 1875 1416 7131 1379 54 66 13
|
||||
396 8181 1380803 6186 7920 545 798 1743 24 67 39
|
||||
398 3 5183 5213 2 291 352 3 53 59 3
|
||||
399 62 51494 59203 61 7073 754 42 55 78 18
|
||||
44
tests/queries/0_stateless/02995_index_4.sh
Executable file
44
tests/queries/0_stateless/02995_index_4.sh
Executable file
@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiquery "
|
||||
|
||||
DROP TABLE IF EXISTS test;
|
||||
CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
DETACH TABLE test;
|
||||
ATTACH TABLE test;
|
||||
"
|
||||
|
||||
for i in {301..400}
|
||||
do
|
||||
echo "
|
||||
WITH ${i} AS try
|
||||
SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
|
||||
WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
|
||||
AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
|
||||
AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
|
||||
AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
|
||||
AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
|
||||
AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
|
||||
HAVING count() > 0;
|
||||
"
|
||||
done | ${CLICKHOUSE_CLIENT} --multiquery
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
|
||||
9
tests/queries/0_stateless/02995_index_5.reference
Normal file
9
tests/queries/0_stateless/02995_index_5.reference
Normal file
@ -0,0 +1,9 @@
|
||||
412 2141 1360120 2189792 2136 2491 5658 1371 71 75 5
|
||||
413 2 2036037 2064917 2 3963 4666 2 43 45 2
|
||||
431 33 2302331 2348449 33 4425 6516 32 69 69 1
|
||||
447 59 25125 33094 59 1176 1817 56 53 58 6
|
||||
456 1 53157 53157 1 1556 1556 1 26 26 1
|
||||
462 5 5456 6280 5 348 4337 5 28 40 5
|
||||
472 1 1443716 1443716 1 6122 6122 1 42 42 1
|
||||
491 34 1066102 1183673 34 6606 6822 32 46 67 15
|
||||
498 896 2230163 3054 895 537 7131 714 24 59 28
|
||||
44
tests/queries/0_stateless/02995_index_5.sh
Executable file
44
tests/queries/0_stateless/02995_index_5.sh
Executable file
@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiquery "
|
||||
|
||||
DROP TABLE IF EXISTS test;
|
||||
CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
DETACH TABLE test;
|
||||
ATTACH TABLE test;
|
||||
"
|
||||
|
||||
for i in {401..500}
|
||||
do
|
||||
echo "
|
||||
WITH ${i} AS try
|
||||
SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
|
||||
WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
|
||||
AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
|
||||
AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
|
||||
AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
|
||||
AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
|
||||
AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
|
||||
HAVING count() > 0;
|
||||
"
|
||||
done | ${CLICKHOUSE_CLIENT} --multiquery
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
|
||||
14
tests/queries/0_stateless/02995_index_6.reference
Normal file
14
tests/queries/0_stateless/02995_index_6.reference
Normal file
@ -0,0 +1,14 @@
|
||||
504 108 12281 25180 108 1318 1784 94 55 66 12
|
||||
515 22 1588883 2640809 22 6554 6571 15 46 59 12
|
||||
518 1 37743 37743 1 1558 1558 1 72 72 1
|
||||
530 1 3033 3033 1 561 561 1 59 59 1
|
||||
532 26 5721 6355 25 549 665 14 44 50 7
|
||||
546 156 2577874 48517 156 1105 324 133 44 51 8
|
||||
554 12 1665194 2640066 12 1817 2951 12 57 57 1
|
||||
564 3865 2028049 2083433 3722 1115 985 2203 44 84 41
|
||||
566 4432 50605 57509 3217 1191 267 459 26 72 39
|
||||
567 8 5221 5893 7 333 558 8 27 35 4
|
||||
582 1172 1320619 2019743 1172 5819 7131 757 26 63 30
|
||||
584 43100 2500 5594 22561 134 4573 1660 48 84 37
|
||||
589 28 6046 6068 19 345 564 27 55 62 8
|
||||
595 139 1585165 1683606 138 2231 3598 132 54 84 28
|
||||
44
tests/queries/0_stateless/02995_index_6.sh
Executable file
44
tests/queries/0_stateless/02995_index_6.sh
Executable file
@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiquery "
|
||||
|
||||
DROP TABLE IF EXISTS test;
|
||||
CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
DETACH TABLE test;
|
||||
ATTACH TABLE test;
|
||||
"
|
||||
|
||||
for i in {501..600}
|
||||
do
|
||||
echo "
|
||||
WITH ${i} AS try
|
||||
SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
|
||||
WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
|
||||
AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
|
||||
AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
|
||||
AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
|
||||
AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
|
||||
AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
|
||||
HAVING count() > 0;
|
||||
"
|
||||
done | ${CLICKHOUSE_CLIENT} --multiquery
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
|
||||
9
tests/queries/0_stateless/02995_index_7.reference
Normal file
9
tests/queries/0_stateless/02995_index_7.reference
Normal file
@ -0,0 +1,9 @@
|
||||
615 3 1056081 1116230 3 5794 5796 2 59 62 3
|
||||
619 7 1543114 5241 7 2442 3105 7 41 45 3
|
||||
634 2722 1221058 4999 2686 2426 7131 1735 54 60 7
|
||||
635 237 2119333 4667 237 561 5999 176 49 60 12
|
||||
644 5 1774169 2056171 5 5591 6091 4 33 39 3
|
||||
647 8 51632 64403 8 1457 1624 8 26 34 5
|
||||
651 1325 1620565 6281 1301 528 792 815 62 63 2
|
||||
665 13 4598 4789 13 511 558 11 39 46 7
|
||||
679 1560 1613200 25940 1552 1569 3118 781 49 84 35
|
||||
44
tests/queries/0_stateless/02995_index_7.sh
Executable file
44
tests/queries/0_stateless/02995_index_7.sh
Executable file
@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiquery "
|
||||
|
||||
DROP TABLE IF EXISTS test;
|
||||
CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
DETACH TABLE test;
|
||||
ATTACH TABLE test;
|
||||
"
|
||||
|
||||
for i in {601..700}
|
||||
do
|
||||
echo "
|
||||
WITH ${i} AS try
|
||||
SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
|
||||
WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
|
||||
AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
|
||||
AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
|
||||
AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
|
||||
AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
|
||||
AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
|
||||
HAVING count() > 0;
|
||||
"
|
||||
done | ${CLICKHOUSE_CLIENT} --multiquery
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
|
||||
12
tests/queries/0_stateless/02995_index_8.reference
Normal file
12
tests/queries/0_stateless/02995_index_8.reference
Normal file
@ -0,0 +1,12 @@
|
||||
704 2 14226 15594 2 1086 1116 2 65 71 2
|
||||
715 25 1199352 3490 25 5036 5112 23 34 55 13
|
||||
716 1253 61989 6735 1050 1203 1625 397 52 65 14
|
||||
730 2584 5560 6170 634 2421 627 293 56 69 14
|
||||
736 8 1433153 4941 8 339 4594 8 28 36 5
|
||||
749 2 1326176 1339862 2 4339 6213 2 49 50 2
|
||||
753 1 53157 53157 1 1556 1556 1 26 26 1
|
||||
761 63 1443230 6881 63 3154 3204 26 56 73 14
|
||||
762 49 1449596 1968154 49 2437 3753 48 54 62 9
|
||||
775 35107 5330 769436 2471 447 6607 656 70 81 12
|
||||
789 1 1552458 1552458 1 2441 2441 1 62 62 1
|
||||
794 158 5585 6585 155 495 929 67 24 50 20
|
||||
44
tests/queries/0_stateless/02995_index_8.sh
Executable file
44
tests/queries/0_stateless/02995_index_8.sh
Executable file
@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiquery "
|
||||
|
||||
DROP TABLE IF EXISTS test;
|
||||
CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
DETACH TABLE test;
|
||||
ATTACH TABLE test;
|
||||
"
|
||||
|
||||
for i in {701..800}
|
||||
do
|
||||
echo "
|
||||
WITH ${i} AS try
|
||||
SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
|
||||
WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
|
||||
AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
|
||||
AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
|
||||
AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
|
||||
AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
|
||||
AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
|
||||
HAVING count() > 0;
|
||||
"
|
||||
done | ${CLICKHOUSE_CLIENT} --multiquery
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
|
||||
9
tests/queries/0_stateless/02995_index_9.reference
Normal file
9
tests/queries/0_stateless/02995_index_9.reference
Normal file
@ -0,0 +1,9 @@
|
||||
839 9 29223 46530 9 1336 1465 9 52 52 1
|
||||
844 5 2377545 2377635 5 5129 6321 5 53 69 5
|
||||
846 50 2172273 2589295 50 1582 3053 48 64 68 5
|
||||
847 2577 56656 63658 1582 1444 838 474 26 63 33
|
||||
861 1333 5570 6909 839 457 489 37 33 70 34
|
||||
873 2360 1519811 50487 2248 1310 1784 316 60 68 9
|
||||
879 228 6704 6785 79 279 507 121 35 66 24
|
||||
889 5130 2070007 39692 5040 1151 6791 2606 44 66 23
|
||||
896 4 511246 859452 4 6554 6561 4 67 71 4
|
||||
44
tests/queries/0_stateless/02995_index_9.sh
Executable file
44
tests/queries/0_stateless/02995_index_9.sh
Executable file
@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiquery "
|
||||
|
||||
DROP TABLE IF EXISTS test;
|
||||
CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
INSERT INTO test
|
||||
SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
|
||||
FROM numbers(100000);
|
||||
|
||||
DETACH TABLE test;
|
||||
ATTACH TABLE test;
|
||||
"
|
||||
|
||||
for i in {801..900}
|
||||
do
|
||||
echo "
|
||||
WITH ${i} AS try
|
||||
SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
|
||||
WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
|
||||
AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
|
||||
AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
|
||||
AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
|
||||
AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
|
||||
AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
|
||||
HAVING count() > 0;
|
||||
"
|
||||
done | ${CLICKHOUSE_CLIENT} --multiquery
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
|
||||
@ -1,92 +0,0 @@
|
||||
MergeTree compact + horizontal merge
|
||||
test
|
||||
16667 Tuple(a Dynamic(max_types=3)):Date
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):String
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
100000 UInt64:None
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
66667 Tuple(a Dynamic(max_types=3)):String
|
||||
100000 UInt64:None
|
||||
16667 Tuple(a Dynamic(max_types=3)):DateTime
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
66667 Tuple(a Dynamic(max_types=3)):String
|
||||
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
|
||||
100000 UInt64:None
|
||||
133333 Tuple(a Dynamic(max_types=3)):None
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
|
||||
100000 UInt64:None
|
||||
116667 Tuple(a Dynamic(max_types=3)):String
|
||||
133333 Tuple(a Dynamic(max_types=3)):None
|
||||
MergeTree wide + horizontal merge
|
||||
test
|
||||
16667 Tuple(a Dynamic(max_types=3)):Date
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):String
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
100000 UInt64:None
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
66667 Tuple(a Dynamic(max_types=3)):String
|
||||
100000 UInt64:None
|
||||
16667 Tuple(a Dynamic(max_types=3)):DateTime
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
66667 Tuple(a Dynamic(max_types=3)):String
|
||||
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
|
||||
100000 UInt64:None
|
||||
133333 Tuple(a Dynamic(max_types=3)):None
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
|
||||
100000 UInt64:None
|
||||
116667 Tuple(a Dynamic(max_types=3)):String
|
||||
133333 Tuple(a Dynamic(max_types=3)):None
|
||||
MergeTree compact + vertical merge
|
||||
test
|
||||
16667 Tuple(a Dynamic(max_types=3)):Date
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):String
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
100000 UInt64:None
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
66667 Tuple(a Dynamic(max_types=3)):String
|
||||
100000 UInt64:None
|
||||
16667 Tuple(a Dynamic(max_types=3)):DateTime
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
66667 Tuple(a Dynamic(max_types=3)):String
|
||||
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
|
||||
100000 UInt64:None
|
||||
133333 Tuple(a Dynamic(max_types=3)):None
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
|
||||
100000 UInt64:None
|
||||
116667 Tuple(a Dynamic(max_types=3)):String
|
||||
133333 Tuple(a Dynamic(max_types=3)):None
|
||||
MergeTree wide + vertical merge
|
||||
test
|
||||
16667 Tuple(a Dynamic(max_types=3)):Date
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):String
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
100000 UInt64:None
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
66667 Tuple(a Dynamic(max_types=3)):String
|
||||
100000 UInt64:None
|
||||
16667 Tuple(a Dynamic(max_types=3)):DateTime
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
66667 Tuple(a Dynamic(max_types=3)):String
|
||||
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
|
||||
100000 UInt64:None
|
||||
133333 Tuple(a Dynamic(max_types=3)):None
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
|
||||
100000 UInt64:None
|
||||
116667 Tuple(a Dynamic(max_types=3)):String
|
||||
133333 Tuple(a Dynamic(max_types=3)):None
|
||||
@ -1,53 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: long
|
||||
|
||||
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# reset --log_comment
|
||||
CLICKHOUSE_LOG_COMMENT=
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 --enable_named_columns_in_function_tuple=0"
|
||||
|
||||
|
||||
function test()
|
||||
{
|
||||
echo "test"
|
||||
$CH_CLIENT -q "system stop merges test"
|
||||
$CH_CLIENT -q "insert into test select number, number from numbers(100000)"
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000)"
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
|
||||
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000)"
|
||||
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
}
|
||||
|
||||
$CH_CLIENT -q "drop table if exists test;"
|
||||
|
||||
echo "MergeTree compact + horizontal merge"
|
||||
$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000;"
|
||||
test
|
||||
$CH_CLIENT -q "drop table test;"
|
||||
|
||||
echo "MergeTree wide + horizontal merge"
|
||||
$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;"
|
||||
test
|
||||
$CH_CLIENT -q "drop table test;"
|
||||
|
||||
echo "MergeTree compact + vertical merge"
|
||||
$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;"
|
||||
test
|
||||
$CH_CLIENT -q "drop table test;"
|
||||
|
||||
echo "MergeTree wide + vertical merge"
|
||||
$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;"
|
||||
test
|
||||
$CH_CLIENT -q "drop table test;"
|
||||
@ -0,0 +1,21 @@
|
||||
16667 Tuple(a Dynamic(max_types=3)):Date
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):String
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
100000 UInt64:None
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
66667 Tuple(a Dynamic(max_types=3)):String
|
||||
100000 UInt64:None
|
||||
16667 Tuple(a Dynamic(max_types=3)):DateTime
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
66667 Tuple(a Dynamic(max_types=3)):String
|
||||
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
|
||||
100000 UInt64:None
|
||||
133333 Tuple(a Dynamic(max_types=3)):None
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
|
||||
100000 UInt64:None
|
||||
116667 Tuple(a Dynamic(max_types=3)):String
|
||||
133333 Tuple(a Dynamic(max_types=3)):None
|
||||
32
tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sh
Executable file
32
tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sh
Executable file
@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: long
|
||||
|
||||
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# reset --log_comment
|
||||
CLICKHOUSE_LOG_COMMENT=
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 --enable_named_columns_in_function_tuple=0"
|
||||
|
||||
$CH_CLIENT -q "drop table if exists test;"
|
||||
$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000;"
|
||||
|
||||
$CH_CLIENT -q "system stop merges test"
|
||||
$CH_CLIENT -q "insert into test select number, number from numbers(100000)"
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000)"
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
|
||||
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000)"
|
||||
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
|
||||
$CH_CLIENT -q "drop table test;"
|
||||
|
||||
@ -0,0 +1,21 @@
|
||||
16667 Tuple(a Dynamic(max_types=3)):Date
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):String
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
100000 UInt64:None
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
66667 Tuple(a Dynamic(max_types=3)):String
|
||||
100000 UInt64:None
|
||||
16667 Tuple(a Dynamic(max_types=3)):DateTime
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
66667 Tuple(a Dynamic(max_types=3)):String
|
||||
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
|
||||
100000 UInt64:None
|
||||
133333 Tuple(a Dynamic(max_types=3)):None
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
|
||||
100000 UInt64:None
|
||||
116667 Tuple(a Dynamic(max_types=3)):String
|
||||
133333 Tuple(a Dynamic(max_types=3)):None
|
||||
32
tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sh
Executable file
32
tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sh
Executable file
@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: long
|
||||
|
||||
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# reset --log_comment
|
||||
CLICKHOUSE_LOG_COMMENT=
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 --enable_named_columns_in_function_tuple=0"
|
||||
|
||||
$CH_CLIENT -q "drop table if exists test;"
|
||||
$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;"
|
||||
|
||||
$CH_CLIENT -q "system stop merges test"
|
||||
$CH_CLIENT -q "insert into test select number, number from numbers(100000)"
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000)"
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
|
||||
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000)"
|
||||
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
|
||||
$CH_CLIENT -q "drop table test;"
|
||||
|
||||
@ -0,0 +1,21 @@
|
||||
16667 Tuple(a Dynamic(max_types=3)):Date
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):String
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
100000 UInt64:None
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
66667 Tuple(a Dynamic(max_types=3)):String
|
||||
100000 UInt64:None
|
||||
16667 Tuple(a Dynamic(max_types=3)):DateTime
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
66667 Tuple(a Dynamic(max_types=3)):String
|
||||
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
|
||||
100000 UInt64:None
|
||||
133333 Tuple(a Dynamic(max_types=3)):None
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
|
||||
100000 UInt64:None
|
||||
116667 Tuple(a Dynamic(max_types=3)):String
|
||||
133333 Tuple(a Dynamic(max_types=3)):None
|
||||
32
tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sh
Executable file
32
tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sh
Executable file
@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: long
|
||||
|
||||
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# reset --log_comment
|
||||
CLICKHOUSE_LOG_COMMENT=
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 --enable_named_columns_in_function_tuple=0"
|
||||
|
||||
$CH_CLIENT -q "drop table if exists test;"
|
||||
$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;"
|
||||
|
||||
$CH_CLIENT -q "system stop merges test"
|
||||
$CH_CLIENT -q "insert into test select number, number from numbers(100000)"
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000)"
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
|
||||
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000)"
|
||||
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
|
||||
$CH_CLIENT -q "drop table test;"
|
||||
|
||||
@ -0,0 +1,21 @@
|
||||
16667 Tuple(a Dynamic(max_types=3)):Date
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):String
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
100000 UInt64:None
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
66667 Tuple(a Dynamic(max_types=3)):String
|
||||
100000 UInt64:None
|
||||
16667 Tuple(a Dynamic(max_types=3)):DateTime
|
||||
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8)
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
66667 Tuple(a Dynamic(max_types=3)):String
|
||||
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
|
||||
100000 UInt64:None
|
||||
133333 Tuple(a Dynamic(max_types=3)):None
|
||||
50000 Tuple(a Dynamic(max_types=3)):UInt64
|
||||
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
|
||||
100000 UInt64:None
|
||||
116667 Tuple(a Dynamic(max_types=3)):String
|
||||
133333 Tuple(a Dynamic(max_types=3)):None
|
||||
32
tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sh
Executable file
32
tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sh
Executable file
@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: long
|
||||
|
||||
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# reset --log_comment
|
||||
CLICKHOUSE_LOG_COMMENT=
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 --enable_named_columns_in_function_tuple=0"
|
||||
|
||||
$CH_CLIENT -q "drop table if exists test;"
|
||||
$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;"
|
||||
|
||||
$CH_CLIENT -q "system stop merges test"
|
||||
$CH_CLIENT -q "insert into test select number, number from numbers(100000)"
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000)"
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
|
||||
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
|
||||
$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000)"
|
||||
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
|
||||
$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
|
||||
|
||||
$CH_CLIENT -q "drop table test;"
|
||||
|
||||
@ -0,0 +1,2 @@
|
||||
1
|
||||
1
|
||||
@ -0,0 +1,4 @@
|
||||
CREATE TABLE t (p UInt8, x UInt64) Engine = MergeTree PARTITION BY p ORDER BY x;
|
||||
INSERT INTO t SELECT 0, number FROM numbers(10) SETTINGS max_block_size = 100;
|
||||
SELECT count() FROM t WHERE p = 0 AND rowNumberInAllBlocks() = 1 SETTINGS allow_experimental_analyzer = 0;
|
||||
SELECT count() FROM t WHERE p = 0 AND rowNumberInAllBlocks() = 1 SETTINGS allow_experimental_analyzer = 1;
|
||||
@ -1,6 +1,6 @@
|
||||
DROP TABLE IF EXISTS hits_none;
|
||||
CREATE TABLE hits_none (Title String CODEC(NONE)) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
|
||||
INSERT INTO hits_none SELECT Title FROM test.hits;
|
||||
INSERT INTO hits_none SELECT Title FROM test.hits SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16;
|
||||
|
||||
SET min_bytes_to_use_mmap_io = 1;
|
||||
SELECT sum(length(Title)) FROM hits_none;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user