From a8f4f3d6b039cc82c30000f52989031784395f13 Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Tue, 9 Feb 2021 21:40:10 +0300 Subject: [PATCH 001/171] init commit with parsing and BAD realisation --- .../data/system/asynchronous_metric_log | 1 + programs/server/data/system/crash_log | 1 + programs/server/data/system/metric_log | 1 + programs/server/data/system/part_log | 1 + programs/server/data/system/query_log | 1 + programs/server/data/system/query_thread_log | 1 + programs/server/data/system/text_log | 1 + programs/server/data/system/trace_log | 1 + programs/server/data/test/hits | 1 + programs/server/data/test/visits | 1 + programs/server/metadata/system | 1 + programs/server/metadata/test | 1 + .../201403_1_6_1/checksums.txt | Bin 0 -> 14305 bytes .../201403_1_6_1/columns.txt | 183 ++++++++++++ .../201403_1_6_1/count.txt | 1 + .../default_compression_codec.txt | 1 + .../201403_1_6_1/minmax_StartDate.idx | 1 + .../201403_1_6_1/partition.dat | Bin 0 -> 4 bytes .../201403_1_6_1/primary.idx | Bin 0 -> 3240 bytes .../201403_7_9_1/checksums.txt | Bin 0 -> 14273 bytes .../201403_7_9_1/columns.txt | 183 ++++++++++++ .../201403_7_9_1/count.txt | 1 + .../default_compression_codec.txt | 1 + .../201403_7_9_1/minmax_StartDate.idx | 1 + .../201403_7_9_1/partition.dat | Bin 0 -> 4 bytes .../201403_7_9_1/primary.idx | Bin 0 -> 1314 bytes .../format_version.txt | 1 + .../202010_1_28_6/checksums.txt | Bin 0 -> 262 bytes .../202010_1_28_6/columns.txt | 259 +++++++++++++++++ .../202010_1_28_6/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202010_1_28_6/minmax_event_date.idx | 1 + .../202010_1_28_6/partition.dat | Bin 0 -> 4 bytes .../202010_1_28_6/primary.idx | 1 + .../202012_29_2358_466/checksums.txt | Bin 0 -> 11371 bytes .../202012_29_2358_466/columns.txt | 259 +++++++++++++++++ .../202012_29_2358_466/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202012_29_2358_466/minmax_event_date.idx | 1 + .../202012_29_2358_466/partition.dat | Bin 0 -> 4 bytes .../202012_29_2358_466/primary.idx | 1 + .../202101_2359_3529_235/checksums.txt | Bin 0 -> 11145 bytes .../202101_2359_3529_235/columns.txt | 259 +++++++++++++++++ .../202101_2359_3529_235/count.txt | 1 + .../default_compression_codec.txt | 1 + .../minmax_event_date.idx | 1 + .../202101_2359_3529_235/partition.dat | Bin 0 -> 4 bytes .../202101_2359_3529_235/primary.idx | Bin 0 -> 18 bytes .../202102_3530_4606_216/checksums.txt | Bin 0 -> 10969 bytes .../202102_3530_4606_216/columns.txt | 259 +++++++++++++++++ .../202102_3530_4606_216/count.txt | 1 + .../default_compression_codec.txt | 1 + .../minmax_event_date.idx | 1 + .../202102_3530_4606_216/partition.dat | Bin 0 -> 4 bytes .../202102_3530_4606_216/primary.idx | 1 + .../202102_4607_4607_0/checksums.txt | Bin 0 -> 260 bytes .../202102_4607_4607_0/columns.txt | 259 +++++++++++++++++ .../202102_4607_4607_0/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202102_4607_4607_0/minmax_event_date.idx | 1 + .../202102_4607_4607_0/partition.dat | Bin 0 -> 4 bytes .../202102_4607_4607_0/primary.idx | 1 + .../202102_4608_4608_0/checksums.txt | Bin 0 -> 260 bytes .../202102_4608_4608_0/columns.txt | 259 +++++++++++++++++ .../202102_4608_4608_0/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202102_4608_4608_0/minmax_event_date.idx | 1 + .../202102_4608_4608_0/partition.dat | Bin 0 -> 4 bytes .../202102_4608_4608_0/primary.idx | 1 + .../format_version.txt | 1 + .../hits.sql | 141 ++++++++++ .../visits.sql | 189 +++++++++++++ .../202010_1_5_1/checksums.txt | Bin 0 -> 260 bytes .../202010_1_5_1/columns.txt | 47 ++++ .../202010_1_5_1/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202010_1_5_1/minmax_event_date.idx | 1 + .../202010_1_5_1/partition.dat | Bin 0 -> 4 bytes .../202010_1_5_1/primary.idx | 1 + .../202012_6_47_9/checksums.txt | Bin 0 -> 262 bytes .../202012_6_47_9/columns.txt | 47 ++++ .../202012_6_47_9/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202012_6_47_9/minmax_event_date.idx | 1 + .../202012_6_47_9/partition.dat | Bin 0 -> 4 bytes .../202012_6_47_9/primary.idx | 1 + .../202101_48_74_6/checksums.txt | Bin 0 -> 261 bytes .../202101_48_74_6/columns.txt | 47 ++++ .../202101_48_74_6/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202101_48_74_6/minmax_event_date.idx | 1 + .../202101_48_74_6/partition.dat | Bin 0 -> 4 bytes .../202101_48_74_6/primary.idx | 1 + .../202102_75_79_1/checksums.txt | Bin 0 -> 260 bytes .../202102_75_79_1/columns.txt | 47 ++++ .../202102_75_79_1/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202102_75_79_1/minmax_event_date.idx | 1 + .../202102_75_79_1/partition.dat | Bin 0 -> 4 bytes .../202102_75_79_1/primary.idx | 1 + .../202102_80_80_0/checksums.txt | Bin 0 -> 260 bytes .../202102_80_80_0/columns.txt | 47 ++++ .../202102_80_80_0/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202102_80_80_0/minmax_event_date.idx | 1 + .../202102_80_80_0/partition.dat | Bin 0 -> 4 bytes .../202102_80_80_0/primary.idx | 1 + .../format_version.txt | 1 + .../all_1_1_0/checksums.txt | Bin 0 -> 188 bytes .../all_1_1_0/columns.txt | 13 + .../all_1_1_0/count.txt | 1 + .../all_1_1_0/default_compression_codec.txt | 1 + .../all_1_1_0/primary.idx | 1 + .../all_2_2_0/checksums.txt | Bin 0 -> 188 bytes .../all_2_2_0/columns.txt | 13 + .../all_2_2_0/count.txt | 1 + .../all_2_2_0/default_compression_codec.txt | 1 + .../all_2_2_0/primary.idx | 1 + .../format_version.txt | 1 + .../202012_1_18_4/checksums.txt | Bin 0 -> 260 bytes .../202012_1_18_4/columns.txt | 20 ++ .../202012_1_18_4/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202012_1_18_4/minmax_event_date.idx | 1 + .../202012_1_18_4/partition.dat | Bin 0 -> 4 bytes .../202012_1_18_4/primary.idx | 1 + .../202101_19_22_1/checksums.txt | Bin 0 -> 260 bytes .../202101_19_22_1/columns.txt | 20 ++ .../202101_19_22_1/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202101_19_22_1/minmax_event_date.idx | 1 + .../202101_19_22_1/partition.dat | Bin 0 -> 4 bytes .../202101_19_22_1/primary.idx | 1 + .../format_version.txt | 1 + .../201403_1_29_2/checksums.txt | Bin 0 -> 10792 bytes .../201403_1_29_2/columns.txt | 135 +++++++++ .../201403_1_29_2/count.txt | 1 + .../default_compression_codec.txt | 1 + .../201403_1_29_2/minmax_EventDate.idx | 1 + .../201403_1_29_2/partition.dat | Bin 0 -> 4 bytes .../201403_1_29_2/primary.idx | Bin 0 -> 15330 bytes .../format_version.txt | 1 + .../202010_1_4_1/checksums.txt | Bin 0 -> 260 bytes .../202010_1_4_1/columns.txt | 12 + .../202010_1_4_1/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202010_1_4_1/minmax_event_date.idx | 1 + .../202010_1_4_1/partition.dat | Bin 0 -> 4 bytes .../202010_1_4_1/primary.idx | 1 + .../202012_35_35_0/checksums.txt | Bin 0 -> 260 bytes .../202012_35_35_0/columns.txt | 12 + .../202012_35_35_0/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202012_35_35_0/minmax_event_date.idx | 1 + .../202012_35_35_0/partition.dat | Bin 0 -> 4 bytes .../202012_35_35_0/primary.idx | 1 + .../202012_5_34_6/checksums.txt | Bin 0 -> 261 bytes .../202012_5_34_6/columns.txt | 12 + .../202012_5_34_6/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202012_5_34_6/minmax_event_date.idx | 1 + .../202012_5_34_6/partition.dat | Bin 0 -> 4 bytes .../202012_5_34_6/primary.idx | 1 + .../202101_36_115_17/checksums.txt | Bin 0 -> 262 bytes .../202101_36_115_17/columns.txt | 12 + .../202101_36_115_17/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202101_36_115_17/minmax_event_date.idx | 1 + .../202101_36_115_17/partition.dat | Bin 0 -> 4 bytes .../202101_36_115_17/primary.idx | 1 + .../202102_116_127_3/checksums.txt | Bin 0 -> 261 bytes .../202102_116_127_3/columns.txt | 12 + .../202102_116_127_3/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202102_116_127_3/minmax_event_date.idx | 1 + .../202102_116_127_3/partition.dat | Bin 0 -> 4 bytes .../202102_116_127_3/primary.idx | 1 + .../202102_128_128_0/checksums.txt | Bin 0 -> 260 bytes .../202102_128_128_0/columns.txt | 12 + .../202102_128_128_0/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202102_128_128_0/minmax_event_date.idx | 1 + .../202102_128_128_0/partition.dat | Bin 0 -> 4 bytes .../202102_128_128_0/primary.idx | 1 + .../202102_129_129_0/checksums.txt | Bin 0 -> 261 bytes .../202102_129_129_0/columns.txt | 12 + .../202102_129_129_0/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202102_129_129_0/minmax_event_date.idx | 1 + .../202102_129_129_0/partition.dat | Bin 0 -> 4 bytes .../202102_129_129_0/primary.idx | 1 + .../format_version.txt | 1 + .../202010_1_5_1/checksums.txt | Bin 0 -> 260 bytes .../202010_1_5_1/columns.txt | 41 +++ .../202010_1_5_1/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202010_1_5_1/minmax_event_date.idx | 1 + .../202010_1_5_1/partition.dat | Bin 0 -> 4 bytes .../202010_1_5_1/primary.idx | 1 + .../202012_6_45_8/checksums.txt | Bin 0 -> 262 bytes .../202012_6_45_8/columns.txt | 41 +++ .../202012_6_45_8/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202012_6_45_8/minmax_event_date.idx | 1 + .../202012_6_45_8/partition.dat | Bin 0 -> 4 bytes .../202012_6_45_8/primary.idx | 1 + .../202101_46_72_6/checksums.txt | Bin 0 -> 262 bytes .../202101_46_72_6/columns.txt | 41 +++ .../202101_46_72_6/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202101_46_72_6/minmax_event_date.idx | 1 + .../202101_46_72_6/partition.dat | Bin 0 -> 4 bytes .../202101_46_72_6/primary.idx | 1 + .../202102_73_77_1/checksums.txt | Bin 0 -> 261 bytes .../202102_73_77_1/columns.txt | 41 +++ .../202102_73_77_1/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202102_73_77_1/minmax_event_date.idx | 1 + .../202102_73_77_1/partition.dat | Bin 0 -> 4 bytes .../202102_73_77_1/primary.idx | 1 + .../202102_78_78_0/checksums.txt | Bin 0 -> 260 bytes .../202102_78_78_0/columns.txt | 41 +++ .../202102_78_78_0/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202102_78_78_0/minmax_event_date.idx | 1 + .../202102_78_78_0/partition.dat | Bin 0 -> 4 bytes .../202102_78_78_0/primary.idx | 1 + .../format_version.txt | 1 + .../202010_1_29_6/checksums.txt | Bin 0 -> 262 bytes .../202010_1_29_6/columns.txt | 15 + .../202010_1_29_6/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202010_1_29_6/minmax_event_date.idx | 1 + .../202010_1_29_6/partition.dat | Bin 0 -> 4 bytes .../202010_1_29_6/primary.idx | 1 + .../202012_2358_2361_1/checksums.txt | Bin 0 -> 260 bytes .../202012_2358_2361_1/columns.txt | 15 + .../202012_2358_2361_1/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202012_2358_2361_1/minmax_event_date.idx | 1 + .../202012_2358_2361_1/partition.dat | Bin 0 -> 4 bytes .../202012_2358_2361_1/primary.idx | 1 + .../202012_30_2357_1895/checksums.txt | Bin 0 -> 263 bytes .../202012_30_2357_1895/columns.txt | 15 + .../202012_30_2357_1895/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202012_30_2357_1895/minmax_event_date.idx | 1 + .../202012_30_2357_1895/partition.dat | Bin 0 -> 4 bytes .../202012_30_2357_1895/primary.idx | 1 + .../202101_2362_3536_470/checksums.txt | Bin 0 -> 263 bytes .../202101_2362_3536_470/columns.txt | 15 + .../202101_2362_3536_470/count.txt | 1 + .../default_compression_codec.txt | 1 + .../minmax_event_date.idx | 1 + .../202101_2362_3536_470/partition.dat | Bin 0 -> 4 bytes .../202101_2362_3536_470/primary.idx | Bin 0 -> 24 bytes .../202101_3537_3539_1/checksums.txt | Bin 0 -> 260 bytes .../202101_3537_3539_1/columns.txt | 15 + .../202101_3537_3539_1/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202101_3537_3539_1/minmax_event_date.idx | 1 + .../202101_3537_3539_1/partition.dat | Bin 0 -> 4 bytes .../202101_3537_3539_1/primary.idx | 1 + .../202102_3540_4637_415/checksums.txt | Bin 0 -> 263 bytes .../202102_3540_4637_415/columns.txt | 15 + .../202102_3540_4637_415/count.txt | 1 + .../default_compression_codec.txt | 1 + .../minmax_event_date.idx | 1 + .../202102_3540_4637_415/partition.dat | Bin 0 -> 4 bytes .../202102_3540_4637_415/primary.idx | 1 + .../202102_4638_4638_0/checksums.txt | Bin 0 -> 260 bytes .../202102_4638_4638_0/columns.txt | 15 + .../202102_4638_4638_0/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202102_4638_4638_0/minmax_event_date.idx | 1 + .../202102_4638_4638_0/partition.dat | Bin 0 -> 4 bytes .../202102_4638_4638_0/primary.idx | 1 + .../format_version.txt | 1 + .../asynchronous_metric_log.sql | 12 + .../crash_log.sql | 17 ++ .../metric_log.sql | 264 ++++++++++++++++++ .../part_log.sql | 25 ++ .../query_log.sql | 52 ++++ .../query_thread_log.sql | 46 +++ .../text_log.sql | 20 ++ .../trace_log.sql | 17 ++ .../202010_1_3_1/checksums.txt | Bin 0 -> 260 bytes .../202010_1_3_1/columns.txt | 7 + .../202010_1_3_1/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202010_1_3_1/minmax_event_date.idx | 1 + .../202010_1_3_1/partition.dat | Bin 0 -> 4 bytes .../202010_1_3_1/primary.idx | 1 + .../202012_4_295_59/checksums.txt | Bin 0 -> 262 bytes .../202012_4_295_59/columns.txt | 7 + .../202012_4_295_59/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202012_4_295_59/minmax_event_date.idx | 1 + .../202012_4_295_59/partition.dat | Bin 0 -> 4 bytes .../202012_4_295_59/primary.idx | 1 + .../202101_296_439_30/checksums.txt | Bin 0 -> 262 bytes .../202101_296_439_30/columns.txt | 7 + .../202101_296_439_30/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202101_296_439_30/minmax_event_date.idx | 1 + .../202101_296_439_30/partition.dat | Bin 0 -> 4 bytes .../202101_296_439_30/primary.idx | 1 + .../202102_440_574_27/checksums.txt | Bin 0 -> 262 bytes .../202102_440_574_27/columns.txt | 7 + .../202102_440_574_27/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202102_440_574_27/minmax_event_date.idx | 1 + .../202102_440_574_27/partition.dat | Bin 0 -> 4 bytes .../202102_440_574_27/primary.idx | 1 + .../202102_575_575_0/checksums.txt | Bin 0 -> 260 bytes .../202102_575_575_0/columns.txt | 7 + .../202102_575_575_0/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202102_575_575_0/minmax_event_date.idx | 1 + .../202102_575_575_0/partition.dat | Bin 0 -> 4 bytes .../202102_575_575_0/primary.idx | 1 + .../202102_576_576_0/checksums.txt | Bin 0 -> 260 bytes .../202102_576_576_0/columns.txt | 7 + .../202102_576_576_0/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202102_576_576_0/minmax_event_date.idx | 1 + .../202102_576_576_0/partition.dat | Bin 0 -> 4 bytes .../202102_576_576_0/primary.idx | 1 + .../202102_577_577_0/checksums.txt | Bin 0 -> 260 bytes .../202102_577_577_0/columns.txt | 7 + .../202102_577_577_0/count.txt | 1 + .../default_compression_codec.txt | 1 + .../202102_577_577_0/minmax_event_date.idx | 1 + .../202102_577_577_0/partition.dat | Bin 0 -> 4 bytes .../202102_577_577_0/primary.idx | 1 + .../format_version.txt | 1 + src/Interpreters/InterpreterSelectQuery.cpp | 49 +++- src/Interpreters/InterpreterSelectQuery.h | 1 + src/Parsers/ASTSelectQuery.cpp | 3 + src/Parsers/ASTSelectQuery.h | 1 + src/Parsers/ParserSelectQuery.cpp | 11 +- src/Processors/QueryPlan/GroupingSetsStep.cpp | 46 +++ src/Processors/QueryPlan/GroupingSetsStep.h | 25 ++ .../Transforms/AggregatingTransform.h | 1 + src/Processors/Transforms/CubeTransform.cpp | 9 +- .../Transforms/GroupingSetsTransform.cpp | 81 ++++++ .../Transforms/GroupingSetsTransform.h | 35 +++ src/Processors/ya.make | 160 +++++++++++ 348 files changed, 4180 insertions(+), 15 deletions(-) create mode 120000 programs/server/data/system/asynchronous_metric_log create mode 120000 programs/server/data/system/crash_log create mode 120000 programs/server/data/system/metric_log create mode 120000 programs/server/data/system/part_log create mode 120000 programs/server/data/system/query_log create mode 120000 programs/server/data/system/query_thread_log create mode 120000 programs/server/data/system/text_log create mode 120000 programs/server/data/system/trace_log create mode 120000 programs/server/data/test/hits create mode 120000 programs/server/data/test/visits create mode 120000 programs/server/metadata/system create mode 120000 programs/server/metadata/test create mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/checksums.txt create mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/columns.txt create mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/count.txt create mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/default_compression_codec.txt create mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/minmax_StartDate.idx create mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/partition.dat create mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/primary.idx create mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/checksums.txt create mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/columns.txt create mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/count.txt create mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/default_compression_codec.txt create mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/minmax_StartDate.idx create mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/partition.dat create mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/primary.idx create mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/format_version.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/checksums.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/columns.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/count.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/default_compression_codec.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/minmax_event_date.idx create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/partition.dat create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/primary.idx create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202012_29_2358_466/checksums.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202012_29_2358_466/columns.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202012_29_2358_466/count.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202012_29_2358_466/default_compression_codec.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202012_29_2358_466/minmax_event_date.idx create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202012_29_2358_466/partition.dat create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202012_29_2358_466/primary.idx create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202101_2359_3529_235/checksums.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202101_2359_3529_235/columns.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202101_2359_3529_235/count.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202101_2359_3529_235/default_compression_codec.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202101_2359_3529_235/minmax_event_date.idx create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202101_2359_3529_235/partition.dat create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202101_2359_3529_235/primary.idx create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_3530_4606_216/checksums.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_3530_4606_216/columns.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_3530_4606_216/count.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_3530_4606_216/default_compression_codec.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_3530_4606_216/minmax_event_date.idx create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_3530_4606_216/partition.dat create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_3530_4606_216/primary.idx create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/checksums.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/columns.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/count.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/default_compression_codec.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/minmax_event_date.idx create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/partition.dat create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/primary.idx create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/checksums.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/columns.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/count.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/default_compression_codec.txt create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/minmax_event_date.idx create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/partition.dat create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/primary.idx create mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/format_version.txt create mode 100644 programs/server/store/2c6/2c6da17b-121a-4775-b287-b0d80cd04080/hits.sql create mode 100644 programs/server/store/2c6/2c6da17b-121a-4775-b287-b0d80cd04080/visits.sql create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/checksums.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/columns.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/count.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/default_compression_codec.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/minmax_event_date.idx create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/partition.dat create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/primary.idx create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/checksums.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/columns.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/count.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/default_compression_codec.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/minmax_event_date.idx create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/partition.dat create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/primary.idx create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202101_48_74_6/checksums.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202101_48_74_6/columns.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202101_48_74_6/count.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202101_48_74_6/default_compression_codec.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202101_48_74_6/minmax_event_date.idx create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202101_48_74_6/partition.dat create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202101_48_74_6/primary.idx create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/checksums.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/columns.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/count.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/default_compression_codec.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/minmax_event_date.idx create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/partition.dat create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/primary.idx create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_80_80_0/checksums.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_80_80_0/columns.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_80_80_0/count.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_80_80_0/default_compression_codec.txt create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_80_80_0/minmax_event_date.idx create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_80_80_0/partition.dat create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_80_80_0/primary.idx create mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/format_version.txt create mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/checksums.txt create mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/columns.txt create mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/count.txt create mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/default_compression_codec.txt create mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/primary.idx create mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/checksums.txt create mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/columns.txt create mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/count.txt create mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/default_compression_codec.txt create mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/primary.idx create mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/format_version.txt create mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/checksums.txt create mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/columns.txt create mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/count.txt create mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/default_compression_codec.txt create mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/minmax_event_date.idx create mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/partition.dat create mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/primary.idx create mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/checksums.txt create mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/columns.txt create mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/count.txt create mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/default_compression_codec.txt create mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/minmax_event_date.idx create mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/partition.dat create mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/primary.idx create mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/format_version.txt create mode 100644 programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/checksums.txt create mode 100644 programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/columns.txt create mode 100644 programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/count.txt create mode 100644 programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/default_compression_codec.txt create mode 100644 programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/minmax_EventDate.idx create mode 100644 programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/partition.dat create mode 100644 programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/primary.idx create mode 100644 programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/format_version.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/checksums.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/columns.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/count.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/default_compression_codec.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/minmax_event_date.idx create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/partition.dat create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/primary.idx create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/checksums.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/columns.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/count.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/default_compression_codec.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/minmax_event_date.idx create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/partition.dat create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/primary.idx create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/checksums.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/columns.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/count.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/default_compression_codec.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/minmax_event_date.idx create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/partition.dat create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/primary.idx create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/checksums.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/columns.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/count.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/default_compression_codec.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/minmax_event_date.idx create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/partition.dat create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/primary.idx create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/checksums.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/columns.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/count.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/default_compression_codec.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/minmax_event_date.idx create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/partition.dat create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/primary.idx create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/checksums.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/columns.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/count.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/default_compression_codec.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/minmax_event_date.idx create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/partition.dat create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/primary.idx create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_129_129_0/checksums.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_129_129_0/columns.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_129_129_0/count.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_129_129_0/default_compression_codec.txt create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_129_129_0/minmax_event_date.idx create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_129_129_0/partition.dat create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_129_129_0/primary.idx create mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/format_version.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/checksums.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/columns.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/count.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/default_compression_codec.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/minmax_event_date.idx create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/partition.dat create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/primary.idx create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/checksums.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/columns.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/count.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/default_compression_codec.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/minmax_event_date.idx create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/partition.dat create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/primary.idx create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/checksums.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/columns.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/count.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/default_compression_codec.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/minmax_event_date.idx create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/partition.dat create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/primary.idx create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/checksums.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/columns.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/count.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/default_compression_codec.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/minmax_event_date.idx create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/partition.dat create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/primary.idx create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/checksums.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/columns.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/count.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/default_compression_codec.txt create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/minmax_event_date.idx create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/partition.dat create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/primary.idx create mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/format_version.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/checksums.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/columns.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/count.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/default_compression_codec.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/minmax_event_date.idx create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/partition.dat create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/primary.idx create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_2358_2361_1/checksums.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_2358_2361_1/columns.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_2358_2361_1/count.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_2358_2361_1/default_compression_codec.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_2358_2361_1/minmax_event_date.idx create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_2358_2361_1/partition.dat create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_2358_2361_1/primary.idx create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_30_2357_1895/checksums.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_30_2357_1895/columns.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_30_2357_1895/count.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_30_2357_1895/default_compression_codec.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_30_2357_1895/minmax_event_date.idx create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_30_2357_1895/partition.dat create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_30_2357_1895/primary.idx create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_2362_3536_470/checksums.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_2362_3536_470/columns.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_2362_3536_470/count.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_2362_3536_470/default_compression_codec.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_2362_3536_470/minmax_event_date.idx create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_2362_3536_470/partition.dat create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_2362_3536_470/primary.idx create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/checksums.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/columns.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/count.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/default_compression_codec.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/minmax_event_date.idx create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/partition.dat create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/primary.idx create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_3540_4637_415/checksums.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_3540_4637_415/columns.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_3540_4637_415/count.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_3540_4637_415/default_compression_codec.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_3540_4637_415/minmax_event_date.idx create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_3540_4637_415/partition.dat create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_3540_4637_415/primary.idx create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/checksums.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/columns.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/count.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/default_compression_codec.txt create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/minmax_event_date.idx create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/partition.dat create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/primary.idx create mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/format_version.txt create mode 100644 programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/asynchronous_metric_log.sql create mode 100644 programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/crash_log.sql create mode 100644 programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/metric_log.sql create mode 100644 programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/part_log.sql create mode 100644 programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/query_log.sql create mode 100644 programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/query_thread_log.sql create mode 100644 programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/text_log.sql create mode 100644 programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/trace_log.sql create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/checksums.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/columns.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/count.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/default_compression_codec.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/minmax_event_date.idx create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/partition.dat create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/primary.idx create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/checksums.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/columns.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/count.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/default_compression_codec.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/minmax_event_date.idx create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/partition.dat create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/primary.idx create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/checksums.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/columns.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/count.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/default_compression_codec.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/minmax_event_date.idx create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/partition.dat create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/primary.idx create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_440_574_27/checksums.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_440_574_27/columns.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_440_574_27/count.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_440_574_27/default_compression_codec.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_440_574_27/minmax_event_date.idx create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_440_574_27/partition.dat create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_440_574_27/primary.idx create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/checksums.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/columns.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/count.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/default_compression_codec.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/minmax_event_date.idx create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/partition.dat create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/primary.idx create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/checksums.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/columns.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/count.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/default_compression_codec.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/minmax_event_date.idx create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/partition.dat create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/primary.idx create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/checksums.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/columns.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/count.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/default_compression_codec.txt create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/minmax_event_date.idx create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/partition.dat create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/primary.idx create mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/format_version.txt create mode 100644 src/Processors/QueryPlan/GroupingSetsStep.cpp create mode 100644 src/Processors/QueryPlan/GroupingSetsStep.h create mode 100644 src/Processors/Transforms/GroupingSetsTransform.cpp create mode 100644 src/Processors/Transforms/GroupingSetsTransform.h create mode 100644 src/Processors/ya.make diff --git a/programs/server/data/system/asynchronous_metric_log b/programs/server/data/system/asynchronous_metric_log new file mode 120000 index 00000000000..e065d7bf424 --- /dev/null +++ b/programs/server/data/system/asynchronous_metric_log @@ -0,0 +1 @@ +/home/max/github/ClickHouse/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1 \ No newline at end of file diff --git a/programs/server/data/system/crash_log b/programs/server/data/system/crash_log new file mode 120000 index 00000000000..7c05d325fb8 --- /dev/null +++ b/programs/server/data/system/crash_log @@ -0,0 +1 @@ +/home/max/github/ClickHouse/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665 \ No newline at end of file diff --git a/programs/server/data/system/metric_log b/programs/server/data/system/metric_log new file mode 120000 index 00000000000..cf9f177d49f --- /dev/null +++ b/programs/server/data/system/metric_log @@ -0,0 +1 @@ +/home/max/github/ClickHouse/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0 \ No newline at end of file diff --git a/programs/server/data/system/part_log b/programs/server/data/system/part_log new file mode 120000 index 00000000000..b19efce9efd --- /dev/null +++ b/programs/server/data/system/part_log @@ -0,0 +1 @@ +/home/max/github/ClickHouse/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d \ No newline at end of file diff --git a/programs/server/data/system/query_log b/programs/server/data/system/query_log new file mode 120000 index 00000000000..4222415ce96 --- /dev/null +++ b/programs/server/data/system/query_log @@ -0,0 +1 @@ +/home/max/github/ClickHouse/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df \ No newline at end of file diff --git a/programs/server/data/system/query_thread_log b/programs/server/data/system/query_thread_log new file mode 120000 index 00000000000..6108d6eab87 --- /dev/null +++ b/programs/server/data/system/query_thread_log @@ -0,0 +1 @@ +/home/max/github/ClickHouse/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc \ No newline at end of file diff --git a/programs/server/data/system/text_log b/programs/server/data/system/text_log new file mode 120000 index 00000000000..4ae519959fe --- /dev/null +++ b/programs/server/data/system/text_log @@ -0,0 +1 @@ +/home/max/github/ClickHouse/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500 \ No newline at end of file diff --git a/programs/server/data/system/trace_log b/programs/server/data/system/trace_log new file mode 120000 index 00000000000..6dfdc4802b3 --- /dev/null +++ b/programs/server/data/system/trace_log @@ -0,0 +1 @@ +/home/max/github/ClickHouse/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a \ No newline at end of file diff --git a/programs/server/data/test/hits b/programs/server/data/test/hits new file mode 120000 index 00000000000..28b16b0be67 --- /dev/null +++ b/programs/server/data/test/hits @@ -0,0 +1 @@ +/home/max/github/ClickHouse/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64 \ No newline at end of file diff --git a/programs/server/data/test/visits b/programs/server/data/test/visits new file mode 120000 index 00000000000..49e67e1c13b --- /dev/null +++ b/programs/server/data/test/visits @@ -0,0 +1 @@ +/home/max/github/ClickHouse/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f \ No newline at end of file diff --git a/programs/server/metadata/system b/programs/server/metadata/system new file mode 120000 index 00000000000..4ff5d3274d6 --- /dev/null +++ b/programs/server/metadata/system @@ -0,0 +1 @@ +/home/max/github/ClickHouse/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681 \ No newline at end of file diff --git a/programs/server/metadata/test b/programs/server/metadata/test new file mode 120000 index 00000000000..b4ed8633adf --- /dev/null +++ b/programs/server/metadata/test @@ -0,0 +1 @@ +/home/max/github/ClickHouse/programs/server/store/2c6/2c6da17b-121a-4775-b287-b0d80cd04080 \ No newline at end of file diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/checksums.txt b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..da44b12b708c635f7165a2c8b8437d5fd40cbb12 GIT binary patch literal 14305 zcmX|IcOaGj_kTaneJ-xOcVuLhl@W=oGD2h*nHM+X%9e~oMo5Svqe3!DMOId1WklJO zQpzaz+GPBm^7;P$z6V{1KAsMK2Ltdvz67Gz@q_Zz z?VqOPubs%3R#X^Fo*H1>j;uTez!g0JHV#K&7&M#%PILz-SGmo243KC&ecX>u2_S-^7rXgRoMniX%9$dE9AAjJQ^BMy58h(CwFJ}iYr*mMP=l9({B-tiDE1Y5JYY;_7 zk4cc?B$St(gyTDrKvAQ3?5TC9HG9&IxhK7vpiEqa0V& zehL99ErTYA%-t4EU-;_2!DnhiYwsSj%fq=EjZjhhBUnt3QiFrSas7IPPGO|8F%?c0 zz(#<~dA#s!_;tI3=I*%fo8r{Tt$E6aix6o zF9EFIU#-6SHR#^{{#Q4&2XnBHZ*YXNODHGgHxAc$BRm=afxW>JCVbrY;)5_(PTx?a zouvF09AK0N_UOHMdizD9){3NGm7V37OZa5mJS#VDpQ~-_0yg2=2d~R3wgJpEZF-7L_(4?;P_v#CGL8|8^GG> z*FCs!T>#pFySQfi;=5ZjSc7f!9cCgp4)oCI-S`xmKlqGZ5x`dHR;9%` zowhcRpTFFu--iX$)E$5vwTYe%1g}ukR(~_HPDAa$q4Axfwp6|I*%R}J=y5|a`ja2Z z`^S09;zc)|al-2dp#L4*m64nW)ZvDu0?I0NL0bYe%J9Kc0aqcY5g{Y%o!ASI!Df)k z?Cp=0k}8@XY&7jqV|{{=QjzgLIM$DJrqqzFrrBXc8Ne6OTkX3YP1w_Ooc1Z`@Vb2D zY9ojO&>E`*kAKoV+E{a%_Dra9w)gI|Bhb(x5}gkNpoZET>A<8uJyQLCERz+wCd5{5 zwA!z<}9rwX#<7DHU>jI8o-y9k9@tO^lQ;@B6@rhy8Q~fe(fISYF>-Z4; zy}MvCC7SUz9zy3PxW2&y6Ka2W{unW1EpNMkJvh%`9HY0>W7L18ui*ZV(|nzQsSuvOpv}&8D#x5CA=V)PU&f2*}(M$vBHQH<+*IyAF9EWg640)3>1aN$rSM z%1PNSZpT_jE#*gtwGQDK9081m#Cbr*oK-eixS4>ze@n2#v+#^b6W|g8Ukx36-0^-| z4t{u7qE9d$Fth_UxF4a@1ZYgsbpf~mp_T#+0L#PA`vL$6eeMJ3AS`?fMt@*3K7}xh zp>O2F`}a*Z0AL(~hyW-aFmz+tVNDo^`lkz^YXeM-Yry-gw;zfXzTNN=XU{WC@7=rN zk(aWJU%ybap}d`lsZ=7HygI- zwM$$01bm!#Rdz~lNl>J0>|O>KbXhj&vK}JGKmSg%l%q?2;bx3y@x@|+FhGK#CC-9} zn7PWkW1+EZ>`I?fC|G(vm_T3p&kiMih)y46_Feh1$CUJ``qu=6MVIA@TY+Szq`;Z$ zf<+5rsp2#f-^c8LmYa`*@81klnT1V0EiegdRZnYw)*{?<)(WUcKb%iL4r0^%LViyZ zuzDguMcE!8|7ENy@uWeK*ItS}YD(Jefm)A(AuVGM2PZq!)@&Q5=H3x5zWcqh#%s4# z9-mIC`Wtt3Yr!5&Gi2?w$cf1Kw>vw*(EsA@AcKi1L)DexlSjF}JrT}-3fhryV;=(1 zhu}Aa;e~l9DgU7i-7bt`jq^FYFVWxUEnu+6u%W-Be`9b4)aIsm54=+$B?-Cx2%i0| zX|^)d`0GNiTy>ZA1;IkfCFt~9vyJ6Q1sUtG#vr#4I~ylJiN>iD8VBiC`Qx1vadlDe z0sp^MxeeN9|r5&Dm zb#1aEKj_$(3hE`Ogk|IpgGJzUYNb)DQh7kZ`@j1_cef0g%zos!q@?;jjw!e*dMc>R z4+Aa&xaY51zQfet*0!In$awXr?a`RNo?fmMyae@TkDnY&a;Hv-*Qpav60PyW?7I>f zW0QRNbLmSpOwWfu*L?UUya4S2SmHN7XYiLVHC=-$ISxpQ_?i}O(|F0ZQ%6QYwLN;D zxELIFKhHYPQN(^btXpG<#Q}K$n*%bGf@C(i9J%*v^T@EO>0&?u8zhSYRtY7-L`BA4 z>wE+g=YDw8?T!*Ml#pzRTjO-! zhT6w>&gam5lL}I*1=usxWbCi}J?K&1J*BJKUAf0=f8g{RZ;4+-4bAC14asPGlcnq- zX!oP>;{0m>qX1)x8lnwNj9=p_d>G;b>)yYcEUOa|Niz|)bZ|H;xGyNIVzj_A(*zHgsJnX0KDFC z7enmv$-vgBH;lZ?R8t|Hfg{d!yg3;%3JRZwD9?=;b4> z^AlL14VX;2t>x{N5WRLZXvik+;}qk+m4|SC{lgd7O*6jEDw6#5^e>sqh zv39yH`7U|Td59{ju5EHw4$zs=3*vn~qj;4#$NrU&ZNX0XshpTF+yCKhkyI55^A~9O zo}LQ1ID74QxoqatQQm!+Cs7t2i+&mBNqzI8^#p~rjrSU$)%N#sz*7Qx^d7qJ--N;* z543KVy}FB(HeUL%8&P5fR$<-+qlKCKdxu@F4f^lMQmoMZyHXOxJ%?b6&p#&nH%25! zh~2jKfclh|^Dey1hx+ZO#9hWmGj`7lMN;p+HG5mb^7$1vM1IgE5@ySNjT0%0iYx)l zl~}36G^pR2qV)W|k)Fysoh`kR$N2nGy5ND$2mi5iy3D3#^Ybb#LU;B!5^J5d!%=e{ z4!&-N4wr~Nlqk=Tr(KZ zty+T>h~e0oxck;uT=ebwZ_S^QF}DVS1TWMRk@hW)S{CM4JsS5j1GWoV4lK%bTBGTq z-&w)%ahgG&sA9MiN1yJTHGuwEB4hf&*;4B=l3ebuV`73SZmodnD!_EYWSrdJa9__6 z#k)=Gu1cPtuIx{@3Uj>II*lW&W4$x1hR+q`uG0;&1iy zzAN<3|2DP^>T*x<7n2RW%3#ks%$rQm$Bi&Cg>`3yLnLx7Tn&813>PtSCD9a z&hsl=DPOFL3uRVUwwh)DtU7p<&O3PMg<@p!zJ3YRJ3LP&5-^Q?yZp3;aXMkEHZ|(W zWIZT3^A~ngErzRaojQEXK+TwARPG&{8Va~z2U8O5c4x|O;*LVDrLF2WH)_Bw4vc*X zSMV~O=yt0ghtzaknn~#r#wiE7dKA6f(BnfZ7~FG<5}mY6I{~lHx8u7CCU~^a3dRSJ z8Tq~)nBa`Aew5W+IhhGpdje)1qJzgdyn~Y)-dDq`l$neyK4F~V)_vRlhY;2qv^#j4 zbKLD5<`1l1{_-pNX^;NLdm`&tvPst>b6iDU=;-#+IVlj z1(b__$)fm_buk9cau_UsACphWUQKicJ1|CFqT{RN1^>aU8;z|j2D)r${zgkBDAQf1 z#Uf<~|CRjk%$+=z(;WcggDhc+mzX$C-o#aG-jtDHe`PLo;VU0@0}76A2{(4N_Me$L z+XPSd-Rl5gKSN?>-Bb?=HHyoP$7=-p{iYrO>^H$n-$O@FY5Dv3;JuvAQ}1!zxm?XF zt)L-x{kBT6&)VhPCQGPe{WYX%c|7jr#Ob>z(IT;LQ3R z|K9xVWt<9F0VvzWv7NT5VSai*xQ;JtV+=}O+=SO-ZEFIy zp31|`^GDB%nD;Hg0Oj)kqJ0Ts_%239*xTLTF9#2vscfh)hJgsMH^uu996Ta1WaNFL zu<{G{FO@qxdd^vrl9AGdmk&a#Q0m=b>dqg`OfRZWw9Uuas_X#Fzgt~`&0hbKp=V4? z&z+MLeljFu0(2mXFQ}nf5~ucwyfF2q0KXw8j}M+UVV}{*Fli%?aPq9|*zt`o>Ssv$ z_ZheTqr)qd!?>QoQ}tt-kMvn25~!mu09rJA=rpmBkdjh}Z6n%b)ggv2K)u$g5 zL^OLyxKU;uJ}Yqk=-k0CTF-wTru#=RNHXfoE;qf6pH@@CECTZ_^4BSQ_3)8yUv1&bxxTW{HI->1LHAsyn< z5SY}U0}Oh`MtHwKqK|tROvVmx)IpWiw*^hIwt*q9$RKl-%$x<-fMrSAUcq%XhS2tb z7K3wDsf;K^N9zE}=G~mGr67sWyNq4T$B|%nK&|KHM6_ceArtq7`@k1ewyLr8QhkO? zx>+l*i)u1+>=J!BTH$h;507TJa%F1J90jlm*u;crPiS4YIv>%Tqmf(wOYbY7+e9g2 zNv?ike%|3D?hG#J_j46g4Sph6$~D-x>|WLqUYTF^tsxSwa!~NUTAk{g%A?DVRqLg= zs8c58p5x;It)8zY(LailgnWE*KuP7o8H(S`9H~czsh+E(NtU1p9++|Qp^D?9DBI3a z&u`uSxA#w*MAwxtfV;Tu_8~A-`r6B@d+C5)&sUom=;c9laIS_)$cHVsfNDj~{l)LE zm%MhcrlBp7mT2&hz3C}veH4mrVLj!(^NPpqqa+DB%cE!f>LfM8dj_jWA_slMQ=n-` zbR-N@laQ|;ndj4w56WJP0;3KgtO-YvJkF(*uyXW(395OfYL7dAy=*?bgB z)MB;gF681kcd1e7>MY7gH4v?VaY7xV^mszq^5#9Yd|+?w;N^@*Us~lHHVb5X;ZRZZ z`-`)=i$KkJ`t`HA-GO z#)TY{M-MC!%P)wY!5pC1cL;D0r6nOZ+jx(Lv(=sMf*zC~n6!G*67snWJ#o@PiF>Kl zoWJb6oa7C4!t8%1j-yMn}Jr05AJLpF{Fbax6VCLJ>15~}l>a3U;R?=8 zoEV52jk|2sL;7-MlNRL@=BPqw2z$}7y?1%t^k~5Eljm#_;F)8)iz1IU!3Xc;_nK>u zqX9BAdU2M(aZgo!sqp!qrNwM0?}*TkWU&}Atok9V-;N8`XB>Go02aOs{$Qke8t>-Y zF<=^O4woD{HNXS7WdL%7b(;%a+$)IgcSZ8Bx^Dck@6&$VrP*!NQbzQ39^|tT&k3hD z!e`5KI$$6IRz#5H-&?mJiq_X0mG*wu@XbNg$b!2c`j#AD$4%_Jv^RWu(?LbP z!35%eZy{_fp8!apfH^SR93vDyp9Nlhkat~hkm-AHB^+r$-#ld2=)Wn~eEByKs^}cl zWMNeX3=V*ig&2g{VBe!}2c~wz1LX;N%HKOYhnF^?FY;^zKKs)TRA{$_J{>M^)0J}o z5SqA9n8V~5sD>>i-;(6n4yL?DIk^ki;(##GIi6#ebLg;q{+q~^N85CX#c5iL@|}y& z{Yt2~@1gIJ^YIGrFYjbl@cyi4|0m>Z!9l4b;Bq`c?Q3`z;VyaD!SjI=X zwXiL2d-=?J%@=g-FUFuIt93=U%cpKz#i}*5STPxBU*K>7FlxsYO@QxT9#dTHw4LnE zpX>e3_I4d`9R?95{tjM#1ixT_U1QI{^ti-I|NiB02=S@d271!Kz{8WX4HVx2xzT`Y@KeOKmq+@{VvBb6#hTWJDp)3eX*b*qDR?Fk=Pr z&=$uOBy->UwQERq%{ZpL?p=wbFBD5dk<60((=ajKU!9mz+!QawjFJ^p8-wT9suc%5 zOg$wo+7;*tdKrUj00UG16GzA$usc`1{x@Gx@A-Ip4EO4%<8kU`=q1NT@}BFpfWxlO zHiN0tRMP(^j$!n278l05?n~gSVw*W%=HLLsS4v@Hw<|0pObpA388-g7S*z;3cQ_N} zPnc3AiVfKBk-(*qk=b(x1Y>e%JRZBEEgQ5ivW}ro{yZ?@!^#tPXSBe4nHmFFg@K0Q z-+G#6hx>k-Kx)kxvwLfo2xl8h2}#%fDfH{}EvWG{Rq>TK(!X7zw+ELi+Ku<=>nerq z^!N1UFoPx@c8%yS-^8i_EsEl;Oe7>@g0fzpU7XvK?sea@Lo9LYSqu#*o}4KcGvn{k zhdiV;gN;8dkpH6=EzU~cnB>c8Vwgx zoII`$RLxvn{XLFEQJyu;WL}}Q+q07T$l;pxG+SL~Cj9YDz~;HMEzvB{YW zUv=A_S3VJcfl9i`>)|AyJ9Mo@UybR(`L^R0ruThz(k;@oF8`H83c&{ERh!R?citHU zW)Cp19;C_$SPuat8m*7Ml98=4@#7cep3^e0zEBbXdgaV$1EqK>>vufW8;!d~hZjT= zj%@5Z*b1i9cIO8!xBHUh?xCl9%(kBayEs5D3&Bq)_qe7upaD^lx`(qr)tWi`7Kn#m z4?x|;`i7w=Jk(xnoUIztZSQw~JNjQivy18e@lc=7l1;2qJ*v5fAU+7V4+CTnEh)!8 zAYB~SK(k0r0ayBo#e_p z)AJX7>3Bdk@Iz^S0Kp00Ou51}Rt>r4DKHnwILTOY)k%r>QJse76AYepq+WW$z^C}Y zajc>E3C$v>7ob8@q&pfGKTQPGU*A8vUQt>b6W`GE5M%;MRQddbuC09)!`2XS94E&4 zVdI#uj5}Qe;S;(x>@2B?aZx{|eljyru&U_{uQ%EUK;6L)tU*`mzu<@q$7BAOON=Kq zNGTnJIZW+%S2*bD6rL@Bq$-(U{2UMCGA$h?|a(4S@r@Ol>G4D z_9Py4xr(=M9!*_Lyy6|xtXID6nGCQH1`nVV@Gph^E+i3p`(IJ!wVe^_l1Yh(Twqv(JHqcOG+UG^&EhgfW^F|+_G$JCRlDu) z?cf~64G!ufb>rtAHlvLX2OkH|R2Ea68MtqSg2ue4B}CP zK&@45?GWZ!pXEJ{m3^sZ6T_e#bI*gpcOV*wNdE%W^DEQ8K+fk0Pinf7$s8@{0S`{$sF%u%XLwT=G4 z15qr-Q_2s@#dhsXaL&NWK z!LuJ&3NtodpiKvL4$Jeex7yUBc=de3dIfo&{jceO!*tFtaA^U@Yj~yc19e*hfCD{r zU%az3*gJ%hzBBk>*}pnj_d8xGEA~^pDgGbYU$6VuQ|5a>Z*DHb-9JsU>}US8E?GaB zTgSNIIn@D#O82Sqh=I)(ACc!0yosZ6(~N7ATYN|QJ|Kf z3a&lQU*)tG0lD-Rv8_d-LDVMvwbnY&EwTwKn{Qtr&=VedlkL+}aje%J0HlU%58`Ai zAHf|~a?Sew@nLrjUeg^l(}mB`d zcoSmG8EukOjGm>EO>jEs@vLP4#-Qoz5~`%!egE)0AM2Vlw^xX6s?&xzpt;KMFGaj* z#~d%PvW*G7DRg6M>Ntplh_#k|zm^h%SfQ0Z&CS|yR)H!mIeFA&e3dU`{BJbP;S z`ufPHHf@~J^AvOaxnJQdV?6z~IhUJ%Nl!HRpI04&1fXNjV zAmHC1yWG8%_C}Fu!3r0QQ;Gh9M#7_Cu~5^>N6I3e_JRiU!YZFn9ieJ1MO*Bsy6FaX zF;juZDm*?ER$miz*v@Y~?E-jEsy>Ax)e&}q$^Gm(cj#sUpBz_5s?^sxx2TU>mPLu%0pe^o(mXJ74 z!V_y^siO9zP3CQ!nkQ8MrU9#R=9!F-S8hbd={rwk%07GiJ*?(Ks}Km3TYhpBcKcp+ zvufQTrv3zk4p<1U2{KBz6K(Xjz@HEwO_rqVaLf{sVKZkdHD7wr&nW5+#oYxNDuRyF^_Q`6=d;E8>o4^u0ZvhX zy^kK(e7aqo&%3pv88`q-4>tLb@atmo~Yc9x&XpdW55mx0wu?Gd8C5)@a zy49fZ9C7&5soKdy_5FZ03BW%o$;iv%urn96b(mLt(j<|R;fXrcp>WJ9d?(g3N&t6f zYV$|gpVz9+X8-Bu&^ZGTDls#>w*H~3#9O#R#sX-168-Sql1oTs(jv4eRA9EsaV4Kc zvb#4a?a2^o2`*)4(K(p&O6YJ!-3EU_#(sZh6;4|n8Ckfy!^9+Z3^e6GwM%PheRq4$oZRCrS@wi(^~-+MFN(DY0a(3 z$tzZH>ORxcKxz8PAFyPL9Q5#K6vVzbKtg(>1Py<4tfZCTY_g@|m2+}(6HV#1;rVga zlOjyKh;qW^;tOJaJbw}FHOfvW8{xoMCH}Kf2_I#$f9q`mF<_#LC%C$$qdH${mELWr zF+q4jam0}B-sz|KLaTIS4ZfwedD{LYzWK8cyQt8+CrfO2fcd9zZ$Q@$P_EX}rbYO^ z?-8OceH`2gj#?6&L-@!@Tc!S-o<*sXkiZC$zTtk&#up7d8liKJ;f^w`D`ECupNSfX zew2FnuQ+uEWSZyt$#bsK(;OlV6Mb{fv^4UKg+^>9dHGNR=u@?M+6KEIwWAdcGHt+}I#00ii_8S6A zn9D!jZG8MHuF)1E{vtH2{{&zZ1@ML_h(b1b))zNm-Pqx~QX>qRZf%{#*~PcsLt9NN z*e5zO&chR(_1`qgdtc~XDq=!I(GF$KxhcmwSiA;|+-=;bRHwE4{xRq4=+XB!6Nv(( zlbdC4R2COWGASZd6qT4A_;D5?ZhrUX}d5fbK$=1#?!zQ3s9d849*5Q}pHP3RCG+^kJJAc?I4 zRR%%}ch3Ig#G%7|27p9uoP8IOmKxuB=zKzAYnJjA;4me4pfiddV=yLH2x>tU$h2vG z9zC7Ql}OI%Gsy)x%?*EoZP0kW{H}CnH(<)wmpk)l zr~F@pZl>OYf*qx9Pn~wL6DHh0u+(_-&|jc$rbg$WI#f@!o*MdfuU(I4wqz`h!&zP& zy9fQs2(LGmWIGf-Hs>W6Hc9=LcgRH{d(dQ@sm7I0<5mcQ*@KdGYB_%;@^4z_eAN`P^e#hJ!-6hp2`4lWKpn7redaFf z$w^UqaXjEMB|4!E1X%{$9X=A$e;teexl<=zME~F>InSV2Sb9%;dymd zu`|@lVl*L8>PKTcGdAA1m~$E}o-3wLyf07eYma1%$N9>Ez;9cJ6wlk&gdl3QfKG}< zT^k)yqD2|Q${(gzBFbMDT#0x$FUT=@&bmht4Lz1`x}q1U(|;5ll^{;wJBI!@%qBY* zrx#}w%vY*EZrwW@FnlFd>4 zK(Rz2v10(*ne97t)V(p_38Ph@zthKlN25<{KLGgGEnmSL9E?9<&h)cE+=6Mn)>x(s zFk1q+n+lz)`oq2W({VSWuplDaN_U`{jQdG;rGi#Jr{hd+)SNT1r3>q#!MR}f7<%>- z!WMe;?UbRLwEjW%%y)O|YEA=oQGm>$F&wjU)`Gz+O1{=Cu>6gbX+q)EzByzMp82Ix zVVD2@Y}sbcz2U^PjsJ$t)@Y~=Kcp05l_C>x91S5o*S`a1GorTvJ^=60jn;-8A9??* zX4Iy9!Y$sln{tKB4Z2*=G=5w}PjHy4)}9qdBNc-L(U_{@A9TnJ7s7&o*IK&Of)DfYP6A5b*5dSF&# zwA#F#9b$0v8dA`cp;;(Jwph+aAPCSz95MV(VI{_|gL2X zHBra+MlD%?mEz*P5uIQ z;d9M-T&=`jf6xq%CqPi;#|K^2>U|Llx7C?M zVf#O6Nau<3KtLMM=cC18{Fdse8)39R6n|CE9{SBZMe-yDquDQdLC`IRU+9N}kIe3F z!-j>RKA?dkz#fKOMJBO_Yg>Aps!PmCsOKBtN8Om>4b>jcaO(u-jgdj9*>CcVWub1r znl-mT=u!0swQ|B4>3emek7RZ}j!P^)pggDtaBwKt#(bUMc> zRXMTwZ!*-4)XSZ)sx&Dvb_PPKfC@23J)t>?{x@%jcVWrRhc?4DY&uUz&OqN@$H3)v zs?EK%IfgatMs#BL5d~obKnF*mGsJq{T+Ek9qTP9}0yRS=n^K?z#tbw5t17bEU9W$4 zetKf2htsrV{MP1{x$qri1Gdus-qm|VEiNiMX6Ne#<_V5KuwS5TU?ka|1#~?o$CeHa zQIr(^+6K&r0c?yaWDNxZ?bLA(^J~rBRch;QHF96T-lzNnFBN~j@=QjL>)^cMb%n}F zI~Ek19l#dF&N*Wlwodt9PV$rnLpK!c024X}c8PI|yLCl+)$ncKO7b>O4_jAt-Odsh zl%KFbshF|tWc@lzTGlv4Kni3M{u>OzV520Wn?A;V7*#LZk-Qv-0URO#+eS}DqN>F2 z?da8*eLYpSIQ}@n+45}A5%k1G?}e7)S6{!J+rV3?c1zs(uWnwVAWL@EN2)k$XJvmF z4~Wi_0rcpKQt6>o+I8-qEdtq9*^~rn9{wb%odblfc?3$Ujm4-TOun@rlLXvq`h1z_ zx#9iZ`a0&bRpzrUzXN1F1)}spxq*MpDr^gyXKJTK<8GN=x-K;Zr9+JjKI zs~O9gfCyVs8AOV;zpbJ?S^SLl=+#@xqhk)=PyA0xA5H1rH0o={E(*Mzyi%Y^7E>GIdIK=T9$&@-D)WG04p>0^C$zF|g-{_l-C{x74%ZhfasLtk$j^wDg zy0tWTrPYd+GIQN05k>$#O1b}zoqHuPkU8tx5SEWWc;Bb{P!X7Dy`BBB!T9T~a|2Snme6pM^@EEEqp=aV~s}@HZMu?(fg#`+ExklO=#c_XWjwU9^nn0=`<1b*t(JHBZ-Y z^q#9mJtIEi;A6SNpF`xIgg&*pmO8ckcfuF?H*-OTgnopkaZQ~k-TU-eg{e$KwFQ~-&_U)?r}1^8ZjRJ z=AaxT*!oExs=RCH91&;ET&8PM#O_ofxY1!l`)|&Q3=JIHJer?%JH5(Ie*J?(tG)^b zm;&TEdZ#t(*xwn!+Y^`qL$mS%@A6ye3D41m^;BdWH7$(boCPh*)~<@eCotH5!;j0j z%}u{M6aMz#_`&*P7ri%x1rOaeJMK|0{_#*p*5u zO>Ai6?T-{n!WY*S%l!&!6|*ybQR3AE{8~(z%JaPXgal?xRNnH>Ri zTJT?{|Ck|iN`owa#ySmG6G@Y){%dr}5v>sW%<%&(#hVG)^K31H;dP_v3;W)RIr(T| z!(Pr}o>ijEGU2-bEufAAuq^cwvYZ`@=@-t`_qT5ycqjHG(;8HkPL zjpoP}v_mU!vR`a^D2-})`X5h-0A2mDs%Ym0D6`XkgamR7Z zC9-~3;ITw@hdB$HaZr_-vGV2b8snjU7p}FH3Yizz)B&p{-Vr5&L>~<&KSDsdD1-P3 zwgdYKolZ=Hl?83O`YSOypG6ofPO)I~{tdJ{0=n3fAJ{DkUe3h8{V(45vHiQZ%ETa_ zp1ig5iBIGdJNKT>>%7rThWB9Z#E|zV95>B=GE6VnMNc;VM{_qY&mwhWHq0vSXkIRO z#kO~w53r&59LxsZ(~>d0u6=L(dDJq4AK51@EqoR+#S+lDIoVjLDay?I9tVI7jCAIwswG*MrH&g_4 z-e`Z15Wqnak#ctML$;5-)l!U#J;l+&h>2}@E(U14eF&ZoKEYDx+eKWnZGXh}%008% Na{YAY`X7sc{{tR9x848% literal 0 HcmV?d00001 diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/columns.txt b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/columns.txt new file mode 100644 index 00000000000..0d87e8d0e8d --- /dev/null +++ b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/columns.txt @@ -0,0 +1,183 @@ +columns format version: 1 +181 columns: +`CounterID` UInt32 +`StartDate` Date +`Sign` Int8 +`IsNew` UInt8 +`VisitID` UInt64 +`UserID` UInt64 +`StartTime` DateTime +`Duration` UInt32 +`UTCStartTime` DateTime +`PageViews` Int32 +`Hits` Int32 +`IsBounce` UInt8 +`Referer` String +`StartURL` String +`RefererDomain` String +`StartURLDomain` String +`EndURL` String +`LinkURL` String +`IsDownload` UInt8 +`TraficSourceID` Int8 +`SearchEngineID` UInt16 +`SearchPhrase` String +`AdvEngineID` UInt8 +`PlaceID` Int32 +`RefererCategories` Array(UInt16) +`URLCategories` Array(UInt16) +`URLRegions` Array(UInt32) +`RefererRegions` Array(UInt32) +`IsYandex` UInt8 +`GoalReachesDepth` Int32 +`GoalReachesURL` Int32 +`GoalReachesAny` Int32 +`SocialSourceNetworkID` UInt8 +`SocialSourcePage` String +`MobilePhoneModel` String +`ClientEventTime` DateTime +`RegionID` UInt32 +`ClientIP` UInt32 +`ClientIP6` FixedString(16) +`RemoteIP` UInt32 +`RemoteIP6` FixedString(16) +`IPNetworkID` UInt32 +`SilverlightVersion3` UInt32 +`CodeVersion` UInt32 +`ResolutionWidth` UInt16 +`ResolutionHeight` UInt16 +`UserAgentMajor` UInt16 +`UserAgentMinor` UInt16 +`WindowClientWidth` UInt16 +`WindowClientHeight` UInt16 +`SilverlightVersion2` UInt8 +`SilverlightVersion4` UInt16 +`FlashVersion3` UInt16 +`FlashVersion4` UInt16 +`ClientTimeZone` Int16 +`OS` UInt8 +`UserAgent` UInt8 +`ResolutionDepth` UInt8 +`FlashMajor` UInt8 +`FlashMinor` UInt8 +`NetMajor` UInt8 +`NetMinor` UInt8 +`MobilePhone` UInt8 +`SilverlightVersion1` UInt8 +`Age` UInt8 +`Sex` UInt8 +`Income` UInt8 +`JavaEnable` UInt8 +`CookieEnable` UInt8 +`JavascriptEnable` UInt8 +`IsMobile` UInt8 +`BrowserLanguage` UInt16 +`BrowserCountry` UInt16 +`Interests` UInt16 +`Robotness` UInt8 +`GeneralInterests` Array(UInt16) +`Params` Array(String) +`Goals.ID` Array(UInt32) +`Goals.Serial` Array(UInt32) +`Goals.EventTime` Array(DateTime) +`Goals.Price` Array(Int64) +`Goals.OrderID` Array(String) +`Goals.CurrencyID` Array(UInt32) +`WatchIDs` Array(UInt64) +`ParamSumPrice` Int64 +`ParamCurrency` FixedString(3) +`ParamCurrencyID` UInt16 +`ClickLogID` UInt64 +`ClickEventID` Int32 +`ClickGoodEvent` Int32 +`ClickEventTime` DateTime +`ClickPriorityID` Int32 +`ClickPhraseID` Int32 +`ClickPageID` Int32 +`ClickPlaceID` Int32 +`ClickTypeID` Int32 +`ClickResourceID` Int32 +`ClickCost` UInt32 +`ClickClientIP` UInt32 +`ClickDomainID` UInt32 +`ClickURL` String +`ClickAttempt` UInt8 +`ClickOrderID` UInt32 +`ClickBannerID` UInt32 +`ClickMarketCategoryID` UInt32 +`ClickMarketPP` UInt32 +`ClickMarketCategoryName` String +`ClickMarketPPName` String +`ClickAWAPSCampaignName` String +`ClickPageName` String +`ClickTargetType` UInt16 +`ClickTargetPhraseID` UInt64 +`ClickContextType` UInt8 +`ClickSelectType` Int8 +`ClickOptions` String +`ClickGroupBannerID` Int32 +`OpenstatServiceName` String +`OpenstatCampaignID` String +`OpenstatAdID` String +`OpenstatSourceID` String +`UTMSource` String +`UTMMedium` String +`UTMCampaign` String +`UTMContent` String +`UTMTerm` String +`FromTag` String +`HasGCLID` UInt8 +`FirstVisit` DateTime +`PredLastVisit` Date +`LastVisit` Date +`TotalVisits` UInt32 +`TraficSource.ID` Array(Int8) +`TraficSource.SearchEngineID` Array(UInt16) +`TraficSource.AdvEngineID` Array(UInt8) +`TraficSource.PlaceID` Array(UInt16) +`TraficSource.SocialSourceNetworkID` Array(UInt8) +`TraficSource.Domain` Array(String) +`TraficSource.SearchPhrase` Array(String) +`TraficSource.SocialSourcePage` Array(String) +`Attendance` FixedString(16) +`CLID` UInt32 +`YCLID` UInt64 +`NormalizedRefererHash` UInt64 +`SearchPhraseHash` UInt64 +`RefererDomainHash` UInt64 +`NormalizedStartURLHash` UInt64 +`StartURLDomainHash` UInt64 +`NormalizedEndURLHash` UInt64 +`TopLevelDomain` UInt64 +`URLScheme` UInt64 +`OpenstatServiceNameHash` UInt64 +`OpenstatCampaignIDHash` UInt64 +`OpenstatAdIDHash` UInt64 +`OpenstatSourceIDHash` UInt64 +`UTMSourceHash` UInt64 +`UTMMediumHash` UInt64 +`UTMCampaignHash` UInt64 +`UTMContentHash` UInt64 +`UTMTermHash` UInt64 +`FromHash` UInt64 +`WebVisorEnabled` UInt8 +`WebVisorActivity` UInt32 +`ParsedParams.Key1` Array(String) +`ParsedParams.Key2` Array(String) +`ParsedParams.Key3` Array(String) +`ParsedParams.Key4` Array(String) +`ParsedParams.Key5` Array(String) +`ParsedParams.ValueDouble` Array(Float64) +`Market.Type` Array(UInt8) +`Market.GoalID` Array(UInt32) +`Market.OrderID` Array(String) +`Market.OrderPrice` Array(Int64) +`Market.PP` Array(UInt32) +`Market.DirectPlaceID` Array(UInt32) +`Market.DirectOrderID` Array(UInt32) +`Market.DirectBannerID` Array(UInt32) +`Market.GoodID` Array(String) +`Market.GoodName` Array(String) +`Market.GoodQuantity` Array(Int32) +`Market.GoodPrice` Array(Int64) +`IslandID` FixedString(16) diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/count.txt b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/count.txt new file mode 100644 index 00000000000..7821086ced1 --- /dev/null +++ b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/count.txt @@ -0,0 +1 @@ +1194218 \ No newline at end of file diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/default_compression_codec.txt b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/minmax_StartDate.idx b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/minmax_StartDate.idx new file mode 100644 index 00000000000..0cb239d932a --- /dev/null +++ b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/minmax_StartDate.idx @@ -0,0 +1 @@ +?? \ No newline at end of file diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/partition.dat b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..9c176950d3f897f5cc9714d7011107a21e47a229 GIT binary patch literal 4 Lcmdlj#LNHy11td1 literal 0 HcmV?d00001 diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/primary.idx b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/primary.idx new file mode 100644 index 0000000000000000000000000000000000000000..4731cba6d463bb6755b51e8dc1e552175657502e GIT binary patch literal 3240 zcmXApc|4SR8^#TXtXXG}B_i1)yOZ-+24R?CG`6uPOGWl!?5||YQXwKsgpjgK2qC-3 z64{AJmV}cm#ru1Xf4`sWzOL)Of6wQUp`xPVfI_j^Ob7Yz8obxM9TZ|mW;$%TuYSIHA}GJR&WA=e{u6-x?-dP}2dM7}E>c4a3290!)z5`(S0Gda0ixqV+<0 zFhM~DtxX}yI=4DnrNeH)ga;zIK43b49!HIsrDJ3=;jBBkwo~{!AI~3Y$1;NY2!t z&(kvwQ*BS$lA5k&hqISb10P~t;a9Sy@BK%@rI>?%sew-hk-w?w^GFQseW|cWM;buB z_hzXw#gGaG8~5!Gb6}Eku-xNuQA92Fm8V7A4h?YLp2qbI9N(c?{3ph_bN?@}DL6sn*VmH|5JYWJ|Cs%|w>5C%1|E9d%3He{+>1f&k-qbw_qSFaM z^@OaAEslNNV4^aC2^aLhtBw%d6Wy09izlBDm6vku%OeZxQBi{=m>{9%M|KKh9oiOd z#pm2l2s3@^v-urvj(2qYelS5nX+LU^-U^ks`0=mIPKW`aCe{LjMWH8e2A+`8=JsFV z)yrM3lXI3Z;fDTM?+rbzF&!@cR_f{rS$7FL+!%^c_Vp_Wh6xW8Z%q|(v!5qeIW5EW zgb0f(9Zjrv_{$c&CBCs|0$W^OJdm$lWny8Apf~HDWCqu`@F)0ECO3x9$o3yJOpwrP z6W!vvdV|KZ?L#D(@Ibi=T6x~Qp4&_d!8XieEMRJAy1ybIXe7DCTH-ectPnLBnw zzPF%s`v2Ns0a;J^>~l!3MRlWZ_}R)H!&79;<=0W@B+W3K%Ur@?zkRQC|)dhX>4 z??&02kSv4l_NeA9+xwGujbOqFnXymFX1VK1R|keUoe=gE>nw|AUy&wne>_aMAl=z3 zh%dN`CyUv}rYA&nNzyCo;r`Mk5hJS;@_udN)0tC?rAY`UO_(5|FTs*%IeJwm&y-*G zCuHT=q*{q;PpMMff(#QB6fA@kdQ9ntWb(}Zn?gZ4#3_gZB z-+`}jErt1d?;8c_9bYy>-6eQ0MssOvhtKDSuSgAEq4cr>&C6mX#OL$-u1gmldt7_L z4zgCJjB)a@wfkDS7cF-5IKb3xmz7y}2CREjT@wZ-JkaO2^hGJudigujN``T?C@|%( zSg(jo&aSjCkqB+0;{h7`IVoY>B7d}E^h&6f3_r+^sp+x=ai-aFz({LO<+Td+GWqn0tpzGXgS zf?~~0xDGV>XJ+5bRho>2hQe?TaO5WwHw~+PjB$iMT5>CRydq&@xSjD~U%C85;Oz0HG{N8UYbG1{nxPUh z6=?SP7Z4LRrud+&4Y#SiG|+jW%=Och9~oonkR2w8N(UO#&^({RY*~h6QiT6i*DUZ} zNOXwHs&WptgMQP_At)OVPxQd@G-&0+6QWLaN8!0k17KJgmA-zXH4e&cMHLH9tqXmcHeR(xL#;EEn!gNpnj? zl9mi}0~-Kw-tSQf_ggg6++FfE5o!Wi%c)i8gxhbsvr+V}XlA_wq&)S~az;yToyxZ7 z?cVd9;9~Wsgmc!=1{p%3xp+m6KCqV;eyXZ_RV|#(n9VM(Y5=_8eqG&O%SIQ4#$Vh? zh;0~TDUu0U>@Rz=$X7^S{hc2{XS`q6ajV8|8w14(Dzc-FcFnJCl{a&Kl++4tQ8hQj5#JZY5 z(bpF((bgKkTSqYO`f_C%ln2L_X&RglsKUsM7vSeP`ZlQU8E zt#8)#+pOY?lmL+mUz^gd__>A0?PCQy)Iev$mQntIyvx;Q#=Zf&NG;%8Ozov>*;h0^ z^XP$rFXt7|kPzprZXQ7*&+Cp^^Vwm6ro@Tw>%9^tvL8O#!{iT4y4#8vrUaYn94;Jc ztH$EN?hzqpnSRK~?74Lf>Bx;*Avhs=_UTEalho_@k|KYzEj!S8WyA>`qlR2)pe?v5 zsNw*89Ql5isS!ELgIU|Hq&HkaL*{kcnvWcY73SqpL9yYUz&U|)T%;l|B5QIS5!P^I@$UsAUP3L>a-xB?^#1uU)5DBtesz*y@A8ejkcImwOJ2DoK zSaCzgI?L3=>V_)a^MOfV$^}K)B05@cUrSNNw-GXc^S?&*e@ymNvQ%+DEQ0Pm0aJR~ zXMPd*coR6X#jmzo#!EQBcWH zZ(8$`C=>MaX$5$T!g-9lnDqD?;cnt>lf_rSc};WXB2xMoQ!)^UN4vfT7t1TPOivDt zk~21T44FS{1e(O0#2u}MWZ#GSyFmuaZ6G~~Z^9UDy}r0{Dj@W?O9vnn#=Y?|TSm1< zGnd_0^}0d2BC|cMmmMj(UprDqL2Cf`AjB}M;*2Xx?$S9%Aw45NgJ*bzx>}%qIh_lM zA#_gw&5ProqD@_m{k)ebK}T$#L7xjaAL?U|2$i4jZ)1(tR=|5v+~VH4B9j-ex{Nvp zKHtG!qMJK~60EZF`|B?LqaE8o!$2vf>XS@Aqa2c@EiJtd2yco;qFOdDZd@}SV?%ok j(w$po#558Imm2ua8h8CU23d;MR=5E3ApT7w51;=7A{bPx literal 0 HcmV?d00001 diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/checksums.txt b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..ab58ad06ceeb0d11e4723395c0f1b415399d055b GIT binary patch literal 14273 zcmX|IcRbbq_kX?L?|bdNHz6e>Nk;Z6$|l-%ac!5b9hH%pB1(2e6j_m3vLcm~vOXj- zl9cYXvi)At_xFzn*L|ORUgMnS^E}VF4lYCocOO4bA2BB{vM0e;%%4d1arGh{6jNfr z$A+4pV9tNsH|Rtd_#W6Cep>|qR$~B`WUgYYx{m&aBxhF=(bz!V-j%eHfLa;ptFG`r zRWu$*OH3UcN=6di33DebTc)d3(ma_{vRqbR_{F9IC`_JYcg22YR&_RRnv+UjldwiP z&%DM_K&$Iav<4L1LN2?fgqp|6+X;^y*OB_{=7x3nGTNeFI2E<{+VRtF7G@p%ZaccY z06hw*OnaG1Cz#je+L+nPzdy9#V?s4(z8K*^?f0lT{UuL z{9~L40ZIV#sK=G5x8Pv!e*7NZ%b7{H*}yJfZ$cnB`w=)13T~~OVPr|A__YqUKbr3Y z5zlh$ls%Y34I1rvFO3GjcOOaX&};ntn#&&G)dA`NX7N97qeGlH@vs+OWhT-HTR))F zH+g|j`CmjJZF5seRusW9+LzZgS+v=uk>^kS&r;gouP-IJdtbd+!c6#GNUs1;Q$XdD zYqm$V@1W#{*t24=r;2DTeuo!lqzSWjU&|R>+ zFN#~6>zVayWD_-NsGQci`e}&p=Hk=Sj50R_bpa0b2D8}5?s99|hWqSCSAL2Ph*`7bvt>-`}TsM*A zYi#ixUcc3w*bVESPc-Q0R@8Bi)l~T>Kb!-zNEgTHsN`wOPxf>(sq4(|wMb+M(} ziVnq8(Nnz7a5(!R^87yQRc~$$fD*wRM(VIkx@%-Tk?%fgZ|wg5XjsR*+`0Rc?Cw%vjI$SM71CPsjx5 zU6#{9os=SV$Kj;5{C$r;hY-NFIVzS}m(s^k;lWWF{5+bk+W!vVi2=tBlfC@BU!oKm znDFBCOoWZhnG-{H{Hk&t-Fy7^q9fQut(wz>O}mqm7^!dVdH1HZ^J)Tsy1>;AZL*#B z=f7(CX=MoCmzJFh=ukxyFXu6YDv%q6zg9}+V16T}_J-p(b8?E$7_#*JCo(oSNq20e ztKne+J?<4YNyBOnP#6*?0Ts)ga(9f|DNO&#*>i->F251*OMp>R0@faEc1(01PDo=5~Nq;221NbpahnW%%+jv83SFetX)0aLt|P z0PbJD%Have-sg8J8b@D#bG_;-L?16dax-8(j&Z`j;s4+rWH@F`^dLGU)6VjJC_;EC7O#cHOVZ;m zn3stAT`<^6Yb-J}Uxw~pA1$&@sy7SfWp)6x+rWhuQU8#=4)z{;TmMpAH1ZV<2*nd^ z2xMoXZy7?Rg;7=X5P6nrsgRrZ8z1tp`(b46-{EpTz9C-rZ4F=R=cithjG(drXHPfFTS&gleCA8?J>;EWu-Ho`p`9jM z7#)E3dP#I(B5#g69XG9gQQEn+EV#HiB&i~zLHjfH65W$R+X})g)g1$;0os74V$;jW ziO3R{;-|l!-U8?42Sepn zdIN)(XXwCwfZ6~m(z%?R%$$4q>n$-S%@w9f5q4xBgsN0p+&V1A74In+K9xk9?2}KMg2|SB6r6iHhmWuIQ*qHV%5r8h|PR^ceK& z(&K|vMuzL>4$64ES86xVCMoAyQ&VAJ>Va(5*fk_h%5b&!~vI}Wmi zJe_K-o2F&CK90F{6=2Kgl@eA~J5fr+jjlK*u7kbJ+hm4{&C~IW?Ugo6nAUAvuS)As z0L=SXa9MDje>z!Al7nx@vU@d!e7UKrt1)%45&}CO*FJ`P7(JL5XVFnnM0Vc>b_(s~ zq(9t`0-7A3 zCB7#r`gi8?pDN)Vht@m@J}#yNH!m`V{NXQ%X4&JY zlsn9=Nmo>BZ}l?gX604}dL|oM>9aVnBUp;Bs(Qvnf7{$VGOz1qdqlR6PI_T6&105^ z<^<CBe_ZP31G(zy|JP{`hoxkzlLEyBtSPz?2}#_2WU=;oBW%7gc*+pRW==0g%6r zQL*P59+o@x!qrRcI(hpGnu-B?2*8?Aw*8gR+GHDxn3h>jU*N!~bs#p1TA%jVfvupo zOv-%B4=Pgg66?RO-l|^U_?zHYkb<@(b5~zqgMM3|ugg`o+UEe9(m80RqmAbHhQI#d z>9XRueQw_#--y;a2bPg?Q3}3HC#X)&)#lM9KOeDkfOG5ZjCU4_g;G9QgwMGP5UR*( zZvoRGvX`e3+%8BNa3gQfb}hMcvPm?RN7k~eK|deGfo1`{9+I6cVN9z#qIZZf=L}mQ zGbE0-fPyhl7COIeFfj=%9U2;_P89)+JAi@>!5IcWUv3_9j_4RM2g;7!Mt?icyw)v3 zUbl(fFR#r9zPuKRoK~vzT%lgEU@M2g(v30EoqRv6J+iC8c;T9vjc1t-eN5FxtX{?W^U3(6-o3|WfMP$cwi1zN8Dl~QQ_7C zs54l{E^|klC%LWWI)xKE6IuCZfPgfx@NvCJRCo=STUMx)ejuz~_mTkThx*FNGaZ)d zBCuk~YMb-s#4QDWKFINL)JkwtP#;z-WI$!J3N2l@siAt9rGAKRPrn-r;5h6>@US8h z99)P#x}LwF~GP5WRh;VT8^OVqug&)lm=b@wLj7&J4J%U2_2I z=#SaqOPg79ankj|t#)rFFK!nE%Scg=GuiTWw5}$v1$TUcIH?U_9*}c)=g~ck9?`q> z@^iI`(|mPrg9@DkSVEd^Hv9~Fe#3r0>&DFQGV)9t0Nc3{YyNTP^rqC|oQ|Fei=^^T zjesW(RQT*vG}QMalZhk;U4~7*CK>BJ!jBYMrRa1t6PNV$lY5cI_l9f^eY@1Xj(*2a z>7Op{vJU)5Y^i8}aQhqEV>&ZOtR99u6MG9;H2^Mv9tR27V)K5r&TgmOx#GR-e#80l zM_zHdfCc1m)JwauA$l*);a9%?N$kd-S%V^-WB9s8^rf!G!#Bw{6Ro$yoQn25!Dy#` zFx7rKHkJ4zNn9D2BcKt`YA{oI=Y`K?h@(tNFDYer@wbhie4|0G_OnYjT2MP!P{Tu7ENY$RU5p}=jYZtWqreeet zHIJY`3Y@nll3fWN7cmO%>2)^4)%>QxWE+K=AU&@c5L_yPEFcBD%~DoumfJ#>qQ%~i z&3LW??C+7zqCLk?3);JdC`g6U`1Ez8<^V&4rXQo?B4U~Lmv72T3Mj~&pR(c0q~+A_ z$Cgl$$U#raiI;O6uKKrY6BE?fAdfx`=4mfyPq!_;2qgpWZB9a%9jq_@16Nx86EmDu01udkNGcMnZ@fqXJ1({Hx(Z; za+^Dl21k1xmRTg(hc*^Z2AT3;L-n@|i z>ah)2LcPyaXaYO44++7`4gK{-tp6aKHer>=BumZ9F*{#BA&fu70x%ep9J~}bAyTfh z^mPqs?kxT`n_6^M1JQonxeB!x+zyUe&HU~%`;eRP#OE+GsMXMDoEYp zUrkOoIZ$iPksYjis^(e>$=W9xFd6&kdHI>}K}$7E=5rs(E4-Xcua4evW8YXQEQG*) zk|Z45|NbFkk1xUUzNP2bKj3EW99ZorXYsmJ$a#$S!H+%@z+&uU;1xjf@FF;t!)lHJu|LjVDo~f6;YZD}h-(u;tT5PlPzL~xdK#$_9Dtd|Pb$k*Kr`v7VVDX2P^yRT$`ho%+(nLHzr#xoDV`4a?^BQekl zg49EayHfG;ha5t+g)~1mzxQ`lwS-O$3bI~d{aDtgOtN8%=+Cp7+-pA@^sT9) z`e8m7SN}&X{Wm{eHUFjgcB0MUUG3WfSHNiEN^-wQJ1-3R;PYhT)VCWF-cDBP-WM6r zwNHVH$Z$snLgzKbea^bvz3-3sVdj^o028ExHc$z?GvH}SKM*CU6rykm?C%6XP?swe z0*-+zndsnqm4A~**Jb?F4{7P?H+HeitLHU>QjmG>=BnkW-G`KAZ2IOsViY+Rg>(QO z3X7}Qs`~Po47qUY6Zc-8cQ7bV0RjqucAaL8TYL9$u8`=Lx3yeQ8zcRD2*=I&8R%`10%x$syMfU>NNbW8&GoeWz!w%Zqk<| z_roL6{8~~z>0D}nl>pc{Ayr-CiJ)svJtKYZmxEaxk8+XRK2rvyRDiK|l#8A2l;WEw z|M4dGTE8Ix1mXk%{K2YsvUiVy(yE7}28V@jDyRbPSRm=;c#?O6w_bR3b~nu`ioTtZ zebCw)rW%1fbUEzC6U65nbNcj{&<&}P26-NMCx~DLZL~=VWced@pry=6BzeDWIM@Rv zWV&SmxcVu&xUOe0GLN;Q@5vThC0|Yf$ak4tS?}Me%qzTOYYABIeZKgE*9o9d_Hl;E z@4u8I-_pBINPQ`|6+NIxP5}He;DV(ef#mDzTgAA@la8LerZg8;E$Z<&GoQ*o5)%aX zWUh|s)K15Bb<&BUBef;pb`_woLcwbT)DtlGp)>>pmxM0zLvO}~QrCYdGh8|=uh8`E z_FcJAO{ChJqkN^s-XKuU#55}Wu(rS+9|Q+s&jHp&4di}N*(WsjOqZk@@X15GU=<+J z09^wMTT7e6oA7+4cD?;3RVvghd|UEt5?mU3%)~t72UTE?#1ouhV~9E*GeQfAhbv%K zL%3OG0ZQegAo2tI6kDJ7`5kYNs216MTSgvMhQ2i)%ZfH*jGb;UNLNbEGqSyy1!T^mN0~J>gg{(TKC4T2o3Hof>>_&eh;;0m+kStz`Sx4}f)mR>H#NB!sqSR65VM zDsi$R>2%`~2Tt4yHfi5SvxFuJXUo$EFedlf7*FkdT^PHKk$Mph@Iw|;Epc^8YW^B)hY!`mQZOB!+yX9a#HY!WJl?+hI$ z(`;$?K zY!A`M_07eVRqui=Rif?((;u~o`OovW2539ROBNKl}{IqirwUaXFp?Q+dO|4EE%r$1@{i7F44cP zH2-bfzmt_83a6+S%;`@~4aa(a^SV8PS4hRzq}>N{z}JfCL?jc*-hhg)DT_riFSNw)ug8H*DVk^a8Ya$V+umpQqWHO}ERB zcG(aE^1#m8mq7L%h4%i7NJj!p!|gd(;?CZvA`yzrSpvz>-hT13^rmm=ShAHK*nDCb zJ|##7^acR+0Tg`ZP317*$DB{syOx$z&i2v)8gpw;NHSl%;+Qv#G&~eE;H4drsFTmY z=#zuY;U1+jIp(rFTOv(N4wEaxb!pA5BRJ-m!gdCIx~!a+cOh3TO|w$e8L*jq6G=Y4 z1Yg|{nq~f%caT4Y<)51)kKYyamh`&Bz|4pIekO9*{Mj9*Dsc<5*XyK$7riv>kZ9(E zCG2CE{qy+Y_Jc%qyBXs8zy-jy6W}5dDsH|2lU&Rz_baOP*a{>LzQ~t>xi!APe|HZtw5vKzuh6x^w=@zj^d;j zlDa9vUKS;qvdL33ub|yT)Ojz&=LzUwjq3&jqhQp8tJ&*=yp}IK) zU8@*FCizkmXEt^&w7iXRm}R_lVhFm1DXzwbo2;%J#MWJd(^RkgmHOdz@CA#=?fgqo z!Vhii_xn6@-&LVpCW`V}?-yF&o%AmE8o=AW?#~NM$2QfLl3kIkLeZ16*REa-tsJ=J z@?YrjYs$&O=vMpD5#BhZV~$N0YV^DctmEn9cfzXy4#6A&aufWOaMLys@61GsOwGFe zfdi?nxm7~8J#Z3~^ZMAz1p7_ftTkw3guT#mP-TIP`x>lr`)HNf6Ay4BNUQ4jun)7s(C=*3jCFwIN1 zx5>*qU4d-<-FmCGwVTy8Yc;Qr|ALJT0-4|$!U+Lvh3~U6@ibG+5%pAF_M+R}XbWpH zQapKFgX13i;A1llIo0`7+CB|#9Pp~1h3`*QQZ9_8#BvVs9kO!ZN%{>qr2wjlQLz4< zj$FQzQ&Qh}d3Mzh9^18x4KuBwPIr$Nn#> z=rIMRcxZ`jKl}$DB@E$tAf9QhH+lETK&)NqOE*eee+AJ<~>@NLj`Ud3w6gE#D z+x7c-ju369&<}wa6eP~-)tRJB+?$)Ml-X={gJur6K{IQ#=~@LFJN$!r zCWSsfCjMy&Mc#eGj=#y&;9qEd_&DpU?UibGHE4$OfHty6o)S^IrG-=(or+xd?dn18hIz--y2c4C7!iN&VK0-U*~o z`Ua9!wPw*}q)t;aZrx^nkz*x&vL@=qQ?9kJSm>=SG~_wu}0m2dGg+zP5u(#R=} z&ksZ_F58!!po$6 zEoLk@YPa1F6Z;gyaeWO^ew7QSCXj)bF<9`^xmvp0G4uWGb9=w} zo}S==Dc%KQPe7fK<;7Lai?ywbvPs5{89;Ckp!tBKVuK3hR*?+R0T!t2KIcn$8%p6{ zAR7^gsVFZa7-U{V=s1izeeVbvz{2ixJG*m1@*|l=D`6HJu;La32%kekQEFfp7g~jH= zIbUq!uNBXYmh);LF#X^pJW~8JHvELtpPYje z*U}KpF&hnp_V4=UQK!DFFDLi=*dcZ*6nv12Ft`DftccEpU32{5VFF({6U48l@^|8^ zN7$Z_tppIt$1CaDoL7{c9&bZUO^r=ES7+XLo%yNbi z9p)7}?!gCo2)3L5gbt)wuV#~zcURg|YQXkCJ zgPG#PaibdEYOTj05?&C%>N`1^LKzpNj7i7xdbVvXph3DJzcY8-z135@>-ifN85ytx zASM!2{JrlHh?YkU&vce1d(T9E<1lUKxe%UtRB>nE!K*E#_f<63pLqhoXY3% z3(W0{pp)XG^y%=ub*2s3U!H(-tC8#Bx7DzZ5nkiJnZv^Q+v}nP@wez>PnJIuIhCMTczw^Tal4>~J0_>s?3CAsINE`tovWjT2;8qV(TvdI1H5_Mc-fbk>ZwP%?m$@sF^=x?Xlq<} zQ7grBbXa2X!99RN&l~`iQN-rCbp{iz#gzp%g+$H@0A?#MdoN!S(I*YMmQl~>zH+zA ze0sSebjIuyNpKhQPGk`sE;y=JD3ffgUc_D;=1kDsW)4gAMhn7xo`L~fA%E&dauO_0 z0S;>-f$ZQis|Y7$zOw0S3puqn+>m>t_E~@+eZHa=tfFT`JRAkj>YuzE*~c8~dX8mV z`k%!V_9*s>v&uf^JDy1&5tg3~*meO+i>(2G!<@^44_aO$?;qLTbftRi@dz-}+8vEB zAe(8i*ie_=Re=u*7i)(5X}=2=|8`_$#DR)WC> zxOU0m{SK~x*7^-U)b_5je!jfxzhE{WhP6F48<#QyrUMWP+Hm^$V<&xEZbP<**h5U( zaeLm5B0L@a?k2li^X0c((t2_{zSDrt8qSGrWts6^4g$Y(-nKJYHI25FdD13*hVyx- zR>z)=w9U_2otMW|b?9>J>EYa-37{}EB=#Q{i2m`EzI&F!bW_q2@LIci!0|(mJ`~$0 zg*=6-<3b+uLCb?}Z_3#C&Iv>t`+Q+fKMkXrFRP2exJ%G~pU8Kc!kDkQ|4$pb%3r1L z2|G2Zw|mif9yg*{&%D3_sdgY9@`D5)*^H)3J z`}GUj`(!UlH>dw+zEauq`0y5Kc*?Megzg-z%xY9w3KYVgW_F!a{pSxBDSlwD^yo}R zHNOClll{-!qE*)xViNJyv&W`my=JbcJ_c zm;k@Emjmn<2(Z$7!?1bPhMI>M4l8Ta20T`|bOq@XP}hS<@yfzLxVBx){8d$uK-8rT z83A252~-N9{-#sT&HED5yBzP~=3=1cri9vP(L2cxXoUb~Zq9uM1mAvG zWHw|ld|;LhxmB6q{eg7$zCHPC#sL_9Y1Qk$cSrDaE)v*FV|Ec1L#PC<40Kp&o z>!rieEZZ7J>;TRcM&6xye-$gkgZ*Z|eRcROs`@2`lougI&P?FJ7sfo;#VDhCeZ%5+ zyB$FOPP|dEc%WEQwBOsgOYCXuYl}e(z!U_qUcp&`rN#=HFKu}x@kdy1-07;4#MjKi zq0NdE*4y27H!Q=ev+ev-6fMgLQsV%?Ca)djF`@RIp9klfE%JAlPd)f7;tV+K0NO|g zT#!|PwNe${f!vFaMqb=IX+*i_Atk>9N%{5`3+GCq8DSiZ_`Wab2$vE)4JXW1_Fx60 zZ*EVyZ|1)9f&V+#bfx~6fM*xL$+E*v(8@uaVB}>PpOtGBY={9IHeTK)M1P`32P_Nw+8E^{ zSLG9u);kU<%8kf<_4wrkGgYUULRE8=t!bie6_+OOZq;qCsC2UxF==WEE=%)~&UbFt zM0o&K8!umihm++v2S)4|mZCC|N3D$VrsWG<_+1Rk9;g9}HOM@WbMN1N{48 zw-DJBSi7sR?mo5Ym?oUZGp9pZvMRShW~X1E0>k7#3~uW_*=vdfi^E@11@z#p-%^ef zZ?YyG#x_MAR7zx7N_%Pnw>&^~VUoG_Lp1!mZF7{<(B2w}5mQ#vK>rc2f$rQl`pn!7 zD~`Px6hv3=vc{tc*66k;mz96!w>xr;k_`vn9DiSXx(VC{N^l}ePhnNyQ4iYmS&`)Y zK=NU|#%pV7X2D=&aV-ylvnDEy{AV|m(QL4;eaUSQf?m?B2=yzwy$~ z_gC+!1v~ZvmJC=OmJEr~dG<57e(mTNtLZyEe>MGi5X^psjyp?4ySVQ3JG<9^Ff*o0FNP6z%6T{+GEwjYSO)Tj= zeaOT5QaoqC>?}dI_vy#-}3At6SzupJ2b-;Wqj3B-pJ4 zSQThs!)q}Kucq_j)vxmC4?Y1LtO%{ zyMm|hk`$Q-yix!!#!bb2=;M&n z`q}XTRc4MD(&4SnN)wQU8-6El!o9F}kLewnMib)hb9ec4VEya?W@(1Cd#>*+aLN8? z`9(f!eRB}7K##7nAdUDx0=Yj}<{!yt53@b;_N#nV(Xy*YADMp|i3AFz-DvtM-j>UH z=AfRQiU%vys!zgBL?Z`vtVw%WH0zzi?t)9HK3gp+d<#r9na?hSlB+$=@R)W)iAU{$BV5zvexq-`(O~0aD1$DW?-q8Tw=m$175H zQE{uO=sgDKV`yG>Y@A2YRBqC@BXFbzG5SoebP!1%kUt_7$+nU5U=d(2H6I65nwO1- zhzlc52E9bi>Ce-3vw-cGjlL@_5<&O3I9#0bk7v)co=RCfU2{(ToV1=F?*Pp*`naIZ z|A6Owzgw~GKB7w;vL#BM0rNCE#4jm2ye>^2DxdePGJF{TtjBCjgSCJ%?Gk@vD^1oL z|A4O&0hM+I^uB=i3Wj7g@6pTUrZ_o{2s-`*gVQcjp8wO)vwV6EocT7KKa+VekwN`# z2jGM=q$goG*W9LS4({1&tl=20)y4eoh(>=H53+{Nb;K26&R=+%j(k*$9(8)k1!=M) z6vf<$uaqLc$`{^4)~@)g^i=^CNV~U&%1M`4seDxww3l2HIXocs%OO_H-LTil8JQUs z%ZPXSdE^ep+FH=XEqYhFW#R9BodAniwYJ?pmWUu(Z_cO<+T}Bifa3(fJ%^Q0<`dk+ zYhKUP>$X~g4-`EMr^)^tR3y7xLM@~6)b|fqt#ar!C5xpkwFmhNWfZ2LI+2S}SBV2x zB(wf_{2CB}Cuw79N_2Gf+xo5mUqwHz==4c`ZL5iY5@;9SCw@xE&%M?D zL&>(=uW-p1p&+FV?(Y~rVf#KB6OQg5GIXBXnqokU2y{Ps^Gv^u8lkbtmw49SlCK}t)2nDUMIjZE;=UStdZK@qCJyQV= zp}-p8p1`M|^nXV3$-l>N_oe^b&zO4x-7b8=#0SQBpQA$}EwgFC^lsksJS7)P6D2s? zu#!K5Ha>T5u|0UjMmB}z#5Rjtz*=H6vPbuch0v{8oD#{A=>Z(#z~b+;UNfW!%}<|jQiymmkMH1jH&MDh%qWbOIS^To7mzXbQI?Yz2#7~2HW)cE!>IK+~} z&CI&zV0Tnp{BL-dyk|`x{3`Wijm}_j@ZmMrcQ^sqG(`g53B7f%pHqFNMtOTOgwY#5 z@K-pXV41RwDvCwzQn7zT@TWvw&Ta{(s5KuIU!7Ql(6Wl$KHR<@*deFVdF*g|$^~Q& zJxonM_;Z3F{yMb4mObga_P_B)Ea_Lc{rJb($t2VP-aMD_ki`mVyxB|df5Jnh)2B<=M&%%% zCvB=0^IxXAfW2Je4@*u*_ElO{lqT;#PzE@50DL*`)^kz=wlusOnIN3@5tn_gsH z2VYlz(k|9_I-J7W0|(4+b+|ZT&-H~2n_~>hN-^NXXa7}d;QaVrZ8DFYE6LF-fJY4@ zaQ{8D8=v;q{-j}=?tJ#4YGk-9Utpj<9^FKj^}^dJc(KNDvqQFhpU?CC>&h%)5((gw zOBctC+Ruph88YwE0XKwar)^uMGqPqMC4WYhkG{|s$&Svx5I5xlMR$hi%%UebPwfzsYx=++e73Tv<|vT?P5@Z|s#LS~Tj_u|_rzcCi{jY3ds7wA zJ8XR|Ox`!pm*%iyd_N!$e;14XTr~Zxa}#iRx{^Ezf#-bbkO+1PldEGOPCrjAKd^+- zHhJls6x{<;Wx(tW`+KhEc_~uzjs#!ano6s_^3@na-WE12=EalUfYF=m>Pa96$-}2} X7;-Zk*B`ohbhoWo-401f;RXK(?Rkf` literal 0 HcmV?d00001 diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/columns.txt b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/columns.txt new file mode 100644 index 00000000000..0d87e8d0e8d --- /dev/null +++ b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/columns.txt @@ -0,0 +1,183 @@ +columns format version: 1 +181 columns: +`CounterID` UInt32 +`StartDate` Date +`Sign` Int8 +`IsNew` UInt8 +`VisitID` UInt64 +`UserID` UInt64 +`StartTime` DateTime +`Duration` UInt32 +`UTCStartTime` DateTime +`PageViews` Int32 +`Hits` Int32 +`IsBounce` UInt8 +`Referer` String +`StartURL` String +`RefererDomain` String +`StartURLDomain` String +`EndURL` String +`LinkURL` String +`IsDownload` UInt8 +`TraficSourceID` Int8 +`SearchEngineID` UInt16 +`SearchPhrase` String +`AdvEngineID` UInt8 +`PlaceID` Int32 +`RefererCategories` Array(UInt16) +`URLCategories` Array(UInt16) +`URLRegions` Array(UInt32) +`RefererRegions` Array(UInt32) +`IsYandex` UInt8 +`GoalReachesDepth` Int32 +`GoalReachesURL` Int32 +`GoalReachesAny` Int32 +`SocialSourceNetworkID` UInt8 +`SocialSourcePage` String +`MobilePhoneModel` String +`ClientEventTime` DateTime +`RegionID` UInt32 +`ClientIP` UInt32 +`ClientIP6` FixedString(16) +`RemoteIP` UInt32 +`RemoteIP6` FixedString(16) +`IPNetworkID` UInt32 +`SilverlightVersion3` UInt32 +`CodeVersion` UInt32 +`ResolutionWidth` UInt16 +`ResolutionHeight` UInt16 +`UserAgentMajor` UInt16 +`UserAgentMinor` UInt16 +`WindowClientWidth` UInt16 +`WindowClientHeight` UInt16 +`SilverlightVersion2` UInt8 +`SilverlightVersion4` UInt16 +`FlashVersion3` UInt16 +`FlashVersion4` UInt16 +`ClientTimeZone` Int16 +`OS` UInt8 +`UserAgent` UInt8 +`ResolutionDepth` UInt8 +`FlashMajor` UInt8 +`FlashMinor` UInt8 +`NetMajor` UInt8 +`NetMinor` UInt8 +`MobilePhone` UInt8 +`SilverlightVersion1` UInt8 +`Age` UInt8 +`Sex` UInt8 +`Income` UInt8 +`JavaEnable` UInt8 +`CookieEnable` UInt8 +`JavascriptEnable` UInt8 +`IsMobile` UInt8 +`BrowserLanguage` UInt16 +`BrowserCountry` UInt16 +`Interests` UInt16 +`Robotness` UInt8 +`GeneralInterests` Array(UInt16) +`Params` Array(String) +`Goals.ID` Array(UInt32) +`Goals.Serial` Array(UInt32) +`Goals.EventTime` Array(DateTime) +`Goals.Price` Array(Int64) +`Goals.OrderID` Array(String) +`Goals.CurrencyID` Array(UInt32) +`WatchIDs` Array(UInt64) +`ParamSumPrice` Int64 +`ParamCurrency` FixedString(3) +`ParamCurrencyID` UInt16 +`ClickLogID` UInt64 +`ClickEventID` Int32 +`ClickGoodEvent` Int32 +`ClickEventTime` DateTime +`ClickPriorityID` Int32 +`ClickPhraseID` Int32 +`ClickPageID` Int32 +`ClickPlaceID` Int32 +`ClickTypeID` Int32 +`ClickResourceID` Int32 +`ClickCost` UInt32 +`ClickClientIP` UInt32 +`ClickDomainID` UInt32 +`ClickURL` String +`ClickAttempt` UInt8 +`ClickOrderID` UInt32 +`ClickBannerID` UInt32 +`ClickMarketCategoryID` UInt32 +`ClickMarketPP` UInt32 +`ClickMarketCategoryName` String +`ClickMarketPPName` String +`ClickAWAPSCampaignName` String +`ClickPageName` String +`ClickTargetType` UInt16 +`ClickTargetPhraseID` UInt64 +`ClickContextType` UInt8 +`ClickSelectType` Int8 +`ClickOptions` String +`ClickGroupBannerID` Int32 +`OpenstatServiceName` String +`OpenstatCampaignID` String +`OpenstatAdID` String +`OpenstatSourceID` String +`UTMSource` String +`UTMMedium` String +`UTMCampaign` String +`UTMContent` String +`UTMTerm` String +`FromTag` String +`HasGCLID` UInt8 +`FirstVisit` DateTime +`PredLastVisit` Date +`LastVisit` Date +`TotalVisits` UInt32 +`TraficSource.ID` Array(Int8) +`TraficSource.SearchEngineID` Array(UInt16) +`TraficSource.AdvEngineID` Array(UInt8) +`TraficSource.PlaceID` Array(UInt16) +`TraficSource.SocialSourceNetworkID` Array(UInt8) +`TraficSource.Domain` Array(String) +`TraficSource.SearchPhrase` Array(String) +`TraficSource.SocialSourcePage` Array(String) +`Attendance` FixedString(16) +`CLID` UInt32 +`YCLID` UInt64 +`NormalizedRefererHash` UInt64 +`SearchPhraseHash` UInt64 +`RefererDomainHash` UInt64 +`NormalizedStartURLHash` UInt64 +`StartURLDomainHash` UInt64 +`NormalizedEndURLHash` UInt64 +`TopLevelDomain` UInt64 +`URLScheme` UInt64 +`OpenstatServiceNameHash` UInt64 +`OpenstatCampaignIDHash` UInt64 +`OpenstatAdIDHash` UInt64 +`OpenstatSourceIDHash` UInt64 +`UTMSourceHash` UInt64 +`UTMMediumHash` UInt64 +`UTMCampaignHash` UInt64 +`UTMContentHash` UInt64 +`UTMTermHash` UInt64 +`FromHash` UInt64 +`WebVisorEnabled` UInt8 +`WebVisorActivity` UInt32 +`ParsedParams.Key1` Array(String) +`ParsedParams.Key2` Array(String) +`ParsedParams.Key3` Array(String) +`ParsedParams.Key4` Array(String) +`ParsedParams.Key5` Array(String) +`ParsedParams.ValueDouble` Array(Float64) +`Market.Type` Array(UInt8) +`Market.GoalID` Array(UInt32) +`Market.OrderID` Array(String) +`Market.OrderPrice` Array(Int64) +`Market.PP` Array(UInt32) +`Market.DirectPlaceID` Array(UInt32) +`Market.DirectOrderID` Array(UInt32) +`Market.DirectBannerID` Array(UInt32) +`Market.GoodID` Array(String) +`Market.GoodName` Array(String) +`Market.GoodQuantity` Array(Int32) +`Market.GoodPrice` Array(Int64) +`IslandID` FixedString(16) diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/count.txt b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/count.txt new file mode 100644 index 00000000000..91d679bb3ab --- /dev/null +++ b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/count.txt @@ -0,0 +1 @@ +485007 \ No newline at end of file diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/default_compression_codec.txt b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/minmax_StartDate.idx b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/minmax_StartDate.idx new file mode 100644 index 00000000000..0cb239d932a --- /dev/null +++ b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/minmax_StartDate.idx @@ -0,0 +1 @@ +?? \ No newline at end of file diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/partition.dat b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..9c176950d3f897f5cc9714d7011107a21e47a229 GIT binary patch literal 4 Lcmdlj#LNHy11td1 literal 0 HcmV?d00001 diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/primary.idx b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/primary.idx new file mode 100644 index 0000000000000000000000000000000000000000..6765990e38d151d57b7f18ee97cef776d10afa02 GIT binary patch literal 1314 zcmWlWeLT|%9LIkfbtfq=jO5`;VIDKrgED^QVO-ecan`Y3+vPFyba~p2$@RqD@@i2{ zj>>tskZy@YSmSc#p(3L^OkGSfgc6qf_V4HQ{k%V)@AqqiAc!t{Fjmofqgz$l&^?9GGcP7X*~wPp&bP(`jJIVv)0UyaSh*M zkuB(XdTF)(i?wCF@H^}&##iOQF2_o~-?g#M!Im&&1`Dn|yKj^p)NMXm>R#*m@vsVz zZPu-jeCFLQ^*t=4wO&zy^zU)wnbq%T1!(hoy2-Dqh&DPQy4R{(Jf|lS`+48Rtq1Z% z+p6j7Aj-yjDKgiGgKD6p7;jXsEbNaG5W+dm#7&^^dvi-JGbrjCHOfQztD2DYU%(d! z_km{*?D<8$u8rD=7HVi!`$;+)YUcXREOR_e7t*5_Sy7x$jSSCB9R@B@AJIWIMzef< z3Uw7OG_zz{t^v53DhRdu7Ts9UVjjbIt1;~5oEx$&ZnkY`XEKg6Q1=~l9;uuQSab-U z#&&&*lslThj3&V`5YsPdF+2MuVBabpIL9b?mi=3wdnT~g2Ro7Np>s~>|6Bs(AUElxC6TdRbjifCdaVjLpWd!AZw5AV&PY5$@T}XA z^(4s1ON}ZHvdGCGNg9RVBH+)R8dqiZA=7xW-8B*1Fucm!@4hc{>u-malLxhWAZu@w z>3}+RKS%!N^eBM4i}qK|Q#IO!>bB`~e)>_iKo*9*A?|!p zXVNE9x)4w7!RVl$OP|C#G)NWF;ge|*8blx;7~?4tsMQkBBDQZviaU_!V*cXFjvM8x z)Y4c6VHDsXJ%ht38|aCSp+a{%E)`~wq?dU2+D(Mbt!XU3xiIML(%*IN=>qKrH$H-# z+{DD-(9xe%Vkx|(>6@RWjC?c}vW707wIzx0JeK%rQn5n{C_33M>P`}RWBddyB&^9P zAYW=9S!vp$h>sXJkV(z`9n#0rXCkeFy>DF)%Am-9xB^-A7uuAI`$;vt9$XA9rx^CC z^v>v3dOAO+mFLBZLrTEKWf^;w`U`FDdE5%?W0!)WUE{P=rKmK}F40f7XsrU5@xd(f zyy@1>lcGKTq?J3M9P+zky~BBxX~_A*+b+EUvTDWzf`uoxMQ-jhNn-v(n8^5p;em`cqBLGPl2ap)i&2)(v|-Qn7p;U%sQbWzw7yEO`m8-r)J^xyXs) zEPHDF#dT6}c_cgvT|2Jy4E0w{MMkC|i|L@K-t{5Ec11Xp*e;)ipUxMUZo|{*nv>t~ zJ*=%i!xs75-@o78c!U$0_#yQ&bs0F+Y{rsMi*sH!>>AObT!YI?zVnalJ{)GSWU3eb EKZSBM)c^nh literal 0 HcmV?d00001 diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/format_version.txt b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/format_version.txt new file mode 100644 index 00000000000..56a6051ca2b --- /dev/null +++ b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/format_version.txt @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/checksums.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..1200b25bd52ed0332f4e976e5e109661007e72bb GIT binary patch literal 262 zcmV+h0r~!8Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$axV}lqE!_QlD*Y>FAiIEP1 z+5i9m&;S4c@jnI$V{dhCbS`vwbOV+!t186UB-eP+=laeWci#XAWMOn+E@EkJtA+vp zC@FDdb-WItZml__r5>08+N>Jd_3*EI_1(J_Im&g zaA9(EX>@6CZZ2eDbOiRJGt`jAL&ph4JQlLni7WsMaB^vFVRCscX=Hc|qQj~+&$gS+ M9BY!1)str&05BDACIA2c literal 0 HcmV?d00001 diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/columns.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/columns.txt new file mode 100644 index 00000000000..4835061786d --- /dev/null +++ b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/columns.txt @@ -0,0 +1,259 @@ +columns format version: 1 +257 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`milliseconds` UInt64 +`ProfileEvent_Query` UInt64 +`ProfileEvent_SelectQuery` UInt64 +`ProfileEvent_InsertQuery` UInt64 +`ProfileEvent_FailedQuery` UInt64 +`ProfileEvent_FailedSelectQuery` UInt64 +`ProfileEvent_FailedInsertQuery` UInt64 +`ProfileEvent_QueryTimeMicroseconds` UInt64 +`ProfileEvent_SelectQueryTimeMicroseconds` UInt64 +`ProfileEvent_InsertQueryTimeMicroseconds` UInt64 +`ProfileEvent_FileOpen` UInt64 +`ProfileEvent_Seek` UInt64 +`ProfileEvent_ReadBufferFromFileDescriptorRead` UInt64 +`ProfileEvent_ReadBufferFromFileDescriptorReadFailed` UInt64 +`ProfileEvent_ReadBufferFromFileDescriptorReadBytes` UInt64 +`ProfileEvent_WriteBufferFromFileDescriptorWrite` UInt64 +`ProfileEvent_WriteBufferFromFileDescriptorWriteFailed` UInt64 +`ProfileEvent_WriteBufferFromFileDescriptorWriteBytes` UInt64 +`ProfileEvent_ReadBufferAIORead` UInt64 +`ProfileEvent_ReadBufferAIOReadBytes` UInt64 +`ProfileEvent_WriteBufferAIOWrite` UInt64 +`ProfileEvent_WriteBufferAIOWriteBytes` UInt64 +`ProfileEvent_ReadCompressedBytes` UInt64 +`ProfileEvent_CompressedReadBufferBlocks` UInt64 +`ProfileEvent_CompressedReadBufferBytes` UInt64 +`ProfileEvent_UncompressedCacheHits` UInt64 +`ProfileEvent_UncompressedCacheMisses` UInt64 +`ProfileEvent_UncompressedCacheWeightLost` UInt64 +`ProfileEvent_IOBufferAllocs` UInt64 +`ProfileEvent_IOBufferAllocBytes` UInt64 +`ProfileEvent_ArenaAllocChunks` UInt64 +`ProfileEvent_ArenaAllocBytes` UInt64 +`ProfileEvent_FunctionExecute` UInt64 +`ProfileEvent_TableFunctionExecute` UInt64 +`ProfileEvent_MarkCacheHits` UInt64 +`ProfileEvent_MarkCacheMisses` UInt64 +`ProfileEvent_CreatedReadBufferOrdinary` UInt64 +`ProfileEvent_CreatedReadBufferAIO` UInt64 +`ProfileEvent_CreatedReadBufferAIOFailed` UInt64 +`ProfileEvent_CreatedReadBufferMMap` UInt64 +`ProfileEvent_CreatedReadBufferMMapFailed` UInt64 +`ProfileEvent_CreatedWriteBufferOrdinary` UInt64 +`ProfileEvent_CreatedWriteBufferAIO` UInt64 +`ProfileEvent_CreatedWriteBufferAIOFailed` UInt64 +`ProfileEvent_DiskReadElapsedMicroseconds` UInt64 +`ProfileEvent_DiskWriteElapsedMicroseconds` UInt64 +`ProfileEvent_NetworkReceiveElapsedMicroseconds` UInt64 +`ProfileEvent_NetworkSendElapsedMicroseconds` UInt64 +`ProfileEvent_ThrottlerSleepMicroseconds` UInt64 +`ProfileEvent_QueryMaskingRulesMatch` UInt64 +`ProfileEvent_ReplicatedPartFetches` UInt64 +`ProfileEvent_ReplicatedPartFailedFetches` UInt64 +`ProfileEvent_ObsoleteReplicatedParts` UInt64 +`ProfileEvent_ReplicatedPartMerges` UInt64 +`ProfileEvent_ReplicatedPartFetchesOfMerged` UInt64 +`ProfileEvent_ReplicatedPartMutations` UInt64 +`ProfileEvent_ReplicatedPartChecks` UInt64 +`ProfileEvent_ReplicatedPartChecksFailed` UInt64 +`ProfileEvent_ReplicatedDataLoss` UInt64 +`ProfileEvent_InsertedRows` UInt64 +`ProfileEvent_InsertedBytes` UInt64 +`ProfileEvent_DelayedInserts` UInt64 +`ProfileEvent_RejectedInserts` UInt64 +`ProfileEvent_DelayedInsertsMilliseconds` UInt64 +`ProfileEvent_DuplicatedInsertedBlocks` UInt64 +`ProfileEvent_ZooKeeperInit` UInt64 +`ProfileEvent_ZooKeeperTransactions` UInt64 +`ProfileEvent_ZooKeeperList` UInt64 +`ProfileEvent_ZooKeeperCreate` UInt64 +`ProfileEvent_ZooKeeperRemove` UInt64 +`ProfileEvent_ZooKeeperExists` UInt64 +`ProfileEvent_ZooKeeperGet` UInt64 +`ProfileEvent_ZooKeeperSet` UInt64 +`ProfileEvent_ZooKeeperMulti` UInt64 +`ProfileEvent_ZooKeeperCheck` UInt64 +`ProfileEvent_ZooKeeperClose` UInt64 +`ProfileEvent_ZooKeeperWatchResponse` UInt64 +`ProfileEvent_ZooKeeperUserExceptions` UInt64 +`ProfileEvent_ZooKeeperHardwareExceptions` UInt64 +`ProfileEvent_ZooKeeperOtherExceptions` UInt64 +`ProfileEvent_ZooKeeperWaitMicroseconds` UInt64 +`ProfileEvent_ZooKeeperBytesSent` UInt64 +`ProfileEvent_ZooKeeperBytesReceived` UInt64 +`ProfileEvent_DistributedConnectionFailTry` UInt64 +`ProfileEvent_DistributedConnectionMissingTable` UInt64 +`ProfileEvent_DistributedConnectionStaleReplica` UInt64 +`ProfileEvent_DistributedConnectionFailAtAll` UInt64 +`ProfileEvent_CompileAttempt` UInt64 +`ProfileEvent_CompileSuccess` UInt64 +`ProfileEvent_CompileFunction` UInt64 +`ProfileEvent_CompiledFunctionExecute` UInt64 +`ProfileEvent_CompileExpressionsMicroseconds` UInt64 +`ProfileEvent_CompileExpressionsBytes` UInt64 +`ProfileEvent_ExternalSortWritePart` UInt64 +`ProfileEvent_ExternalSortMerge` UInt64 +`ProfileEvent_ExternalAggregationWritePart` UInt64 +`ProfileEvent_ExternalAggregationMerge` UInt64 +`ProfileEvent_ExternalAggregationCompressedBytes` UInt64 +`ProfileEvent_ExternalAggregationUncompressedBytes` UInt64 +`ProfileEvent_SlowRead` UInt64 +`ProfileEvent_ReadBackoff` UInt64 +`ProfileEvent_ReplicaPartialShutdown` UInt64 +`ProfileEvent_SelectedParts` UInt64 +`ProfileEvent_SelectedRanges` UInt64 +`ProfileEvent_SelectedMarks` UInt64 +`ProfileEvent_SelectedRows` UInt64 +`ProfileEvent_SelectedBytes` UInt64 +`ProfileEvent_Merge` UInt64 +`ProfileEvent_MergedRows` UInt64 +`ProfileEvent_MergedUncompressedBytes` UInt64 +`ProfileEvent_MergesTimeMilliseconds` UInt64 +`ProfileEvent_MergeTreeDataWriterRows` UInt64 +`ProfileEvent_MergeTreeDataWriterUncompressedBytes` UInt64 +`ProfileEvent_MergeTreeDataWriterCompressedBytes` UInt64 +`ProfileEvent_MergeTreeDataWriterBlocks` UInt64 +`ProfileEvent_MergeTreeDataWriterBlocksAlreadySorted` UInt64 +`ProfileEvent_CannotRemoveEphemeralNode` UInt64 +`ProfileEvent_RegexpCreated` UInt64 +`ProfileEvent_ContextLock` UInt64 +`ProfileEvent_StorageBufferFlush` UInt64 +`ProfileEvent_StorageBufferErrorOnFlush` UInt64 +`ProfileEvent_StorageBufferPassedAllMinThresholds` UInt64 +`ProfileEvent_StorageBufferPassedTimeMaxThreshold` UInt64 +`ProfileEvent_StorageBufferPassedRowsMaxThreshold` UInt64 +`ProfileEvent_StorageBufferPassedBytesMaxThreshold` UInt64 +`ProfileEvent_DictCacheKeysRequested` UInt64 +`ProfileEvent_DictCacheKeysRequestedMiss` UInt64 +`ProfileEvent_DictCacheKeysRequestedFound` UInt64 +`ProfileEvent_DictCacheKeysExpired` UInt64 +`ProfileEvent_DictCacheKeysNotFound` UInt64 +`ProfileEvent_DictCacheKeysHit` UInt64 +`ProfileEvent_DictCacheRequestTimeNs` UInt64 +`ProfileEvent_DictCacheRequests` UInt64 +`ProfileEvent_DictCacheLockWriteNs` UInt64 +`ProfileEvent_DictCacheLockReadNs` UInt64 +`ProfileEvent_DistributedSyncInsertionTimeoutExceeded` UInt64 +`ProfileEvent_DataAfterMergeDiffersFromReplica` UInt64 +`ProfileEvent_DataAfterMutationDiffersFromReplica` UInt64 +`ProfileEvent_PolygonsAddedToPool` UInt64 +`ProfileEvent_PolygonsInPoolAllocatedBytes` UInt64 +`ProfileEvent_RWLockAcquiredReadLocks` UInt64 +`ProfileEvent_RWLockAcquiredWriteLocks` UInt64 +`ProfileEvent_RWLockReadersWaitMilliseconds` UInt64 +`ProfileEvent_RWLockWritersWaitMilliseconds` UInt64 +`ProfileEvent_DNSError` UInt64 +`ProfileEvent_RealTimeMicroseconds` UInt64 +`ProfileEvent_UserTimeMicroseconds` UInt64 +`ProfileEvent_SystemTimeMicroseconds` UInt64 +`ProfileEvent_SoftPageFaults` UInt64 +`ProfileEvent_HardPageFaults` UInt64 +`ProfileEvent_VoluntaryContextSwitches` UInt64 +`ProfileEvent_InvoluntaryContextSwitches` UInt64 +`ProfileEvent_OSIOWaitMicroseconds` UInt64 +`ProfileEvent_OSCPUWaitMicroseconds` UInt64 +`ProfileEvent_OSCPUVirtualTimeMicroseconds` UInt64 +`ProfileEvent_OSReadBytes` UInt64 +`ProfileEvent_OSWriteBytes` UInt64 +`ProfileEvent_OSReadChars` UInt64 +`ProfileEvent_OSWriteChars` UInt64 +`ProfileEvent_PerfCpuCycles` UInt64 +`ProfileEvent_PerfInstructions` UInt64 +`ProfileEvent_PerfCacheReferences` UInt64 +`ProfileEvent_PerfCacheMisses` UInt64 +`ProfileEvent_PerfBranchInstructions` UInt64 +`ProfileEvent_PerfBranchMisses` UInt64 +`ProfileEvent_PerfBusCycles` UInt64 +`ProfileEvent_PerfStalledCyclesFrontend` UInt64 +`ProfileEvent_PerfStalledCyclesBackend` UInt64 +`ProfileEvent_PerfRefCpuCycles` UInt64 +`ProfileEvent_PerfCpuClock` UInt64 +`ProfileEvent_PerfTaskClock` UInt64 +`ProfileEvent_PerfContextSwitches` UInt64 +`ProfileEvent_PerfCpuMigrations` UInt64 +`ProfileEvent_PerfAlignmentFaults` UInt64 +`ProfileEvent_PerfEmulationFaults` UInt64 +`ProfileEvent_PerfMinEnabledTime` UInt64 +`ProfileEvent_PerfMinEnabledRunningTime` UInt64 +`ProfileEvent_PerfDataTLBReferences` UInt64 +`ProfileEvent_PerfDataTLBMisses` UInt64 +`ProfileEvent_PerfInstructionTLBReferences` UInt64 +`ProfileEvent_PerfInstructionTLBMisses` UInt64 +`ProfileEvent_PerfLocalMemoryReferences` UInt64 +`ProfileEvent_PerfLocalMemoryMisses` UInt64 +`ProfileEvent_CreatedHTTPConnections` UInt64 +`ProfileEvent_CannotWriteToWriteBufferDiscard` UInt64 +`ProfileEvent_QueryProfilerSignalOverruns` UInt64 +`ProfileEvent_CreatedLogEntryForMerge` UInt64 +`ProfileEvent_NotCreatedLogEntryForMerge` UInt64 +`ProfileEvent_CreatedLogEntryForMutation` UInt64 +`ProfileEvent_NotCreatedLogEntryForMutation` UInt64 +`ProfileEvent_S3ReadMicroseconds` UInt64 +`ProfileEvent_S3ReadBytes` UInt64 +`ProfileEvent_S3ReadRequestsCount` UInt64 +`ProfileEvent_S3ReadRequestsErrors` UInt64 +`ProfileEvent_S3ReadRequestsThrottling` UInt64 +`ProfileEvent_S3ReadRequestsRedirects` UInt64 +`ProfileEvent_S3WriteMicroseconds` UInt64 +`ProfileEvent_S3WriteBytes` UInt64 +`ProfileEvent_S3WriteRequestsCount` UInt64 +`ProfileEvent_S3WriteRequestsErrors` UInt64 +`ProfileEvent_S3WriteRequestsThrottling` UInt64 +`ProfileEvent_S3WriteRequestsRedirects` UInt64 +`ProfileEvent_QueryMemoryLimitExceeded` UInt64 +`CurrentMetric_Query` Int64 +`CurrentMetric_Merge` Int64 +`CurrentMetric_PartMutation` Int64 +`CurrentMetric_ReplicatedFetch` Int64 +`CurrentMetric_ReplicatedSend` Int64 +`CurrentMetric_ReplicatedChecks` Int64 +`CurrentMetric_BackgroundPoolTask` Int64 +`CurrentMetric_BackgroundMovePoolTask` Int64 +`CurrentMetric_BackgroundSchedulePoolTask` Int64 +`CurrentMetric_BackgroundBufferFlushSchedulePoolTask` Int64 +`CurrentMetric_BackgroundDistributedSchedulePoolTask` Int64 +`CurrentMetric_BackgroundMessageBrokerSchedulePoolTask` Int64 +`CurrentMetric_CacheDictionaryUpdateQueueBatches` Int64 +`CurrentMetric_CacheDictionaryUpdateQueueKeys` Int64 +`CurrentMetric_DiskSpaceReservedForMerge` Int64 +`CurrentMetric_DistributedSend` Int64 +`CurrentMetric_QueryPreempted` Int64 +`CurrentMetric_TCPConnection` Int64 +`CurrentMetric_MySQLConnection` Int64 +`CurrentMetric_HTTPConnection` Int64 +`CurrentMetric_InterserverConnection` Int64 +`CurrentMetric_PostgreSQLConnection` Int64 +`CurrentMetric_OpenFileForRead` Int64 +`CurrentMetric_OpenFileForWrite` Int64 +`CurrentMetric_Read` Int64 +`CurrentMetric_Write` Int64 +`CurrentMetric_SendScalars` Int64 +`CurrentMetric_SendExternalTables` Int64 +`CurrentMetric_QueryThread` Int64 +`CurrentMetric_ReadonlyReplica` Int64 +`CurrentMetric_MemoryTracking` Int64 +`CurrentMetric_EphemeralNode` Int64 +`CurrentMetric_ZooKeeperSession` Int64 +`CurrentMetric_ZooKeeperWatch` Int64 +`CurrentMetric_ZooKeeperRequest` Int64 +`CurrentMetric_DelayedInserts` Int64 +`CurrentMetric_ContextLockWait` Int64 +`CurrentMetric_StorageBufferRows` Int64 +`CurrentMetric_StorageBufferBytes` Int64 +`CurrentMetric_DictCacheRequests` Int64 +`CurrentMetric_Revision` Int64 +`CurrentMetric_VersionInteger` Int64 +`CurrentMetric_RWLockWaitingReaders` Int64 +`CurrentMetric_RWLockWaitingWriters` Int64 +`CurrentMetric_RWLockActiveReaders` Int64 +`CurrentMetric_RWLockActiveWriters` Int64 +`CurrentMetric_GlobalThread` Int64 +`CurrentMetric_GlobalThreadActive` Int64 +`CurrentMetric_LocalThread` Int64 +`CurrentMetric_LocalThreadActive` Int64 +`CurrentMetric_DistributedFilesToInsert` Int64 diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/count.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/count.txt new file mode 100644 index 00000000000..1e3852840b5 --- /dev/null +++ b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/count.txt @@ -0,0 +1 @@ +211 \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/default_compression_codec.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/minmax_event_date.idx b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/minmax_event_date.idx new file mode 100644 index 00000000000..4b6c49f8d50 --- /dev/null +++ b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/partition.dat b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..b9ea5569e3a3b34e3a1b6a1c4cbcb46f5e2d3fac GIT binary patch literal 4 Lcmb0VWo7^X0I>iw literal 0 HcmV?d00001 diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/primary.idx b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/primary.idx new file mode 100644 index 00000000000..714ba773856 --- /dev/null +++ b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/primary.idx @@ -0,0 +1 @@ +H-_H._ \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202012_29_2358_466/checksums.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202012_29_2358_466/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..8031891da7f5bd9e7a744c4e46bca92f86a61006 GIT binary patch literal 11371 zcmYLPWk8f!7ryt6iFe{6BOODTASt1MgoHtZf{DX0z`!trGlPJLEP`Mwirs;Yxng&& z3brDyg|)^?n1I4}2Xw!GZ{9lRp65BwxzaS5G((e}sj*5?sWT;7t0^+IMy^s0wi?K6 zW%(_Ax9Uw%Cs*1leY>)FhMxlff0qK#8BQ+=2hf}xqKRzfe1@63$QiXG9$%}iEg9`GF^A|RoKKy+X18a0Xvu(BUq@n9ZMMb6< zY+Oa67{G?*;3s+CVc^~M{y{EFG(kPZ$}tU~e>tGsSn3eDLZ*pRt$<(=Ko40WVFN(X zfP@QyU?9oA2U$ZvWUNe?TndbMD+$hS78nM)W+>*vv%y(uvP_v;qKHr>mqP@hi1(CW zLIBFcOJJD?_J=A|NfJd|np!6L1e%Gt^z_R;y?6X=vSH1r4j1u?0bHUTR!QDpda}aK zn%|2%vtaC`gQoyD7{DCRPALCY@2;|c(d*dzr76Sh_8$Nj2#za=4#1Y1@&z=JGw3m6 zU!*u09qYa77Wdc$w<4EP+}`Z>*TvjP)umhdmafS#a9x$aLL&jdp4CopE?68{BAWf; zyUW7QuV?991_NOEu(-IWp(>?Pc9}+v(1_LPnWR`f16cE% zI4A;;Xf~dn#cbe(P5%?3Y(m5fBsf6;WdoAO&}^3N`4L%E(WvbM9 zacaDu<;v7H*hF1=4?}(P=&k)KTYAnM&yKvgzhNq#LELSv&GElAZXGKew^}56ZEc6V zVE|6U4sEb#jEgfm{49UNi zC_e7300^EFyaM1KnI%)+rH5kYiy1>Tkw-bDDTiDy-wQ7-SbyI9ai8O=CeRM8pG-gR z^z}*4_4V;SlY21I0?^$?C$Vrq=dp2_Yl2(-gQ`?6vtI&tzz&F4%e7;1V{M!XLW|2m zwGTcgZfqVgd0Obo>A0~ef_6|)1%95ry{t!JN!9or1baFlh67lLO}I3(S6d6~a^B3Q zquO~z-q(O0`A3vQtqsrCN-P*Ko8eq^0PR6hlv*as%+kt^vydD+|IYt>2l6~Vuw3eyQFiqqF#^R!=X4cnrz+ms=g+AM5>8ZHyFA@6`(@pIQOY_uIJlo!Eav)F` zHs4Ip2^}SFuU*O)E|;9LU48Q1(tlty(&AnjV5qYIRrD~#YsjU#z7hzbso{4Lj}By!O=KxeN9}o2=^HxT z;^4~{wGZj79oVdBYNuP=@KsCmFHc)|YxVWCJtXu6FctiOOFunL3Gp0napdWfo#lL| z4UsI+AM8^p6?rkTEQMSuQ3Lc9FYj9VmuJ2N0(Stj!9omDLv=8lilggoS}+`-LTpz% zisbMhrmC3GGwQ@{eS$#U?brY8A92?2p9< zG7>`Xe66{kNOs4{A8?7+t+tmt>OLJF)N|3LYsZ)I`0x!4)Oceprqjcgxo=O7Ngp3N za9*z6is^o=UACW0kQd=s9#=SK+uIiFMf>y zhASgp1MJ3L>I)}ItxCbD7d!=Xti%A!v?Kuvg-UuKPy0K)<<$K3n`ck(sXfwq{^*0- zr{BX)VuZPVc0z;OI#1L2gEkWVp>-%=FhX<^;TM7jd~QC&u_(KA_{hD+bzmT%1Ba$% zD_z-5WD9+ca$g%P-FEDS`rFOMgA3);@GqCG(eGE@wSze zz5C&2RBcAA|f&T8`>QISNsnW(^?3f~3qL0K}L$Q4{=&1wfB0}su1VfuM zX&t9x17ft93bQx7UA+1KXDX)RAh1Z4nT2N@pw(XHqcwz)*eV3mIt*)~ME@snPCkHtI$SPQt28pHN|`*HWz72miLsFV9B@Yiv=T#MJR~ARvX#>7 z$c&jwl5+svy{(WjL8xN0rBa!Og2w19AgFF$pnK6Xc@KyQ&Xq~Cx3chkUqK|$F-G1O z(pr_dmavi~59AXENw+|Ss*~45md9R{WqldCp{gTntH&a6zV==X^^=f!zmm3|&Gfx6 zY1W~fy|s-F=>%vbFyR51@ce)W91a@!ta`f1!)?-!8Nyu?9#E(pKx!|yi8@>XE4L0w zpJsjZ#WRcf6K*w?O)JNd?q&Va%Tvr=M{{x)7V1w=B5@!ik4Q2`IWcfoYV2Ls2Qq&<~VpH4Zx~W%> z>N{Q--*SC4)d}VI={-r4MY*3|<>kyi8J+h7^66-Ry|@#G^Y;%cT<17u*0Le5seiwv z?SLeECl{R|@N6N`jYEPYN_WHHG3A3)?jI#Ug6$9JF|dh+6QIp`&&-;xDHD#) zo8jj>bd%-ZHalS_+{c`7PJDGZIO}oawVL{ve_jCrjVD|VTFEAdV7uU4O?yaR`u`1kO_-hxcNSnp03Dg-MH8H0 zNP7e94d%=;Hd+8k;+Kg%NW9X7Q@C7DXo6mW9(YT1d6=D?Ai->NFn}YD2&Rlt5lOMT z0SkEn4rH@Y2{1t^QFX0y_&n&}f6ux7+`)AgQ=p=%~{V#;ZV%vI1 z(%nIvA4&Aku!*Q(g(OQej9{vJG?722*k^6tDA-$0Z0#_3@os%%+(OO|IIm~PoPVc1 zTVGkyZXO-Y;|?UTldy3X*g;ITT(RiR)xB?D4)w`9mJ{X<`~kcMQ3b%8cg0~yOKC{i!1scaCkCuoSaQ(XH6tFYj0#cyw&SC3eE$T1QKWM&Vg4!_YM+s_t-1UCjsI zDt)!g)XAqopRM1QRc;GLFo^+L3?)-&Bc?$gjRt(U5(r|r0w+iva$;5j{kR`I6u}h< z93hGWY_t@w{6a1Y01#ACE|{VjnIjiyWbSqa3ReN{7=YG;-ifhVi9%M!SmGPK?q{Uf z>lFr?*gU25or5#ZVk@cjs%&j=u2d#VmakJGkLuU;KZ zNV?bv+F(S7+>|&Y`E>1oQIr`$agQoYt%02sO*_)f+Es?J& z9^bt@X-?wsF^&c0O9I8GdMAE2z&HTe+sRLy6>%TjV~YW=VQ71=X=_N#xt&qAkk!R@Pyqb!Ut^ZmUW;ynO?`@r@;wD3? zSDorT^}x9;TTFHYK0LYRr5=tmnDVs|F}4S?QH|5gEBp#W4y*7WSPpH5N1m}|8pTc2 z$+eLA2kvq5z5ZbCsEU*CH*Om{9k+Ji`dnVp+tJti1A)iQ^dmdd1YJaS2A*l}e~wjO zbUr@*pJT?GQ%?3NYzJIV0D1f-@{eho23)gn@o#zMI;eN6)x?mu1-L0^^6rj>yX^yM zd*>PDRUS{nW~V;&5pGLKw9og6uYGJA4#)b@=3}Y1*sbJ#1f+&)f6)Y-bRL zLyasS537^vv(zZI<(Omh*7Mfd>_a?mlb*+c9=FiTVxG}>k z`WD3(8{3Y~+haVId9&vT`P90&-Ps56!h zvKc1-ikWUFJ$SWICF^v+;K-8c&CTmpZ)3wYINP?`-79}#ab4@N&GDW#Sp^W!fxub} zICFKux{A98wkz&U@3kQz`7K4e0uaI{4)@XNFg`QOqHDgTZCG3zYS&WCS%_T@a*8kf zyp=oH`em5(>+{-UkKR(<{w*lCH!)NZv=66={1o+9{gpmG17v;C%7n1_KEBqIF?GtB zv)TU2S?f!lGs{LUXrHyl2*(+mD7M2*ZkZK(SLXSd{|w9v7==teTjIw>aX1MN!ocjp zF;rJ#x;QJ>_;5zQhY8Due*6#yg_#rY=Hua1hMn`-eyHWVW`~6H_nBn2H-U97HYOk) zWVGezF)Js3Fx?&{<=?7&@)~#^#YhC8Re<8GhhE-jF7X(<(3~8v@Dhuo{oO(j-Cp

tE*-EM!D=JNt|@+#bc$NKO04cs5&wmf+Gu>W_lI>|ngkKO=) zBKe?+y1fey%C5RKs^DYPnIRirzL<6X|;J~dwz#j+T zCcJ@voL_Y@sQ~1DuGzQiqT4wc$kj)i@UmbK(@~#_w9_8K+X{~d4dU_TIhn?VWy|oK zp9MelYm8emy$2Z>e`KRC2%%%cW_&I(}!ANkVFv8=^n!(`991K?XEigeByruJ zy}wz*<3L}N!DddGG?C9Hid;KwlsO@zr(EWU!)}Ku=iv6@kB48n?02b5Qa;#K8`V&s zPXJ*Az-+u7ZVaol+{)Wld!((^Xx06my)5*fM#!`|Ds@JTOe&L4c`d?oKEYJ%bLi3E z*>eArdHOC9$rp1f@SGPly?lQDX6tE}#FMDJ+v5}o$KGW6|FtJay5jkT z#Iy0`??hxsx9GYCTbFEPnLoZ%ddYn-J8?pw9{X;O{%df18oyQN{2sJ(BS`j0n>%-} zz7zc-9PKcnPA}bj&Ex1n6YjS@|M+qE0k#|FgU4+-qim=SU*xYRVh*r%JX04M4zQWw z2>Ah$^szu|%#9ei0cJWpv|NY1%(=+ep;4no%hlR!iK3BU?in_bE7|E29`1OrPJDaQ zwDE$dwJ#X+6{}T+=Fmq0LICBW; z7y%Zf+F=3lETUj-+@;lxa~7AR3l|lbL`KFAk1Pcku{Y2czD)Cm&` zD@Jcg&sed%dURyVR?j&aK-dmVGjPShgKdz#Yv$69aO(|M%s)ClN%C&Qu2!%hxyQ+U z1-ILh#!d;fx$;f}AkoE9eh`B#LI+U_ip(5Z6pn^-k=~7fyj%modIs5A% z5ff+TcsS#+j!q=f%43^`>|js2le4kxx~*$Z9=8iGI^pK5CEl~1Y-Ak>lPdgj*jZSm z#)PmdY;>}o_fC@y`;pMP>hQsph0EXCWrJ3t;zPlqpcnD)nsV&@1gEvemc7|zHXwe% z&+=KXtr_l!L!-*8{Eq1`E+A zU?KE|oNjeOm``AJc1Da0*VAN5X}A!np$I?ZcV#e!#F|i4gT#1HfY+VVFkdNH1f8?8 zhrSwsNRE-vhoHL!f8>IhY9`?brU0V5PoW7pd1zL4xI9&zX^LKv#um2K6w^g84GMKY z6&Duxk+2XJg6@YT)5!+!Ac@Iu3t=sEr-j|wnlOh92+qt_%z&tccIqA)$?Rw0ZN>gT zV>WEX&=&vE@eW4l3(z`f&2%uoskShRX4^p2k2DslNpAO233m`MC4~48j2uV{eDk}r z`;d*3>t*==G7(q?bwwbIfd*0Ga%He`tM4DlG1*Eb&QY8MsG2qsF6=53;V(#G35AP9 zXcjON_Mmv(+N5wZVI3no!6FD2I>|D`x4(N2q|~~EMj=af<$@t`ATJDOhK51~fhR)f z@(P?(@U24BTnBiN5Y-%nRe{*lL%6dW|N%fYos2bO4u`gQMUMOx?PIAbYwll}H znl;RJOYwZKL9=T92sl?yN+uS{(dvQK|FjKqirW~y7NcbH^qq**@DZ$yN{|gzz36QmOecn zpg&0w<3=#^06Pl6`9N@nV46w1idau@#bxn!5p%)?YsKuAhzTL<3ktW?a7(7A31LN(zT-#1II_ZKS~~>cWA}Krfr?uG7l|y7HwF zGn9*mPoA|Qe`SUmQh+yTBFmhS)Bi;yGY=e%fSw1avMUXeX{2g-mbNIWiM%l3H3E#H7ru-7lg~yKf z_uHM9I*P=&S}%gzF|0ZE^jr2pRB2ySJceI=E}c6;j79(fI^}J_;LC|vKHdQYa*|2p zf{}p#$+C%!!EVk1n0>?S!^|0nr3)>V9w@E%j`hRKZ`B%peQcJ~+T9+9JO8)||C+?@ zsza3kL7SJr3m|J#Oc~d*Gi)^)N8(YnYecHY_@r&Z5?p57GNn3wVA7)WMZUzn=X;l! z^-dv#U6rYJxFctIcHE*B_ZM~S7mS;AYUNW#8Y4@Uf%Z=UcSaOr$w*(J`pn?BN~Om~Xa4*${%bqpNYROePv*%9NWo%qcL zSJu;Uejhr{n)eMb-6h?2$X8RaFap>a?~_Xuv1!@bWL1u`9iY#&2*1L1C3GmmccGy- z(1BULzEhZLnY#(UmJoD@HhvL7c>*AS$T-ph-{8gQ61SLj0$gl^s1Idf=3qbwv7X(n zlwoqkCGeZfNEB^k!n|Ub#enG+*U$vW#WBwkVMHeTnBiiYPV?ZobGc(6IT%<+rchRB z9WusGupcvBetntX!UX`<(C7v!Lg|o8{u0+X6hV%L)X8`pit$+k9@h}Ex^;CXn6Zj= zTg;+G9K^=38O;24{A(294EV(YG!-N9H3~aP*YN{{lX=Qm5;+u~Ix$KzW+Dk<24MbE z_qgz`KkCG|Tt+Y@;8*{;n?Yb5_U#INj+=8pY#{g7Xtzy6gWQt{| zGGTW#(8YZIEdnkVAOq5zvFBpiqAR5Hml8PJMMk78e**EV6o!>Wf~h6YeFHk`2H zn)@KYkT(ca3JjUkEM6%fyH}E5PH@ivWIBVI=#wmxo7qCo;SxWF@Qb* z%J7D>WPDk6smHQe;i0OQTW7zpdU4bPyC_yA3;Y=FpRmU-kryz_F4DaCoC1H(Hs7h) z_17T|wQTn|^z7!gDRa zu2bSgwLm9f3^Qt?o>V~p_`L?RdPtvLdnZu0md^TD$CSTA!%^WCPwmH)9I-y`CgtWORAgr?oAH< z!MZ7s82ElpVW7o__;m-%pJFnG`4K+X(FsG>a97n?^&MRyA929M%fT}Tw@wEZQuqfG=PD+e^|vgNWOcu0LPD7E-=y0=q64$C;a zI;kBxapdAlZi-P2b|);j{qA^Xu;3el#ABX<&B9r3;bXgb?Hh;Gp6k;YZSY!8|5$}8 z=PMUnAkhT2$kY^2RF@7Y2t>REaIh*xYvb8Wd}M833XW#4o4M*FH*D60ygtUG+hO## z`Z3`et1U8NHN2lBo$oG$WZ&g;+lb=JgmU^V1;f=)V0qS45bu_BE*obC*cCQYUI z8E3*oyKplg8i%(7E=!nW>604yB+5&)ZJkTk*Xc!cNzg+PXTLhp` zFb?3K?BFdH3WYZK^4AlRw~8B`r_stXy#a4}d^7Qx^>NO%>ZTV%w~aEs$|@%3TnTQ6 zn{#tF9(#1J>hp~vCcW6I{3;(uTwS579ZoErSo5rZ#w*{qAFL~P*A;wW^y1=dC~Z6{ z9Bs**V18w60t<>6N0CL`LYX>Nfs4Wv-#+2;gf#UTQB6y5-}g(e@b#4 z6kvkeqn8D9kM}z+G0{-q0~W^1N3l>rS$qFvBQY=uZIj8 zds__+uoRD82uq9e`;p9MFvz zJsr>LR^C%Q*uFrx!yH-wupDoiD`|-@NkXI&+WjoT)~5X<^BKH1hrgcRSuOr4}2xgMOZ_}lmFVzAW=>->oJ#gyaTRd{F; zF|umc#hUGtoyJADKfkvP$A_NNK-m&?&z{?sFPgJvh$6^;7@7nO;BHL*{Jz}}o~C-A z;aZpFxa!J+pCAlapabR>CWAL}(QbgqNdQ&>0Yfrcnl6z-e@LG3Lb*o9%&hZlal8iB7qv5aevKziE9)#09 zBPOLk>k}ZlGCtGD;9R+8q@f8umPL`>Ngxx3=)J%xR3;^inUUxh?l6WN+6DZf6Wnto zYFTrbURLc$coU&?3|fYq-_C(gkWhZZm2xgFZo88d>`F&Wox|FD0X7o7nVL8oG*HbOb%@4@OW#j!c0 zGw6m#W)DLTfGT{NOsSC^_Od|sY>L~BddP@PfjK-x64oe|pm^OR0QnIV;n|M~8G_1i zZoWiFOQE0}`A{6B^x}B~FlEwYsgxAk1f{{>wK;39^p-_4N1Lmq%a)vAfi-&buF0x{ zga3?N0fe%C8Njay*pc6ikHZ%I58~OiLe1;%+x%*01RY~yDrFM@ta|7dnNW0kelkv<-s@P}yqg?lCRrP$Aw2wFT z`~&MYQ3qGJfBzo7E60sEzj;Mf%8~>CJpfDqE#$`x!PQC67W|uK8D+h}bC@qMn+kvp zuushF)1#Sudzy9UPTIgyX6}!_FZjpXEswD`>?CYzx{XgCuFFRHM^fJ}<_*f8Ub4<@8sZ0Kzn>fVeNiQTHmCb7Fv;i zUG8LXEw4Jb<500;eH4F9kBU?mgsx0Ns5yQT(=)hGr&DSS(Mp3(m6;x-$jr;pX$v)3 zL511bN?n+`P@kKSnXAkyR4Ze(T6LmApVvJ@rP)VaUpe&4oV2iCldZ+KJ{gu~;wkHy z5j%rCC0kCbHQGo#ek$$Yn}!^ldHK3L&xjFk<1UY%l)Ld^^UKpi#=0k?TEsmPqSC{( z8HEOARxJu3X-Gh75iRc(k2#N`m3qA*M;WBk<|%cXQ73ut;iv=I3V(NwMqD>-Q5|-W zcZoq}WIEh)@XS{?ad{W%3?FksVnD>v0c9=KceerOT0}+#?xT zlO>3@aSTBpwHkx6#1N&;%u80N3|+BI-aY~e$rwaiJ9wiKy6kvCKyYFk0>?_ zky1DU`%<(sT&>Mes1tK_N<}@YqfgN{t)@I4-cG!3ynjW|o}*-TiS;-Rouv4lC@v5TSM&wO+ep*VG`Gu@ zIpY1%HIruYhIAi;ns9R2+r7`9f0@)d{oI@>uP3?;=8^#jfr~Uz!-}@qwO{18GNsqJ zmj^A+6(d(%9g&zA8?4o6lxH~PBbEt!QeY)qhQxW4b*Dlc7YSC@Rnl@@1#%R+@yS3$ zlc}&jyQ#<*Z2uFJLP{zOqWB?*6e3#4a(Ety+l0Z;QehY+X|<@Z5kHPr=4*AOi8=^O zl_sYV%(%MXT^C;;wTv1W>3Ku{bYGgL6b|HGQskm}Ha`YzF7+oQ2#-PH z$CPjk3WzCCYX0T~gY(4?X6opDTtl@evCLZh?_};O&-L`^cXe>&=F`9N`kbxv_uewe zQfkqDQA1*`t$-M|+n-W$Gt?n`waz8DdD zT&zN8h%PiJY*{pnB6q*bM;od6dWO(Ab!#H ze0+J>%JodC$Dd_x8_yi(U>7t#R;N_v7Z{WWxI_iPu<3uUgoWN5+yikvd3E$F%0)qj zZ}a=U_gL75Uk`?5oOlDf*6+qOLtd=5J5cEUF>3hP1s#nJg$rTJfnPTpmT;e{cYe-y z^qWz*1pUi(h)?zuQtlj-NINQZN7y=g6TeOr8M(=QJO1_Ki)h01vJQ?QNvdH4-O6K&+Wg{XNKi$>t#AL)W#}JuoY5{f-FfC7sCD_{H8x5;N z1#}&~l0Lco?&Z_rj#c|TwysrLu5tba`$=l{SU3kw!Z@=_^s2{P7#(fda0ed)~ zB|cf>e%8cds;1&)A8rg6bwxY08g*&BvOulMROk@-4@|O*)8^gJk=WaKzOw+Zf{l=y z0~z`{M;wVrIrxSX!}USwVM>65AE+~P22rhfPq>B8^+sgA$cCImT-=(^ZLNnJ{Ti0YT$`?cMQ*Lm|UG{&-1W6q@mXbnFr4UKL3a5BMpjLxZ#g0cxj4A#QBvllX2 zi$5C9$h)oTs~wX*MJhry=t^h_cjwLOafc)dN#Tm3Exdf(e%Sv^8m&D-d z;yQX6eP_>))f4~vF~ncW56t;@oMQoGqCp|wmml7%mOfo^YL2^4T|+k-vO6w?b-%xO zrjtp(Za3F`TFsmvefdN<-+ho$_l!cpV)q=S?mn3CF!f~Ig&$ni@ke(Kz3dwP$4lE$ zV8Y$Q{mFMlKM#)uCG>!=_Q~mVYjXR6+5QFoHbazqw*E1;zQf>}P@xFF&hSBol)W<_#K?fMVbTj)(w@TlJZ6c z&Z2Lc3g-^hY6mEl1xnp0Gh{&IFOF2$i2`FoQV}Q7!owVn4{}dX>eU#`@5Z!wfjPw~ zM<&Tj%mMOb!l%YO3~8PD!jIKyvjI6ni-2#X2LkI-1gh2A%v-SSdis2aB?qc4qsp|E z6_<7?*(q;vGj(l&)AtQqUr#7{l=^Xm@19!6aKr-0G*iXm5zE!u{Qu5p)E%mwB^rc! zAWl$lZlT7QDxT9Teq9P)n{s3Rjfz)q@7vc0j)qqXv|HbM42^4da_^_f<4m?BwjYfc zx+Q0Sr&oq%=477<^*cXp^=SVNx4NRP-spU=LZi_d;sJh&lyl68Two!0i9_v-AU~dx zyY~Zhbf+B?wLC=SakRkXAYA&0{-5nw%R|s?ZGHjlIM84?BO*&FncN`^C>{X4yt5a= zL7^oDy7{!Zeho#e8R`F%IX6EfIufnQ)M@p~OsytsD%Vo@1ye&Y?+FqNLZk-JG#OJd zVTGE^i^PVVPSev6r!}mIB}Hiy3Ntg6dInpPV~Ao}xrOnMc3Im{d}xU>vv55Zt~-lk zkdZwKztE1_{AHAbGOLG(+C}#WQfr%qb(GDX3g^eH_ml=hN@(uKQ|!h|%unj{(y(nq zH&uTf@cfu-$xfFg-9}L8BL$oZ0cZMw%_7D+ueXAr<^9u|$lz^Tp2oktSU4N<-0fF41}!aGGpdR-52NPB3!fqrAT4mrP4pNilYnr^ zj8MraHPIGpeUa^D#EGy$y#NNE6_t6&Cp2Db+Oe#3PNl&x7LNxj`yawX*Jt%EFfSBI zj#LDXwhM-QqeqRj--x?;cS?S~j{nx+H|8(LEvS<_8UYLo64_@9vVyV{swpE&!Bdg= zZw~2$s3}p}oKTHHS9;ExY@pgx)6nnJI)x>Ei2Rh=3pW_|&4Ufna_4mzbG>Qzz{nU! zidbQ(ykit1fC)6Qz?7g8#e^wT>P=EB<0ZU>jc<|jNE8ztttgOT8R63ucLu`I$cVG# zG!miS2nMo~yROATUIL_ku+%mIrKPx+fMV#fRJRv|NWyXD)Hd}YeP4m&tW6W zdJSTlan+?o@r7p^#fi~Yi;sCNPW*vIoR)KK#x@$!`0|J?cDGz)#JqWnV)oqAHyM1gexSRo&Yt1+t;R)i=Fiok4xa+kxd5S6ha(}(G_`HoVwje@h4 zT6+Ta4n|_K3{21?6hit3hpa?h670b@+JklEar`?Vp>sNfrb@SfYsGG30R4?~(X53?&euM={jcR0gFWt9cMQuz0SI5fWFCUhy_8}hv${OGsqW91 zx3(*%Htf1S4;b5pujfq{@7TO~^S*TtB?)c?-R&?*giXx{)VgaIp$WF5t+k1N)n0j8 zoAStvC+WVP8I^gsikNFhhYJ~AVNkOyG#M;A!N$WjY?)w5TCZSZ5 zInc!&O5nB33T>ewv?Noh%*x)((2$|xViDFu3Q zgi2|qxmzYwI_eJwxE`Opif_(u#I7Ao?u7ik_e-q9tJ}eU`Q`h{_`wJga5MgxvO!vP z)9)OLwT<0$bhpE21TdvkD>DtH$d-IYnBTUdjizXt3%b}91_?G-RT?L;A}G9dx3iX&2}TY)aBkZz&QJ^6poGFcQ$O!)2744g0wsbB5J8(oA6kf zVD2p>(!+$I&R*V0NBAXXo*Pp} z!H_m%f)=mwSBkY$FFv9)m;IlaK+*y7*GLuL>;J@UTfF-}<6LUDNLI{13vv6%m=)IS zJ9!;FJGt6n`ID$M-ob~KT^}*6e4Nb~7|i`}gMR*QRu(sX#fFr5-j3N~(jN>w9yXdi z;nA!fogRsPoZI>N;@7N4jY!ZFVS%WQu8n@vv|*3c>upO?Q|`>(l>XOg<4`BRjSP;n zvw3(pdrT#L@Qy_zYF$?oo+G}x?fW06)(uOW%DLU%YI>zL^%swGgCaAyQZ|}ywV&9~ zn=KXb+!4CN7nO>D9!ptQE=0S)hzeHrd$?2p;xz%4U+Z`ESa#LQ)k{~0jz9e2%l7m65&iwsYfN@UYKpY#LX81Ls|s8Hgkn{LudV1BhMdm@OcI+=>~RzrtHeE~MQiB?@Hn%jM}ZlXzMu%)>6Vh|7Z5V9?;tSKP5@6Qn_1&O`l2adt5duYPC zTjrarUVXW0`$ziYgw;&+D~98VP$*ETPR1gkfwFMm+6MOSWjo}3QH^C+@CK~>^de{d zp#Gm<@JH8GEY1#i#k9Jfcy~uD3oXUz!>^;q+h~#&oqsa1{SxyBg)93X^4K*JC=TN- z*E)>cy8lbFKxrFN8X|*)0M%S0)hy+Um+bf=og5K5ZLG@%8acCYKOhmx>oOU3=UeF3 z3hU^{JulhRE!tnZt@fgiZJIQj#FoKwqS;lQo(ul8dB+)d!7ss(-iHD~iw7j^7yWH% zyXWiTf84I>mV0&9Zl43F=YD_;gdGCahkQ&*lLXWQ;I>FAH;SD``H~$dA}Vv={k8<6_&D%eGM_{xz& z<<8;LsomSdJ+c+3(9D<%aWmcG5J!6Jcnh6-=+3!)JFzG6>JvlQ0$6@($B6ikeX=H< z=_c4R{LQT%%S0pwaqx5qlrugu)5-g{ol`hI-oL){oU2hH;s=Y15`k8_-|z2~$F&Vx zi{@mVd9?Z7*1`jeV6bA%tbxqikcUo3UoIQf>yIjU12QaL2WPQ+eC+DIFPAhP(@)+r zKG7Ky% zi3&m(?1*o9-n_Bi?@5xY?)ikhlQU9RVJy!QU|7}!*S_VKE~S*!NOGp_nK0c{M2&^# zCz^#>%=++(%*r1!;-_`${Z$yjCQ-{KS1V$734V63ksW{Q;(7hq^?FBq0Ga7kJobQ9 z9bF;+cXjCX!;|NBSmbieYeo41i)k?4wfpx^1-{%B@M`=Mb?ON7@=+9$3`Epa^qsoq zGJIi#>3>-^r#f7{v7LuJEAgX&N<*<$mlvCTMKT z@Y_xk09CvKOrct;nD@z^8o50ni#jmVa?MLA9fr_P zSMVO@6LpCR>-@w1%g!HXbJpj~5j5g|ZeObn5cF+Wx_*|lz8k_XrEM7Bg!6io z?WJs|e+`PbZ!UF3mDi_x%g3g!q{4{J9tJ`XN>S2JT{CFe*ia+p7z#m_7wRS zs-h%B7J{=f?D3d{;Mk-=DxINFq5ep**DO9!rsYJK9?*?%1PD1LwUMyD-eM;wUIvL2!;h=F-b z^-s5F%0~)skDr~byP%=WYpMx_AL(9io@{Tu#4e+8M2_wPZG>M?ewuK3+)qPPv5Uv+ zQ{fK1lxyafTE)a9M8-@)0EnCD;{%u9*Sgj&Tzkgj;buSehe=?t6MLhBCkDKw1KB-2 zVm6##+;I~&vR0!MHB!Kaa2x>nEh&diEg|%Y9&_*Psq$+thol@?y7QHS7M;t6sE!{Pa? zNPV#XQo!AvaZ`sjphuW$p#e=8M}6vlwuh?s!RStQ zELN&=u;t0e`Y&b9qodGDc?4R*SfO5cjT#fI7%Fn2B3eFgBFJ-YC6L zA6%OG%iEg0C#qj)h6Xc0-u!PhjJ4qkRZ>{7dWfZ8N74b@_dX$>X0TQCaQn)E< zVf+?UbT@K=$YD`P*jfXy;~6|2yB8D&KXN4t`7((g#c9>oi3d1;KGhu+A*yvwi4{FE zxS%jvm7^PBOTX~vt}OJjc8ztpD9O)H%HQ1 z-bjk+z|j0cbp>C%c7tP>?Qw_ba7n0XBAB0E^6Hqg9$v&C0&x zvf>Lh8esKqVzP%Llgw)A%fJ(~WFiAy88QXgNK6@FtB@yILpjDP4{<4qOa9_oz~%4R zgBYE$1W+rpJO!v{BGN@cDPRDU1>rHOdyX@dAhTMGldgu?Ww# zMZ3L8yu?;`fdi)`Ffw>6i?2LRds`4b=r60v8-!WU9#7uAg_ZBzeLRFT;?8-4Q}6AM z6z!><_=9cy1MRY;T14I^96r(P-m0|A)q6I)IR57Lf%VT!a989Wt_ z3jDGVU+B2}^2-0Np;<*u2OOtCDr^3F+W!~9^PWywEqXts>G)S;{lvjDfIP`B`J~1! zZMVp4*NLqgyX0X?ht4#7FPLP3Dr`w8XguC9}jZD_$e%_V3F@i>kyc zz7Lf{jUy)e``EGYGYtgSS|rY;*)#!yEH^+7c)G0Hon!y5n45U!XSCX9-JHpjH}wM> z2;`sk9UoDibSAZ;?&0CWlWE4#!qCC|NgZe(&}l{Ns^xvDC*hXz9ycI^6S*}ott-sM zkrx_zt^9S2_vy``N&oykdLpbg!YBH~fWP_AXUq!h-fUkd3Mx`v`WYaT=^>f1wedju54{f0RTw)5I*f<`PVV6auL_fnj2*j{{Vjw6TM|&y5mv^K7?~COjAiE_wpE-WsaOCh z(4K%?MLXL-Bg&{52)zW<&NxsD9b>ZZIFhBr67GO!vqq9d&QJne1?_1exk5|H2rQu6 z#3?>Q(YBb`oeY`TQg~b_u)y3_O_~j69TL3ea5<)Y!UGr$W`Bpm1qS$sXuOz=0%X3x z7)xSf`=3Dl%M|&>1jE-Gk_<~WhN3DE+lq{!jtTJP0H6$(>|~0ah}z~iUi3n`cx9GK zr_8+Al~ln@lPM!1FzVt^DG2n?h}RknYNu2(1Lmugv}On+2#Bsx_~Ed z%UHDlf90=Vue~HNVh2cvO}I#Wq+6+zc5$PVe5YuSwRD~g?4SmcCYi-be{b~MP1D*=s@ou z4BQOpQp-@Kue;@W&wR5p<)rtrjIkBYFq9{>&?kf9v5x+dwK*;2)VWvl=jAfZ zz}zeOM~3V#>AXpL!a3vW=$+!$v5a(&cp};>UZIh&qy#+bQU@j*Lm^~r3z1b6$go~5 zCivR9+;U;G?;Ebw?weh_SNu8c(MM282Pe-}mvwcWW%GJ_?~`=JL;IOD`llz|#(Mko zJhNEbZcstGTjYwjYPXmDA_EfC+TsQQIVF)3_KE0=B)^FMKi7O6fHWw>B}G@UgLxN0V+0ncOw0 zPp9LY?k7S$1w)78Zyd)M&E*2}6ll53`3>_#CsB9{7OmS9jz!?hsa@2BF)UnT7)Di z$S8P{2rNM-J>gCVg0Mt948coI_zIVV<`k{NE{+J&*WQ4}7l+ z^URk_7fac9_xrSd@pq?&5E^Q|3|P6QZY&J zg;Nj>5q6LX^6*p27jq7Uz1jG#!r3V<2)=77c|S?NHqpQK<;?B>)Snz2{K;4n8b4zY zm-k>3v3$_HKHb?jA|tUg(wzvJwc+6mG1b?`12~2R5qGYPO~t_WewP;}7%~~cxm-xr zG8V3WtI`=hyuL^hZHw&?S_l9=!FOrWvytIlwkLU(zvP{FuGY91t65pSBoE%k=ATa?OFU5-g6}<6N18!MA`C1f!BEC zM1PePSvL#IW_3wDTT?%1N7*m;JNWoid^d4see-QgH)j{cw0keMcOX3w)}aPGa-Zq> z6z%CzlXgf`g4dC#K}c$t;aP9FaUM11+{4AzmD4v$MsFPF`gwy2Pn=6-2`G9X zVprs68QsrmFv30v4@RHqwdE53u01!)GcM$IJ$_`k9NCOQs6BE?&v!PhV}7ln-@MOS z7W^D_%b58pQ+!tGB@x0`nL~o#6~}xzzGqY4j#Spd%eKMbEbx^E&7UrNq2-JvQuX{i;_@X^&08 zH;19m>_6SNh^BpslDz!<_18*?<0NFBuhQfzO46%QovHET&F-qK66T`y(datM>Z!=; z*>+xD!f7NffG@*TLA*NB9Ufv>1{fJ;aAp9pP!tuhyA|WvEjG5# zXFt2U8;qX{DBm6i-}m1=H&5)d*4k^|%v?ogo~|%o=a8+@=F9aCMGCD>sZn=x=*rgq zqnTf@_WN~A*;aRiE=@wHDSi^iFQibbRjBn*3cXgDnciETnU|y06sog& z7iMQGwBf2kU2Z~Vt|F^YrHIvNREct3Ugr#@`Y$5ZtclyBaTr}!Id|1h7yskfo~( zg?dHSMifNTkdSUfjD0{n=01v2=ydWNMQ^PpPodq6Tye^ueTI^uo zJ_ec6N;ucH2jU8fNiDNW7@l&+qW`$RPN2N!rrnx64;GUqGXFHsV#?8H;iK;F)AkQZ zRJ!y-Qak}QF{6$==SYe^zFSq)^TKKOTXm?TKjPAD7}qk}$g?$eGcOG}bEsX})Uihz|r-v7X-U#^%v$$Jp;4nj?) zA##|!P-Uhb7DKKrNh-*a>lJZ@ib6$ixgIX*mN6FgT?ZpOT!kcy7<-QhWER*@QL+MA z+IuD=Yq}IMHjbg_u|};|jL`SjWacHym3nvR-zM^P-;>3!KOX24gcJ6X_Jif8a1)76 zKA1K5N838zv;X`GZ!@RHsSgt2G0?xDuF`Jd$p6X|)q=6RvKreX2Osn(RH2fWD6%5e zI)zp*z?Py@NECtaC?uJSZJb9TPw2Tn(Znl;7b@MZ5xDQKZNU*Jz^@+8o6M=;0aBQ6+Eq`Md2R^ea902V|LCoq!u~_Jhcd*=m3=VHm>2sDWfJ%yeLZ8}v8tzLO!^dbiU_<=TrjM8 z(KmSCUJ`S|@#31%Hi2O*)GUX9#*Fqc@3DKb~XY93Q z$WhcqK%)^uD`0qb704fA<`<@gL?-G@1fhr)B1Xh=`JRZ^faxeC6NMA0aY03{%;P9U zzD8S;s0AQUs&ndL6(7nkU%55lXVcmqAv9vP{iu2MB`_cDhq7}PS2r8Kj}c*Wzm z9FE_S4@Z99DNad`nLY*?-_b;7j6Swi=yt4LwM28G`he~v@Vxq zMCfE#sK+E0&_Y}C9Rg`5nNn4XF6`gE@UidxgKbj_``5yDGC%jZdec8Xn>wBRw@O@O z?f;k%m_e9899kiqKYdY5*7d_FkEYoMUp~Uc9nkn#twNDspjRB^Q6+4JP5(s_wzkW| zxGUm$@oSmQB$H43Yc1^n*5uRV{)mB+13SvN9oH4cyBiB_;C|`Le5?-k_A7wIqx(b+jbj+dm@|z z6Wi)W(Thv(E&G=4{Z*SDe#k$DhuqN)japR_uP9I{Gv!)D3t(=Sb6cYHGnV)m=4mDn zU5J6Ia{y|uawU<7mck;pQ#moGrw|WEAlR{t zt#u0VE>i0L<{1{MF>3DzkCEe|A6N89t(ju}S32DpJG(W0YZlnMME?V|m-a(+0t~{E5&EORcujEzV@WTwT$XKSvgIWB{6R3C!}RA?sx`ZuB^d zVr^qL9jkVT5O@qwXrB=jBJs#kXz#(oZ6)qY@^Ut`J`{4#zx}s?cb4863GuXc!p9F8 z!#ZrKII||bW8d>>>5LJY8i+L>8+fKFWkOzzV@*Wc2kX%oE`G&DTG(8hNJ;J!y4Z)V zi0J$E%-r{j!?dWLv}zrHx@K35h2r_?RL^g723rncjJr$n%~TY~ojX{g>8DT>D6}I@ zksi?+u1w@gU|9hNBuNY`F4rSLp;KWv>cVMRZ8O5HMkdKi&48gSL{AO435&}4CWzH) zvVrKria?L02ZJh;2dh+?%$qRKD{h zeC|=p^X85bhz0I$A_py-Mqd~@|I53_W75w}nI}M95w~|pZlT(+wf`ko7dYiPpX%wK zld&nCDLQg$v_aPdzBO^G4;iO_%A^L zkoahnGE=M3DKa(ctO}l`=o==3G5-k?4nTA<5MVMUG2w;k%nQ_ptz?*~h-=(F)RGWQ zLSbg6LdU_D^f)3MqZ%+=Xq~kU#fObhWETF#gSML_1{nmV=o{mx$zM(!6j@!xWH;m8 zTcv3d!DL#l9@<`>%6g71O>6MBwOMuTBIg%*R{zCHS|GQU`8pJSsM@dTIEjoxcLriE=zfR%XWGiPE&b*X~YkrX;4Kv~s0&AKV@3IbKo(~fk92v6)TiCbjk0?MnL(MS} zmZO+(xl*-BW@Tu?BiQf=J%_|GQBm>&3v59J4B<@!R5SoPL;pr%)D2N9Y-itTEf(<; z0O(=4wh4)b@GgQh&}J!bI-oWnWY*bCeY2}yxyLtK{cu``4tBP?#c4uy|5)sPQvC2M zx475h@-z1uv4q<^vrYJ?#C}JgO4Kv0a#rQZg)^EqVurs#NRJU~x)Bp#7CaeAGB}0? z3tiZVy=Vf^Hhi8A#DpJ&xPx#V7m}r4eHCMlN59#fc57N<{qxglKRjs@Zc|Y3KSe7S z`7!;FwTW*&6}&(aE(~`z`pF!)b7P6i%(yFy#=0-qQ}OR8WHpiu4~E8@dKDYyvCV?L?S+||S;mk&8vMuT)u#XPoSkmzfhi_yl>W>7N&!-cS6=fI> znutI^GzVWb;V^C_Fj1L58L}>DvBpl&dWOTm_h({Dd*{2D;Tlm~979^^_#`$=B`?tR zA?%)3wafz774bJK+u8l>srFCweD3qMZvfkf^Y;vP-_*ma{LMUDTCR9CvMU2o!iSXc zH#reHqO$no&{-!W*S02-6OAYc;q#czL!e?yONO-}w>^T?Dhb>q!&Mv-iK zupX$SkZZv{GNl8M7%xVWIH9C7=5%V$E=Hz_4I(;&Cxa94Ha?vXb3a2!gAqn+Ii8@X zTH8`jY+>vD50)-L!a<0xMD5ZO^m3JA659qECdPA&z5fd!nuHQ{W^@M+$avRsR%#0M zVIwjXimVL2m3=@ua-y3dN_4H-h8BwL$zQ0QBI~}4j)YkGTS{XQk4}fnV=Dk7jC5>Gtt@O=D={66OlTF{1f7jQh#TofJ7dTH9-SkmJrVLrQfJzkQ4efu z<(o<95O4|B!6OejG!iiQ3>OV@rJ)=P!-+)18V1bn=5vA}i_~cK#~{>p^QFdxX1@vq zzKBY&11k`MEN2X;!}2it#p8TuFyEEV!WPi}bGyF-o;iOSn?D%p80J3xw^;0-?@7^!g?H>cY5V6sWRPZ z)`dOqXN5X%XvKvn$%pk`H6?9c(01+EuQ^FOCFUW%-`db_)}UnO_g(%^gIGIiN61WX zp~EYS>h^k6kCWZE?T{9CU9jH|-+Q-PGs=OAn-l#8ti8FX^Bnwj$moO8jZasf%X(&V zqK{mg6$|D|xV%u6?p;UzAdD77(Ur0#lM`1?cRb!%boV7%h@B#1R$8;~?Q5CxRGDqY*^$z%^9Rh# z+wM5m(d0XH&+~r+MxS~+b4ibm58kDjoypFY(7v!z<6)eo^_l@0Sv}Wx8(Y?Q$!Ff@ zdL;BiSSYR|=i+Y4mshTGzwF_X_GtBxa%)n37W{f4kItFElq!TZ|waIy+rI_)D2@tMOKbWGeKPFpH{TX_4TpckK?u_e5%~9 z`ih%yszn;tP4M-9pZyRX4+$HCjYDX_b1ROB_WM^J#J77bcMfI9v47#7p z{n6uw3%)xPTun)JHJ@X(4Vh~|7W>sQd#o>*oEMx*nmOU^zTcOt`<3d3g8=d^TrpY~ zaV{j&HG8Vw!q3tI_Bf(7D3NC$HZfyTrA7Wx$KzMi;N^rr;^@K~w!^?nGwJ zn}7RF=)=`zXDh?!O+kJKfB_J82v!~TH7Q9FlKW!14D_hM;51m4Y_yoZLa0qh8~EJs zL3Ia~w>vN-_f}E*qy1NFA_oM+#5R3g;$uGOQevl?7xQ)xAKM5|0Fr_2S5Gb-+-|Mh zvu5ql>qkCZ+2MQ=6`C5L4L32hJdh7lqFR^6uN#~{(0T5P*E^-EHNO@>_uXE%UK-G- zS*^O$Nw?PJ%iQH+8iTmF5+-Hr#W9<*^P38Gr6%XgR($Q)Urc)-MEi@uAH5rG5;Eo2 z%Fll_9+}d2TG`%)VT<5p?ss|HKR#cW9j^~Ke!FhVb$9|W6R(50*j4vQzH8TezwIyD z_Bxd1+YQ;+5^grwITIY)=~`7DC|RsKwDRQE1<3=FD&b9yxy1o@Z}mKLd~tT?dE1Ya z=vn9z>5VWfBi?Z4-y5lHC9K*@zC8e|D_Mwq=Cr-5@6)^2(v)n6-8Yg}9(vkM09$Hv$IB-3+uJ?3cDn3>iDW>n@6DZo4ebn0YQM5u|tkLGhD>465fjTHI^O!wXVADn@rjB4PSg- z$(V42eszN20gk^;Jj+Fc`@c{PjhAE`yVdlmcp3UhdUXnyR+-6G9oAZAO!jh%SV9<*+AW5<+5=1}L@qLb>V_VXv4Q#<#t*Y5tt}{3oG#FRIrc z3x9nSBG;sBcg6~F(4`ea=M6{?e)#?!GLR-{4#@z*?xHvtq2*o0Gk|(7kh-=H#vDue z@cM-Fjx6WJoP{%Bez-rX=Dr^n(4%sqHB%cz@s3Z8?fcq@5^&OSN#fm3ng<|N3SPDSOb+Lm`46;SpbJ>BKD z_!`7DghV;K%>00fMZvj}dDYBK^EcZb ztVlE;4L#$B4!N`F*0B#eZC-zvv@C6&r^v_yO?cUj_&YD26nl-BK2_OqOp!>3h;v9T z^{OK`Nljgw@UG4Q%UsS65bzoc-uN})8AIXtcZ+QoANY9R|31dR-Ng8GDrUtzeN2W8%D4XN-xxflw|() zv1TtwfIVn2aHm5KA9*1hBBfuDowPd^7;|yyC<2yaS-b*@O$v2pKPlB>8JrY1>veN7 zKZ2g8_RuXPGAej>JO_`*9tDLVk2=wX0t;ym!Zn)O)DxsuKIx2#5HWsKYQ+o>DJYCm z=4gjm(-({-ThyBI!Te_RWrnWMz{LK&KN34>DPmg`SrZ>sBL)=QXW}UJ;Yy7einN#s z4$Ci8m0|js+r`*7hTX}7UBG|O1qJB961)E-=xEWbPIPJ*TLF3QIZel*9lMCSaav55H4V4ZcAG6pfr8-Q#s%LCge4$#s z*s+T;KT2vTo!MLpfg2d+66$-kjv*ut6sWlPKK#&_Zf9e4rxg^m9 zfvvPJ%QMclC<2T}jjAL^qt*pyWht@}HBd2p=3*)8FEm(-%`V}2F5{0JBGqFNo@ez~@>eDJSq=rhT|R{zd`MJab!X?2bI5h6 zOXH=Kx?VygH}F3Qq;I-Q_XOAt7X7{Q_{+5g6-ocebD#X94 zq@9*uk$#w}t*txVe=QnsOT)mI@c}w|+Ohm?FaHIZS+dXVb5v^&OzX57$hrsmToW_y zS^7(_;JEQ$=MMGz#-XW5hSvihz4Z17>4bL=sg}v&`Zr#z7-SDQjfC#+ZpXr58hE7} zktCO4Q-uhyt*<>0=*o6|^fIAv!|e39`<~BBj|FV(1v7u>;*CURTDi08?2Zg(mt#|^ zVLxH075*ZBUEZ&JyD_xyl+CjTnVopH3y_<(u?C%Yi!4m?zvGtuTqp0B`1^mS7A>6! zgg11Bm2Xy>K)9(Pta}xi7{%cmjGJcmT<&mh+=K_~Z;l%OZeeUKZ}NRMGhUIS7*TN3 z1}rH<^@*h1ZZ%=}n@LMbMnxC6YaPeJ3)k;68E13g(%%#B4mV5Fei(9}HKq$vU7Scu zLWe=ld$2w2F0iz3F{dT!Ll|4k*^^ACAWP8+k=zpOOD%={qUgexhr@{BirA8Pq zK)v`1!Sf~5jIm`q2tkh+YQ*CC1k?z%V4TINq@~CMUnlG(LI;|g8!k^oY|H2a5DO9u z%5LWDQJ!cLVvM{go=t?uz&2(a;0TbF)50q(A8xk9=4>CKXa?dNImOB>QskMSP;gYW z%o>}HTn8^-ZR7ZB)ot&5-Q>{ax2PU}vi>4oz1Fn;(Syq$BQoryv&?W0gcYC)UzW8W zca<}s?$o~3SLH>q%wir}-IX3SlZpOe_{P-!v>AI*$bZz8I<?bd89{I6*Qm(eN-2cMXV-+BUnsA2> zGj6D^4J*8UUxFq)>|NZY@sn~lJZO6L_n@&J$;F6+GQ>LaX zsNmELVRx@#U6V6CO49W%|A0?fMsG?IL`5mM znbtcP5T)B`w{v?$w#b1^awb2re?GWA?XlgFbHWM zEJY(3x)%O4LRczELGVype3?fhuoE1%oMA+ZgmEQ0C|Y$#B3*JVbB@fLKG#qGzn*CYSyb;}cM=4`V4M-13zsGKz5%o8D#Km{MV>LYtt1LL(77v*!eVfQ>x|xn5U%r#$%pnq!1pn~# zW`OK&6OHwtIl}V;RwWIJkA0bP=0tC%0rx)ieZOmUv3TU+m!kY8-qSh^{WKKnC2n49 zBDNnew|i%HAKxIM4C0~>!`kq1hJ>4F3)4zHcou9b22u4#$(vxn6NGblu$C4qwEoz? za$kN=Bwc5V?GRcBtav53(C4*s#KOTA5zZG&x;g!ndH|1emyWr-)6F9?vrWnT>oM(j z@%Zp=VvoUA;WpbUYjt(LF5EHmm>|uBtihp^(Ha}DRVcS73~*Dn`DE*?ii^EpPA;vU zQj#gZ{ui*>2+uA*PE~B~et!HDmxwc0NBH%j>BtOk1_x?;NBj7|UZ%e`cm?+)`;Xs8 z{gE9>3q|q3He8uiw=f~y0zY~TMR+Tc2*Kgza=kNfAZ8Bw(?SaX>;IQ4-&wYM^Z!bE z2MECX%pcrm$BDTB{X4AA6q#2 zCl=se15TGn^}Qr3>cenwv%a_P(qGw1Xa(TXa>9Tyob;$dm0tOdSWqv9Lt08lulU^c z*ds=tyMfT}*q(jW91_jge++TR&uq$vI$kjpR*aoA9NH%oGakrDJ?vJN+X-vnCnU8_ zzSG~1cHnb_Mn<6)d=9D*QHD6u1i}#+DV7Ej4)JQw2KAyd1TuDxl$(hqMpmJTn1g=~ zgJ0Nm1sXMnZ$)$QpW2S2F!PdKoj^ZA&v~{e**Z_TebKai9hUijt*k>*g_Bg!$;hX*2Cf^eubFp+-yZu)(?TWA#eZ{3~O_y|V zuSk1%^-lcnB5FGT$@G<8Uo0<_k)ug^+sRX^!=)ArtEVk__C1wWu62&84{1O5W$=pydeKkva!Tswv%GI9yN4WS(7%J0w}Pll zL7&+#GuJA6EwQR-bIPXQY_A@#ky*Y{oi87e4%O>2Tq{;)jo>f=i>=b@c7EEqH}$k# zq5n-JDS%&}l)d@2E}h}uM^3O+c1c-$Bk#?NfvV>iAT$cR><@3KuhWpPvmL#83`$Lc z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOB(Jyhd}l+lM_DLQ|R1S9JggWMOn+E@EkJ+)pEt zS0B@B+5na71A~^(h?oKJe_7^kUF)~R705&JjF@Ba(+MsB@NR8#YcsGwevk!s&HdgP zC{Zt|h7q9Z02OU%Zf#+BUuAY>ZggK{VRU6KX=Hc=VZjEVxwC1aJrFfB;*n}Y01a?q za&&2QX>V>WWMOm!K^bOwf>Qhj>E&4HXupM*01I$(X>DP0c`j*WcnnTemQio;UGet* Kde0cxq6Ywpes2{3 literal 0 HcmV?d00001 diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/columns.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/columns.txt new file mode 100644 index 00000000000..4835061786d --- /dev/null +++ b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/columns.txt @@ -0,0 +1,259 @@ +columns format version: 1 +257 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`milliseconds` UInt64 +`ProfileEvent_Query` UInt64 +`ProfileEvent_SelectQuery` UInt64 +`ProfileEvent_InsertQuery` UInt64 +`ProfileEvent_FailedQuery` UInt64 +`ProfileEvent_FailedSelectQuery` UInt64 +`ProfileEvent_FailedInsertQuery` UInt64 +`ProfileEvent_QueryTimeMicroseconds` UInt64 +`ProfileEvent_SelectQueryTimeMicroseconds` UInt64 +`ProfileEvent_InsertQueryTimeMicroseconds` UInt64 +`ProfileEvent_FileOpen` UInt64 +`ProfileEvent_Seek` UInt64 +`ProfileEvent_ReadBufferFromFileDescriptorRead` UInt64 +`ProfileEvent_ReadBufferFromFileDescriptorReadFailed` UInt64 +`ProfileEvent_ReadBufferFromFileDescriptorReadBytes` UInt64 +`ProfileEvent_WriteBufferFromFileDescriptorWrite` UInt64 +`ProfileEvent_WriteBufferFromFileDescriptorWriteFailed` UInt64 +`ProfileEvent_WriteBufferFromFileDescriptorWriteBytes` UInt64 +`ProfileEvent_ReadBufferAIORead` UInt64 +`ProfileEvent_ReadBufferAIOReadBytes` UInt64 +`ProfileEvent_WriteBufferAIOWrite` UInt64 +`ProfileEvent_WriteBufferAIOWriteBytes` UInt64 +`ProfileEvent_ReadCompressedBytes` UInt64 +`ProfileEvent_CompressedReadBufferBlocks` UInt64 +`ProfileEvent_CompressedReadBufferBytes` UInt64 +`ProfileEvent_UncompressedCacheHits` UInt64 +`ProfileEvent_UncompressedCacheMisses` UInt64 +`ProfileEvent_UncompressedCacheWeightLost` UInt64 +`ProfileEvent_IOBufferAllocs` UInt64 +`ProfileEvent_IOBufferAllocBytes` UInt64 +`ProfileEvent_ArenaAllocChunks` UInt64 +`ProfileEvent_ArenaAllocBytes` UInt64 +`ProfileEvent_FunctionExecute` UInt64 +`ProfileEvent_TableFunctionExecute` UInt64 +`ProfileEvent_MarkCacheHits` UInt64 +`ProfileEvent_MarkCacheMisses` UInt64 +`ProfileEvent_CreatedReadBufferOrdinary` UInt64 +`ProfileEvent_CreatedReadBufferAIO` UInt64 +`ProfileEvent_CreatedReadBufferAIOFailed` UInt64 +`ProfileEvent_CreatedReadBufferMMap` UInt64 +`ProfileEvent_CreatedReadBufferMMapFailed` UInt64 +`ProfileEvent_CreatedWriteBufferOrdinary` UInt64 +`ProfileEvent_CreatedWriteBufferAIO` UInt64 +`ProfileEvent_CreatedWriteBufferAIOFailed` UInt64 +`ProfileEvent_DiskReadElapsedMicroseconds` UInt64 +`ProfileEvent_DiskWriteElapsedMicroseconds` UInt64 +`ProfileEvent_NetworkReceiveElapsedMicroseconds` UInt64 +`ProfileEvent_NetworkSendElapsedMicroseconds` UInt64 +`ProfileEvent_ThrottlerSleepMicroseconds` UInt64 +`ProfileEvent_QueryMaskingRulesMatch` UInt64 +`ProfileEvent_ReplicatedPartFetches` UInt64 +`ProfileEvent_ReplicatedPartFailedFetches` UInt64 +`ProfileEvent_ObsoleteReplicatedParts` UInt64 +`ProfileEvent_ReplicatedPartMerges` UInt64 +`ProfileEvent_ReplicatedPartFetchesOfMerged` UInt64 +`ProfileEvent_ReplicatedPartMutations` UInt64 +`ProfileEvent_ReplicatedPartChecks` UInt64 +`ProfileEvent_ReplicatedPartChecksFailed` UInt64 +`ProfileEvent_ReplicatedDataLoss` UInt64 +`ProfileEvent_InsertedRows` UInt64 +`ProfileEvent_InsertedBytes` UInt64 +`ProfileEvent_DelayedInserts` UInt64 +`ProfileEvent_RejectedInserts` UInt64 +`ProfileEvent_DelayedInsertsMilliseconds` UInt64 +`ProfileEvent_DuplicatedInsertedBlocks` UInt64 +`ProfileEvent_ZooKeeperInit` UInt64 +`ProfileEvent_ZooKeeperTransactions` UInt64 +`ProfileEvent_ZooKeeperList` UInt64 +`ProfileEvent_ZooKeeperCreate` UInt64 +`ProfileEvent_ZooKeeperRemove` UInt64 +`ProfileEvent_ZooKeeperExists` UInt64 +`ProfileEvent_ZooKeeperGet` UInt64 +`ProfileEvent_ZooKeeperSet` UInt64 +`ProfileEvent_ZooKeeperMulti` UInt64 +`ProfileEvent_ZooKeeperCheck` UInt64 +`ProfileEvent_ZooKeeperClose` UInt64 +`ProfileEvent_ZooKeeperWatchResponse` UInt64 +`ProfileEvent_ZooKeeperUserExceptions` UInt64 +`ProfileEvent_ZooKeeperHardwareExceptions` UInt64 +`ProfileEvent_ZooKeeperOtherExceptions` UInt64 +`ProfileEvent_ZooKeeperWaitMicroseconds` UInt64 +`ProfileEvent_ZooKeeperBytesSent` UInt64 +`ProfileEvent_ZooKeeperBytesReceived` UInt64 +`ProfileEvent_DistributedConnectionFailTry` UInt64 +`ProfileEvent_DistributedConnectionMissingTable` UInt64 +`ProfileEvent_DistributedConnectionStaleReplica` UInt64 +`ProfileEvent_DistributedConnectionFailAtAll` UInt64 +`ProfileEvent_CompileAttempt` UInt64 +`ProfileEvent_CompileSuccess` UInt64 +`ProfileEvent_CompileFunction` UInt64 +`ProfileEvent_CompiledFunctionExecute` UInt64 +`ProfileEvent_CompileExpressionsMicroseconds` UInt64 +`ProfileEvent_CompileExpressionsBytes` UInt64 +`ProfileEvent_ExternalSortWritePart` UInt64 +`ProfileEvent_ExternalSortMerge` UInt64 +`ProfileEvent_ExternalAggregationWritePart` UInt64 +`ProfileEvent_ExternalAggregationMerge` UInt64 +`ProfileEvent_ExternalAggregationCompressedBytes` UInt64 +`ProfileEvent_ExternalAggregationUncompressedBytes` UInt64 +`ProfileEvent_SlowRead` UInt64 +`ProfileEvent_ReadBackoff` UInt64 +`ProfileEvent_ReplicaPartialShutdown` UInt64 +`ProfileEvent_SelectedParts` UInt64 +`ProfileEvent_SelectedRanges` UInt64 +`ProfileEvent_SelectedMarks` UInt64 +`ProfileEvent_SelectedRows` UInt64 +`ProfileEvent_SelectedBytes` UInt64 +`ProfileEvent_Merge` UInt64 +`ProfileEvent_MergedRows` UInt64 +`ProfileEvent_MergedUncompressedBytes` UInt64 +`ProfileEvent_MergesTimeMilliseconds` UInt64 +`ProfileEvent_MergeTreeDataWriterRows` UInt64 +`ProfileEvent_MergeTreeDataWriterUncompressedBytes` UInt64 +`ProfileEvent_MergeTreeDataWriterCompressedBytes` UInt64 +`ProfileEvent_MergeTreeDataWriterBlocks` UInt64 +`ProfileEvent_MergeTreeDataWriterBlocksAlreadySorted` UInt64 +`ProfileEvent_CannotRemoveEphemeralNode` UInt64 +`ProfileEvent_RegexpCreated` UInt64 +`ProfileEvent_ContextLock` UInt64 +`ProfileEvent_StorageBufferFlush` UInt64 +`ProfileEvent_StorageBufferErrorOnFlush` UInt64 +`ProfileEvent_StorageBufferPassedAllMinThresholds` UInt64 +`ProfileEvent_StorageBufferPassedTimeMaxThreshold` UInt64 +`ProfileEvent_StorageBufferPassedRowsMaxThreshold` UInt64 +`ProfileEvent_StorageBufferPassedBytesMaxThreshold` UInt64 +`ProfileEvent_DictCacheKeysRequested` UInt64 +`ProfileEvent_DictCacheKeysRequestedMiss` UInt64 +`ProfileEvent_DictCacheKeysRequestedFound` UInt64 +`ProfileEvent_DictCacheKeysExpired` UInt64 +`ProfileEvent_DictCacheKeysNotFound` UInt64 +`ProfileEvent_DictCacheKeysHit` UInt64 +`ProfileEvent_DictCacheRequestTimeNs` UInt64 +`ProfileEvent_DictCacheRequests` UInt64 +`ProfileEvent_DictCacheLockWriteNs` UInt64 +`ProfileEvent_DictCacheLockReadNs` UInt64 +`ProfileEvent_DistributedSyncInsertionTimeoutExceeded` UInt64 +`ProfileEvent_DataAfterMergeDiffersFromReplica` UInt64 +`ProfileEvent_DataAfterMutationDiffersFromReplica` UInt64 +`ProfileEvent_PolygonsAddedToPool` UInt64 +`ProfileEvent_PolygonsInPoolAllocatedBytes` UInt64 +`ProfileEvent_RWLockAcquiredReadLocks` UInt64 +`ProfileEvent_RWLockAcquiredWriteLocks` UInt64 +`ProfileEvent_RWLockReadersWaitMilliseconds` UInt64 +`ProfileEvent_RWLockWritersWaitMilliseconds` UInt64 +`ProfileEvent_DNSError` UInt64 +`ProfileEvent_RealTimeMicroseconds` UInt64 +`ProfileEvent_UserTimeMicroseconds` UInt64 +`ProfileEvent_SystemTimeMicroseconds` UInt64 +`ProfileEvent_SoftPageFaults` UInt64 +`ProfileEvent_HardPageFaults` UInt64 +`ProfileEvent_VoluntaryContextSwitches` UInt64 +`ProfileEvent_InvoluntaryContextSwitches` UInt64 +`ProfileEvent_OSIOWaitMicroseconds` UInt64 +`ProfileEvent_OSCPUWaitMicroseconds` UInt64 +`ProfileEvent_OSCPUVirtualTimeMicroseconds` UInt64 +`ProfileEvent_OSReadBytes` UInt64 +`ProfileEvent_OSWriteBytes` UInt64 +`ProfileEvent_OSReadChars` UInt64 +`ProfileEvent_OSWriteChars` UInt64 +`ProfileEvent_PerfCpuCycles` UInt64 +`ProfileEvent_PerfInstructions` UInt64 +`ProfileEvent_PerfCacheReferences` UInt64 +`ProfileEvent_PerfCacheMisses` UInt64 +`ProfileEvent_PerfBranchInstructions` UInt64 +`ProfileEvent_PerfBranchMisses` UInt64 +`ProfileEvent_PerfBusCycles` UInt64 +`ProfileEvent_PerfStalledCyclesFrontend` UInt64 +`ProfileEvent_PerfStalledCyclesBackend` UInt64 +`ProfileEvent_PerfRefCpuCycles` UInt64 +`ProfileEvent_PerfCpuClock` UInt64 +`ProfileEvent_PerfTaskClock` UInt64 +`ProfileEvent_PerfContextSwitches` UInt64 +`ProfileEvent_PerfCpuMigrations` UInt64 +`ProfileEvent_PerfAlignmentFaults` UInt64 +`ProfileEvent_PerfEmulationFaults` UInt64 +`ProfileEvent_PerfMinEnabledTime` UInt64 +`ProfileEvent_PerfMinEnabledRunningTime` UInt64 +`ProfileEvent_PerfDataTLBReferences` UInt64 +`ProfileEvent_PerfDataTLBMisses` UInt64 +`ProfileEvent_PerfInstructionTLBReferences` UInt64 +`ProfileEvent_PerfInstructionTLBMisses` UInt64 +`ProfileEvent_PerfLocalMemoryReferences` UInt64 +`ProfileEvent_PerfLocalMemoryMisses` UInt64 +`ProfileEvent_CreatedHTTPConnections` UInt64 +`ProfileEvent_CannotWriteToWriteBufferDiscard` UInt64 +`ProfileEvent_QueryProfilerSignalOverruns` UInt64 +`ProfileEvent_CreatedLogEntryForMerge` UInt64 +`ProfileEvent_NotCreatedLogEntryForMerge` UInt64 +`ProfileEvent_CreatedLogEntryForMutation` UInt64 +`ProfileEvent_NotCreatedLogEntryForMutation` UInt64 +`ProfileEvent_S3ReadMicroseconds` UInt64 +`ProfileEvent_S3ReadBytes` UInt64 +`ProfileEvent_S3ReadRequestsCount` UInt64 +`ProfileEvent_S3ReadRequestsErrors` UInt64 +`ProfileEvent_S3ReadRequestsThrottling` UInt64 +`ProfileEvent_S3ReadRequestsRedirects` UInt64 +`ProfileEvent_S3WriteMicroseconds` UInt64 +`ProfileEvent_S3WriteBytes` UInt64 +`ProfileEvent_S3WriteRequestsCount` UInt64 +`ProfileEvent_S3WriteRequestsErrors` UInt64 +`ProfileEvent_S3WriteRequestsThrottling` UInt64 +`ProfileEvent_S3WriteRequestsRedirects` UInt64 +`ProfileEvent_QueryMemoryLimitExceeded` UInt64 +`CurrentMetric_Query` Int64 +`CurrentMetric_Merge` Int64 +`CurrentMetric_PartMutation` Int64 +`CurrentMetric_ReplicatedFetch` Int64 +`CurrentMetric_ReplicatedSend` Int64 +`CurrentMetric_ReplicatedChecks` Int64 +`CurrentMetric_BackgroundPoolTask` Int64 +`CurrentMetric_BackgroundMovePoolTask` Int64 +`CurrentMetric_BackgroundSchedulePoolTask` Int64 +`CurrentMetric_BackgroundBufferFlushSchedulePoolTask` Int64 +`CurrentMetric_BackgroundDistributedSchedulePoolTask` Int64 +`CurrentMetric_BackgroundMessageBrokerSchedulePoolTask` Int64 +`CurrentMetric_CacheDictionaryUpdateQueueBatches` Int64 +`CurrentMetric_CacheDictionaryUpdateQueueKeys` Int64 +`CurrentMetric_DiskSpaceReservedForMerge` Int64 +`CurrentMetric_DistributedSend` Int64 +`CurrentMetric_QueryPreempted` Int64 +`CurrentMetric_TCPConnection` Int64 +`CurrentMetric_MySQLConnection` Int64 +`CurrentMetric_HTTPConnection` Int64 +`CurrentMetric_InterserverConnection` Int64 +`CurrentMetric_PostgreSQLConnection` Int64 +`CurrentMetric_OpenFileForRead` Int64 +`CurrentMetric_OpenFileForWrite` Int64 +`CurrentMetric_Read` Int64 +`CurrentMetric_Write` Int64 +`CurrentMetric_SendScalars` Int64 +`CurrentMetric_SendExternalTables` Int64 +`CurrentMetric_QueryThread` Int64 +`CurrentMetric_ReadonlyReplica` Int64 +`CurrentMetric_MemoryTracking` Int64 +`CurrentMetric_EphemeralNode` Int64 +`CurrentMetric_ZooKeeperSession` Int64 +`CurrentMetric_ZooKeeperWatch` Int64 +`CurrentMetric_ZooKeeperRequest` Int64 +`CurrentMetric_DelayedInserts` Int64 +`CurrentMetric_ContextLockWait` Int64 +`CurrentMetric_StorageBufferRows` Int64 +`CurrentMetric_StorageBufferBytes` Int64 +`CurrentMetric_DictCacheRequests` Int64 +`CurrentMetric_Revision` Int64 +`CurrentMetric_VersionInteger` Int64 +`CurrentMetric_RWLockWaitingReaders` Int64 +`CurrentMetric_RWLockWaitingWriters` Int64 +`CurrentMetric_RWLockActiveReaders` Int64 +`CurrentMetric_RWLockActiveWriters` Int64 +`CurrentMetric_GlobalThread` Int64 +`CurrentMetric_GlobalThreadActive` Int64 +`CurrentMetric_LocalThread` Int64 +`CurrentMetric_LocalThreadActive` Int64 +`CurrentMetric_DistributedFilesToInsert` Int64 diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/count.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/count.txt new file mode 100644 index 00000000000..301160a9306 --- /dev/null +++ b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/count.txt @@ -0,0 +1 @@ +8 \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/default_compression_codec.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/minmax_event_date.idx b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/minmax_event_date.idx new file mode 100644 index 00000000000..16cc6680505 --- /dev/null +++ b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/partition.dat b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..da540c2d1a825be3ef80d13baa834834b2248491 GIT binary patch literal 4 LcmXRDWo7^X0wDm8 literal 0 HcmV?d00001 diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/primary.idx b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/primary.idx new file mode 100644 index 00000000000..3709b3e4f82 --- /dev/null +++ b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/primary.idx @@ -0,0 +1 @@ +H"`H"` \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/checksums.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..df2f868f3e238b0545f87bcd10382a47e43e9b21 GIT binary patch literal 260 zcmV+f0sH=AXk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$c3z@`!4L&OV%^7r2tD!?&< z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOEL%%`Kt3xZ#Bpu>(3pY7Tm^@j&dUd`w1=p@NR8#YcsGwZggK{VRU6KX=Hc=VZjEVxwC1aJrFfB;*n}Y01a?q za&&2QX>V>WWMOm!K^bOwf>Qhj>E&4HXupM*01I$(X>DP0c`j*Wcnsy^i!!b(-JKN- K=7_>$hb#bNOKuqe literal 0 HcmV?d00001 diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/columns.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/columns.txt new file mode 100644 index 00000000000..4835061786d --- /dev/null +++ b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/columns.txt @@ -0,0 +1,259 @@ +columns format version: 1 +257 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`milliseconds` UInt64 +`ProfileEvent_Query` UInt64 +`ProfileEvent_SelectQuery` UInt64 +`ProfileEvent_InsertQuery` UInt64 +`ProfileEvent_FailedQuery` UInt64 +`ProfileEvent_FailedSelectQuery` UInt64 +`ProfileEvent_FailedInsertQuery` UInt64 +`ProfileEvent_QueryTimeMicroseconds` UInt64 +`ProfileEvent_SelectQueryTimeMicroseconds` UInt64 +`ProfileEvent_InsertQueryTimeMicroseconds` UInt64 +`ProfileEvent_FileOpen` UInt64 +`ProfileEvent_Seek` UInt64 +`ProfileEvent_ReadBufferFromFileDescriptorRead` UInt64 +`ProfileEvent_ReadBufferFromFileDescriptorReadFailed` UInt64 +`ProfileEvent_ReadBufferFromFileDescriptorReadBytes` UInt64 +`ProfileEvent_WriteBufferFromFileDescriptorWrite` UInt64 +`ProfileEvent_WriteBufferFromFileDescriptorWriteFailed` UInt64 +`ProfileEvent_WriteBufferFromFileDescriptorWriteBytes` UInt64 +`ProfileEvent_ReadBufferAIORead` UInt64 +`ProfileEvent_ReadBufferAIOReadBytes` UInt64 +`ProfileEvent_WriteBufferAIOWrite` UInt64 +`ProfileEvent_WriteBufferAIOWriteBytes` UInt64 +`ProfileEvent_ReadCompressedBytes` UInt64 +`ProfileEvent_CompressedReadBufferBlocks` UInt64 +`ProfileEvent_CompressedReadBufferBytes` UInt64 +`ProfileEvent_UncompressedCacheHits` UInt64 +`ProfileEvent_UncompressedCacheMisses` UInt64 +`ProfileEvent_UncompressedCacheWeightLost` UInt64 +`ProfileEvent_IOBufferAllocs` UInt64 +`ProfileEvent_IOBufferAllocBytes` UInt64 +`ProfileEvent_ArenaAllocChunks` UInt64 +`ProfileEvent_ArenaAllocBytes` UInt64 +`ProfileEvent_FunctionExecute` UInt64 +`ProfileEvent_TableFunctionExecute` UInt64 +`ProfileEvent_MarkCacheHits` UInt64 +`ProfileEvent_MarkCacheMisses` UInt64 +`ProfileEvent_CreatedReadBufferOrdinary` UInt64 +`ProfileEvent_CreatedReadBufferAIO` UInt64 +`ProfileEvent_CreatedReadBufferAIOFailed` UInt64 +`ProfileEvent_CreatedReadBufferMMap` UInt64 +`ProfileEvent_CreatedReadBufferMMapFailed` UInt64 +`ProfileEvent_CreatedWriteBufferOrdinary` UInt64 +`ProfileEvent_CreatedWriteBufferAIO` UInt64 +`ProfileEvent_CreatedWriteBufferAIOFailed` UInt64 +`ProfileEvent_DiskReadElapsedMicroseconds` UInt64 +`ProfileEvent_DiskWriteElapsedMicroseconds` UInt64 +`ProfileEvent_NetworkReceiveElapsedMicroseconds` UInt64 +`ProfileEvent_NetworkSendElapsedMicroseconds` UInt64 +`ProfileEvent_ThrottlerSleepMicroseconds` UInt64 +`ProfileEvent_QueryMaskingRulesMatch` UInt64 +`ProfileEvent_ReplicatedPartFetches` UInt64 +`ProfileEvent_ReplicatedPartFailedFetches` UInt64 +`ProfileEvent_ObsoleteReplicatedParts` UInt64 +`ProfileEvent_ReplicatedPartMerges` UInt64 +`ProfileEvent_ReplicatedPartFetchesOfMerged` UInt64 +`ProfileEvent_ReplicatedPartMutations` UInt64 +`ProfileEvent_ReplicatedPartChecks` UInt64 +`ProfileEvent_ReplicatedPartChecksFailed` UInt64 +`ProfileEvent_ReplicatedDataLoss` UInt64 +`ProfileEvent_InsertedRows` UInt64 +`ProfileEvent_InsertedBytes` UInt64 +`ProfileEvent_DelayedInserts` UInt64 +`ProfileEvent_RejectedInserts` UInt64 +`ProfileEvent_DelayedInsertsMilliseconds` UInt64 +`ProfileEvent_DuplicatedInsertedBlocks` UInt64 +`ProfileEvent_ZooKeeperInit` UInt64 +`ProfileEvent_ZooKeeperTransactions` UInt64 +`ProfileEvent_ZooKeeperList` UInt64 +`ProfileEvent_ZooKeeperCreate` UInt64 +`ProfileEvent_ZooKeeperRemove` UInt64 +`ProfileEvent_ZooKeeperExists` UInt64 +`ProfileEvent_ZooKeeperGet` UInt64 +`ProfileEvent_ZooKeeperSet` UInt64 +`ProfileEvent_ZooKeeperMulti` UInt64 +`ProfileEvent_ZooKeeperCheck` UInt64 +`ProfileEvent_ZooKeeperClose` UInt64 +`ProfileEvent_ZooKeeperWatchResponse` UInt64 +`ProfileEvent_ZooKeeperUserExceptions` UInt64 +`ProfileEvent_ZooKeeperHardwareExceptions` UInt64 +`ProfileEvent_ZooKeeperOtherExceptions` UInt64 +`ProfileEvent_ZooKeeperWaitMicroseconds` UInt64 +`ProfileEvent_ZooKeeperBytesSent` UInt64 +`ProfileEvent_ZooKeeperBytesReceived` UInt64 +`ProfileEvent_DistributedConnectionFailTry` UInt64 +`ProfileEvent_DistributedConnectionMissingTable` UInt64 +`ProfileEvent_DistributedConnectionStaleReplica` UInt64 +`ProfileEvent_DistributedConnectionFailAtAll` UInt64 +`ProfileEvent_CompileAttempt` UInt64 +`ProfileEvent_CompileSuccess` UInt64 +`ProfileEvent_CompileFunction` UInt64 +`ProfileEvent_CompiledFunctionExecute` UInt64 +`ProfileEvent_CompileExpressionsMicroseconds` UInt64 +`ProfileEvent_CompileExpressionsBytes` UInt64 +`ProfileEvent_ExternalSortWritePart` UInt64 +`ProfileEvent_ExternalSortMerge` UInt64 +`ProfileEvent_ExternalAggregationWritePart` UInt64 +`ProfileEvent_ExternalAggregationMerge` UInt64 +`ProfileEvent_ExternalAggregationCompressedBytes` UInt64 +`ProfileEvent_ExternalAggregationUncompressedBytes` UInt64 +`ProfileEvent_SlowRead` UInt64 +`ProfileEvent_ReadBackoff` UInt64 +`ProfileEvent_ReplicaPartialShutdown` UInt64 +`ProfileEvent_SelectedParts` UInt64 +`ProfileEvent_SelectedRanges` UInt64 +`ProfileEvent_SelectedMarks` UInt64 +`ProfileEvent_SelectedRows` UInt64 +`ProfileEvent_SelectedBytes` UInt64 +`ProfileEvent_Merge` UInt64 +`ProfileEvent_MergedRows` UInt64 +`ProfileEvent_MergedUncompressedBytes` UInt64 +`ProfileEvent_MergesTimeMilliseconds` UInt64 +`ProfileEvent_MergeTreeDataWriterRows` UInt64 +`ProfileEvent_MergeTreeDataWriterUncompressedBytes` UInt64 +`ProfileEvent_MergeTreeDataWriterCompressedBytes` UInt64 +`ProfileEvent_MergeTreeDataWriterBlocks` UInt64 +`ProfileEvent_MergeTreeDataWriterBlocksAlreadySorted` UInt64 +`ProfileEvent_CannotRemoveEphemeralNode` UInt64 +`ProfileEvent_RegexpCreated` UInt64 +`ProfileEvent_ContextLock` UInt64 +`ProfileEvent_StorageBufferFlush` UInt64 +`ProfileEvent_StorageBufferErrorOnFlush` UInt64 +`ProfileEvent_StorageBufferPassedAllMinThresholds` UInt64 +`ProfileEvent_StorageBufferPassedTimeMaxThreshold` UInt64 +`ProfileEvent_StorageBufferPassedRowsMaxThreshold` UInt64 +`ProfileEvent_StorageBufferPassedBytesMaxThreshold` UInt64 +`ProfileEvent_DictCacheKeysRequested` UInt64 +`ProfileEvent_DictCacheKeysRequestedMiss` UInt64 +`ProfileEvent_DictCacheKeysRequestedFound` UInt64 +`ProfileEvent_DictCacheKeysExpired` UInt64 +`ProfileEvent_DictCacheKeysNotFound` UInt64 +`ProfileEvent_DictCacheKeysHit` UInt64 +`ProfileEvent_DictCacheRequestTimeNs` UInt64 +`ProfileEvent_DictCacheRequests` UInt64 +`ProfileEvent_DictCacheLockWriteNs` UInt64 +`ProfileEvent_DictCacheLockReadNs` UInt64 +`ProfileEvent_DistributedSyncInsertionTimeoutExceeded` UInt64 +`ProfileEvent_DataAfterMergeDiffersFromReplica` UInt64 +`ProfileEvent_DataAfterMutationDiffersFromReplica` UInt64 +`ProfileEvent_PolygonsAddedToPool` UInt64 +`ProfileEvent_PolygonsInPoolAllocatedBytes` UInt64 +`ProfileEvent_RWLockAcquiredReadLocks` UInt64 +`ProfileEvent_RWLockAcquiredWriteLocks` UInt64 +`ProfileEvent_RWLockReadersWaitMilliseconds` UInt64 +`ProfileEvent_RWLockWritersWaitMilliseconds` UInt64 +`ProfileEvent_DNSError` UInt64 +`ProfileEvent_RealTimeMicroseconds` UInt64 +`ProfileEvent_UserTimeMicroseconds` UInt64 +`ProfileEvent_SystemTimeMicroseconds` UInt64 +`ProfileEvent_SoftPageFaults` UInt64 +`ProfileEvent_HardPageFaults` UInt64 +`ProfileEvent_VoluntaryContextSwitches` UInt64 +`ProfileEvent_InvoluntaryContextSwitches` UInt64 +`ProfileEvent_OSIOWaitMicroseconds` UInt64 +`ProfileEvent_OSCPUWaitMicroseconds` UInt64 +`ProfileEvent_OSCPUVirtualTimeMicroseconds` UInt64 +`ProfileEvent_OSReadBytes` UInt64 +`ProfileEvent_OSWriteBytes` UInt64 +`ProfileEvent_OSReadChars` UInt64 +`ProfileEvent_OSWriteChars` UInt64 +`ProfileEvent_PerfCpuCycles` UInt64 +`ProfileEvent_PerfInstructions` UInt64 +`ProfileEvent_PerfCacheReferences` UInt64 +`ProfileEvent_PerfCacheMisses` UInt64 +`ProfileEvent_PerfBranchInstructions` UInt64 +`ProfileEvent_PerfBranchMisses` UInt64 +`ProfileEvent_PerfBusCycles` UInt64 +`ProfileEvent_PerfStalledCyclesFrontend` UInt64 +`ProfileEvent_PerfStalledCyclesBackend` UInt64 +`ProfileEvent_PerfRefCpuCycles` UInt64 +`ProfileEvent_PerfCpuClock` UInt64 +`ProfileEvent_PerfTaskClock` UInt64 +`ProfileEvent_PerfContextSwitches` UInt64 +`ProfileEvent_PerfCpuMigrations` UInt64 +`ProfileEvent_PerfAlignmentFaults` UInt64 +`ProfileEvent_PerfEmulationFaults` UInt64 +`ProfileEvent_PerfMinEnabledTime` UInt64 +`ProfileEvent_PerfMinEnabledRunningTime` UInt64 +`ProfileEvent_PerfDataTLBReferences` UInt64 +`ProfileEvent_PerfDataTLBMisses` UInt64 +`ProfileEvent_PerfInstructionTLBReferences` UInt64 +`ProfileEvent_PerfInstructionTLBMisses` UInt64 +`ProfileEvent_PerfLocalMemoryReferences` UInt64 +`ProfileEvent_PerfLocalMemoryMisses` UInt64 +`ProfileEvent_CreatedHTTPConnections` UInt64 +`ProfileEvent_CannotWriteToWriteBufferDiscard` UInt64 +`ProfileEvent_QueryProfilerSignalOverruns` UInt64 +`ProfileEvent_CreatedLogEntryForMerge` UInt64 +`ProfileEvent_NotCreatedLogEntryForMerge` UInt64 +`ProfileEvent_CreatedLogEntryForMutation` UInt64 +`ProfileEvent_NotCreatedLogEntryForMutation` UInt64 +`ProfileEvent_S3ReadMicroseconds` UInt64 +`ProfileEvent_S3ReadBytes` UInt64 +`ProfileEvent_S3ReadRequestsCount` UInt64 +`ProfileEvent_S3ReadRequestsErrors` UInt64 +`ProfileEvent_S3ReadRequestsThrottling` UInt64 +`ProfileEvent_S3ReadRequestsRedirects` UInt64 +`ProfileEvent_S3WriteMicroseconds` UInt64 +`ProfileEvent_S3WriteBytes` UInt64 +`ProfileEvent_S3WriteRequestsCount` UInt64 +`ProfileEvent_S3WriteRequestsErrors` UInt64 +`ProfileEvent_S3WriteRequestsThrottling` UInt64 +`ProfileEvent_S3WriteRequestsRedirects` UInt64 +`ProfileEvent_QueryMemoryLimitExceeded` UInt64 +`CurrentMetric_Query` Int64 +`CurrentMetric_Merge` Int64 +`CurrentMetric_PartMutation` Int64 +`CurrentMetric_ReplicatedFetch` Int64 +`CurrentMetric_ReplicatedSend` Int64 +`CurrentMetric_ReplicatedChecks` Int64 +`CurrentMetric_BackgroundPoolTask` Int64 +`CurrentMetric_BackgroundMovePoolTask` Int64 +`CurrentMetric_BackgroundSchedulePoolTask` Int64 +`CurrentMetric_BackgroundBufferFlushSchedulePoolTask` Int64 +`CurrentMetric_BackgroundDistributedSchedulePoolTask` Int64 +`CurrentMetric_BackgroundMessageBrokerSchedulePoolTask` Int64 +`CurrentMetric_CacheDictionaryUpdateQueueBatches` Int64 +`CurrentMetric_CacheDictionaryUpdateQueueKeys` Int64 +`CurrentMetric_DiskSpaceReservedForMerge` Int64 +`CurrentMetric_DistributedSend` Int64 +`CurrentMetric_QueryPreempted` Int64 +`CurrentMetric_TCPConnection` Int64 +`CurrentMetric_MySQLConnection` Int64 +`CurrentMetric_HTTPConnection` Int64 +`CurrentMetric_InterserverConnection` Int64 +`CurrentMetric_PostgreSQLConnection` Int64 +`CurrentMetric_OpenFileForRead` Int64 +`CurrentMetric_OpenFileForWrite` Int64 +`CurrentMetric_Read` Int64 +`CurrentMetric_Write` Int64 +`CurrentMetric_SendScalars` Int64 +`CurrentMetric_SendExternalTables` Int64 +`CurrentMetric_QueryThread` Int64 +`CurrentMetric_ReadonlyReplica` Int64 +`CurrentMetric_MemoryTracking` Int64 +`CurrentMetric_EphemeralNode` Int64 +`CurrentMetric_ZooKeeperSession` Int64 +`CurrentMetric_ZooKeeperWatch` Int64 +`CurrentMetric_ZooKeeperRequest` Int64 +`CurrentMetric_DelayedInserts` Int64 +`CurrentMetric_ContextLockWait` Int64 +`CurrentMetric_StorageBufferRows` Int64 +`CurrentMetric_StorageBufferBytes` Int64 +`CurrentMetric_DictCacheRequests` Int64 +`CurrentMetric_Revision` Int64 +`CurrentMetric_VersionInteger` Int64 +`CurrentMetric_RWLockWaitingReaders` Int64 +`CurrentMetric_RWLockWaitingWriters` Int64 +`CurrentMetric_RWLockActiveReaders` Int64 +`CurrentMetric_RWLockActiveWriters` Int64 +`CurrentMetric_GlobalThread` Int64 +`CurrentMetric_GlobalThreadActive` Int64 +`CurrentMetric_LocalThread` Int64 +`CurrentMetric_LocalThreadActive` Int64 +`CurrentMetric_DistributedFilesToInsert` Int64 diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/count.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/count.txt new file mode 100644 index 00000000000..c7930257dfe --- /dev/null +++ b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/count.txt @@ -0,0 +1 @@ +7 \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/default_compression_codec.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/minmax_event_date.idx b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/minmax_event_date.idx new file mode 100644 index 00000000000..16cc6680505 --- /dev/null +++ b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/partition.dat b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..da540c2d1a825be3ef80d13baa834834b2248491 GIT binary patch literal 4 LcmXRDWo7^X0wDm8 literal 0 HcmV?d00001 diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/primary.idx b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/primary.idx new file mode 100644 index 00000000000..dc6e4026178 --- /dev/null +++ b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/primary.idx @@ -0,0 +1 @@ +H"`H"` \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/format_version.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/format_version.txt new file mode 100644 index 00000000000..56a6051ca2b --- /dev/null +++ b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/format_version.txt @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/programs/server/store/2c6/2c6da17b-121a-4775-b287-b0d80cd04080/hits.sql b/programs/server/store/2c6/2c6da17b-121a-4775-b287-b0d80cd04080/hits.sql new file mode 100644 index 00000000000..81ffc1acf48 --- /dev/null +++ b/programs/server/store/2c6/2c6da17b-121a-4775-b287-b0d80cd04080/hits.sql @@ -0,0 +1,141 @@ +ATTACH TABLE _ UUID '72667623-6371-4258-be57-799522ee4e64' +( + `WatchID` UInt64, + `JavaEnable` UInt8, + `Title` String, + `GoodEvent` Int16, + `EventTime` DateTime, + `EventDate` Date, + `CounterID` UInt32, + `ClientIP` UInt32, + `ClientIP6` FixedString(16), + `RegionID` UInt32, + `UserID` UInt64, + `CounterClass` Int8, + `OS` UInt8, + `UserAgent` UInt8, + `URL` String, + `Referer` String, + `URLDomain` String, + `RefererDomain` String, + `Refresh` UInt8, + `IsRobot` UInt8, + `RefererCategories` Array(UInt16), + `URLCategories` Array(UInt16), + `URLRegions` Array(UInt32), + `RefererRegions` Array(UInt32), + `ResolutionWidth` UInt16, + `ResolutionHeight` UInt16, + `ResolutionDepth` UInt8, + `FlashMajor` UInt8, + `FlashMinor` UInt8, + `FlashMinor2` String, + `NetMajor` UInt8, + `NetMinor` UInt8, + `UserAgentMajor` UInt16, + `UserAgentMinor` FixedString(2), + `CookieEnable` UInt8, + `JavascriptEnable` UInt8, + `IsMobile` UInt8, + `MobilePhone` UInt8, + `MobilePhoneModel` String, + `Params` String, + `IPNetworkID` UInt32, + `TraficSourceID` Int8, + `SearchEngineID` UInt16, + `SearchPhrase` String, + `AdvEngineID` UInt8, + `IsArtifical` UInt8, + `WindowClientWidth` UInt16, + `WindowClientHeight` UInt16, + `ClientTimeZone` Int16, + `ClientEventTime` DateTime, + `SilverlightVersion1` UInt8, + `SilverlightVersion2` UInt8, + `SilverlightVersion3` UInt32, + `SilverlightVersion4` UInt16, + `PageCharset` String, + `CodeVersion` UInt32, + `IsLink` UInt8, + `IsDownload` UInt8, + `IsNotBounce` UInt8, + `FUniqID` UInt64, + `HID` UInt32, + `IsOldCounter` UInt8, + `IsEvent` UInt8, + `IsParameter` UInt8, + `DontCountHits` UInt8, + `WithHash` UInt8, + `HitColor` FixedString(1), + `UTCEventTime` DateTime, + `Age` UInt8, + `Sex` UInt8, + `Income` UInt8, + `Interests` UInt16, + `Robotness` UInt8, + `GeneralInterests` Array(UInt16), + `RemoteIP` UInt32, + `RemoteIP6` FixedString(16), + `WindowName` Int32, + `OpenerName` Int32, + `HistoryLength` Int16, + `BrowserLanguage` FixedString(2), + `BrowserCountry` FixedString(2), + `SocialNetwork` String, + `SocialAction` String, + `HTTPError` UInt16, + `SendTiming` Int32, + `DNSTiming` Int32, + `ConnectTiming` Int32, + `ResponseStartTiming` Int32, + `ResponseEndTiming` Int32, + `FetchTiming` Int32, + `RedirectTiming` Int32, + `DOMInteractiveTiming` Int32, + `DOMContentLoadedTiming` Int32, + `DOMCompleteTiming` Int32, + `LoadEventStartTiming` Int32, + `LoadEventEndTiming` Int32, + `NSToDOMContentLoadedTiming` Int32, + `FirstPaintTiming` Int32, + `RedirectCount` Int8, + `SocialSourceNetworkID` UInt8, + `SocialSourcePage` String, + `ParamPrice` Int64, + `ParamOrderID` String, + `ParamCurrency` FixedString(3), + `ParamCurrencyID` UInt16, + `GoalsReached` Array(UInt32), + `OpenstatServiceName` String, + `OpenstatCampaignID` String, + `OpenstatAdID` String, + `OpenstatSourceID` String, + `UTMSource` String, + `UTMMedium` String, + `UTMCampaign` String, + `UTMContent` String, + `UTMTerm` String, + `FromTag` String, + `HasGCLID` UInt8, + `RefererHash` UInt64, + `URLHash` UInt64, + `CLID` UInt32, + `YCLID` UInt64, + `ShareService` String, + `ShareURL` String, + `ShareTitle` String, + `ParsedParams.Key1` Array(String), + `ParsedParams.Key2` Array(String), + `ParsedParams.Key3` Array(String), + `ParsedParams.Key4` Array(String), + `ParsedParams.Key5` Array(String), + `ParsedParams.ValueDouble` Array(Float64), + `IslandID` FixedString(16), + `RequestNum` UInt32, + `RequestTry` UInt8 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(EventDate) +ORDER BY (CounterID, EventDate, intHash32(UserID), EventTime) +SAMPLE BY intHash32(UserID) +SETTINGS index_granularity = 8192 diff --git a/programs/server/store/2c6/2c6da17b-121a-4775-b287-b0d80cd04080/visits.sql b/programs/server/store/2c6/2c6da17b-121a-4775-b287-b0d80cd04080/visits.sql new file mode 100644 index 00000000000..cd2d3509c4e --- /dev/null +++ b/programs/server/store/2c6/2c6da17b-121a-4775-b287-b0d80cd04080/visits.sql @@ -0,0 +1,189 @@ +ATTACH TABLE _ UUID '044d7000-c91c-482d-9f0f-719f226eff0f' +( + `CounterID` UInt32, + `StartDate` Date, + `Sign` Int8, + `IsNew` UInt8, + `VisitID` UInt64, + `UserID` UInt64, + `StartTime` DateTime, + `Duration` UInt32, + `UTCStartTime` DateTime, + `PageViews` Int32, + `Hits` Int32, + `IsBounce` UInt8, + `Referer` String, + `StartURL` String, + `RefererDomain` String, + `StartURLDomain` String, + `EndURL` String, + `LinkURL` String, + `IsDownload` UInt8, + `TraficSourceID` Int8, + `SearchEngineID` UInt16, + `SearchPhrase` String, + `AdvEngineID` UInt8, + `PlaceID` Int32, + `RefererCategories` Array(UInt16), + `URLCategories` Array(UInt16), + `URLRegions` Array(UInt32), + `RefererRegions` Array(UInt32), + `IsYandex` UInt8, + `GoalReachesDepth` Int32, + `GoalReachesURL` Int32, + `GoalReachesAny` Int32, + `SocialSourceNetworkID` UInt8, + `SocialSourcePage` String, + `MobilePhoneModel` String, + `ClientEventTime` DateTime, + `RegionID` UInt32, + `ClientIP` UInt32, + `ClientIP6` FixedString(16), + `RemoteIP` UInt32, + `RemoteIP6` FixedString(16), + `IPNetworkID` UInt32, + `SilverlightVersion3` UInt32, + `CodeVersion` UInt32, + `ResolutionWidth` UInt16, + `ResolutionHeight` UInt16, + `UserAgentMajor` UInt16, + `UserAgentMinor` UInt16, + `WindowClientWidth` UInt16, + `WindowClientHeight` UInt16, + `SilverlightVersion2` UInt8, + `SilverlightVersion4` UInt16, + `FlashVersion3` UInt16, + `FlashVersion4` UInt16, + `ClientTimeZone` Int16, + `OS` UInt8, + `UserAgent` UInt8, + `ResolutionDepth` UInt8, + `FlashMajor` UInt8, + `FlashMinor` UInt8, + `NetMajor` UInt8, + `NetMinor` UInt8, + `MobilePhone` UInt8, + `SilverlightVersion1` UInt8, + `Age` UInt8, + `Sex` UInt8, + `Income` UInt8, + `JavaEnable` UInt8, + `CookieEnable` UInt8, + `JavascriptEnable` UInt8, + `IsMobile` UInt8, + `BrowserLanguage` UInt16, + `BrowserCountry` UInt16, + `Interests` UInt16, + `Robotness` UInt8, + `GeneralInterests` Array(UInt16), + `Params` Array(String), + `Goals.ID` Array(UInt32), + `Goals.Serial` Array(UInt32), + `Goals.EventTime` Array(DateTime), + `Goals.Price` Array(Int64), + `Goals.OrderID` Array(String), + `Goals.CurrencyID` Array(UInt32), + `WatchIDs` Array(UInt64), + `ParamSumPrice` Int64, + `ParamCurrency` FixedString(3), + `ParamCurrencyID` UInt16, + `ClickLogID` UInt64, + `ClickEventID` Int32, + `ClickGoodEvent` Int32, + `ClickEventTime` DateTime, + `ClickPriorityID` Int32, + `ClickPhraseID` Int32, + `ClickPageID` Int32, + `ClickPlaceID` Int32, + `ClickTypeID` Int32, + `ClickResourceID` Int32, + `ClickCost` UInt32, + `ClickClientIP` UInt32, + `ClickDomainID` UInt32, + `ClickURL` String, + `ClickAttempt` UInt8, + `ClickOrderID` UInt32, + `ClickBannerID` UInt32, + `ClickMarketCategoryID` UInt32, + `ClickMarketPP` UInt32, + `ClickMarketCategoryName` String, + `ClickMarketPPName` String, + `ClickAWAPSCampaignName` String, + `ClickPageName` String, + `ClickTargetType` UInt16, + `ClickTargetPhraseID` UInt64, + `ClickContextType` UInt8, + `ClickSelectType` Int8, + `ClickOptions` String, + `ClickGroupBannerID` Int32, + `OpenstatServiceName` String, + `OpenstatCampaignID` String, + `OpenstatAdID` String, + `OpenstatSourceID` String, + `UTMSource` String, + `UTMMedium` String, + `UTMCampaign` String, + `UTMContent` String, + `UTMTerm` String, + `FromTag` String, + `HasGCLID` UInt8, + `FirstVisit` DateTime, + `PredLastVisit` Date, + `LastVisit` Date, + `TotalVisits` UInt32, + `TraficSource.ID` Array(Int8), + `TraficSource.SearchEngineID` Array(UInt16), + `TraficSource.AdvEngineID` Array(UInt8), + `TraficSource.PlaceID` Array(UInt16), + `TraficSource.SocialSourceNetworkID` Array(UInt8), + `TraficSource.Domain` Array(String), + `TraficSource.SearchPhrase` Array(String), + `TraficSource.SocialSourcePage` Array(String), + `Attendance` FixedString(16), + `CLID` UInt32, + `YCLID` UInt64, + `NormalizedRefererHash` UInt64, + `SearchPhraseHash` UInt64, + `RefererDomainHash` UInt64, + `NormalizedStartURLHash` UInt64, + `StartURLDomainHash` UInt64, + `NormalizedEndURLHash` UInt64, + `TopLevelDomain` UInt64, + `URLScheme` UInt64, + `OpenstatServiceNameHash` UInt64, + `OpenstatCampaignIDHash` UInt64, + `OpenstatAdIDHash` UInt64, + `OpenstatSourceIDHash` UInt64, + `UTMSourceHash` UInt64, + `UTMMediumHash` UInt64, + `UTMCampaignHash` UInt64, + `UTMContentHash` UInt64, + `UTMTermHash` UInt64, + `FromHash` UInt64, + `WebVisorEnabled` UInt8, + `WebVisorActivity` UInt32, + `ParsedParams.Key1` Array(String), + `ParsedParams.Key2` Array(String), + `ParsedParams.Key3` Array(String), + `ParsedParams.Key4` Array(String), + `ParsedParams.Key5` Array(String), + `ParsedParams.ValueDouble` Array(Float64), + `Market.Type` Array(UInt8), + `Market.GoalID` Array(UInt32), + `Market.OrderID` Array(String), + `Market.OrderPrice` Array(Int64), + `Market.PP` Array(UInt32), + `Market.DirectPlaceID` Array(UInt32), + `Market.DirectOrderID` Array(UInt32), + `Market.DirectBannerID` Array(UInt32), + `Market.GoodID` Array(String), + `Market.GoodName` Array(String), + `Market.GoodQuantity` Array(Int32), + `Market.GoodPrice` Array(Int64), + `IslandID` FixedString(16) +) +ENGINE = CollapsingMergeTree(Sign) +PARTITION BY toYYYYMM(StartDate) +ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID) +SAMPLE BY intHash32(UserID) +SETTINGS index_granularity = 8192 diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/checksums.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..939ea6ccf187b34a8db6e4def524f9dd13691092 GIT binary patch literal 260 zcmV+f0sH=AXk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$bhh(-qXp!7uuz9v-dm@^cD z*Z=?k&Hw-a@jV6!V{dhCbS`vwbON#VDF3f}{FN>b+}S`mO_BfzWMOn+E@EkJ`W*io z<|Oi02OU%Zf#+BUuAY>ZggK{VRU6KX=Hc=Yhv{9L4)i+mY_Q2(jfMG01a?q za&&2QX>V>WWMOm!_Mvet<#01I$(X>DP0c`j*Wcns^T1OQK`i$9(N KipbM-Mdtv4FmBEO literal 0 HcmV?d00001 diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/columns.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/columns.txt new file mode 100644 index 00000000000..5ea415161bf --- /dev/null +++ b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/columns.txt @@ -0,0 +1,47 @@ +columns format version: 1 +45 columns: +`type` Enum8('QueryStart' = 1, 'QueryFinish' = 2, 'ExceptionBeforeStart' = 3, 'ExceptionWhileProcessing' = 4) +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`query_start_time` DateTime +`query_start_time_microseconds` DateTime64(6) +`query_duration_ms` UInt64 +`read_rows` UInt64 +`read_bytes` UInt64 +`written_rows` UInt64 +`written_bytes` UInt64 +`result_rows` UInt64 +`result_bytes` UInt64 +`memory_usage` UInt64 +`current_database` String +`query` String +`exception_code` Int32 +`exception` String +`stack_trace` String +`is_initial_query` UInt8 +`user` String +`query_id` String +`address` IPv6 +`port` UInt16 +`initial_user` String +`initial_query_id` String +`initial_address` IPv6 +`initial_port` UInt16 +`interface` UInt8 +`os_user` String +`client_hostname` String +`client_name` String +`client_revision` UInt32 +`client_version_major` UInt32 +`client_version_minor` UInt32 +`client_version_patch` UInt32 +`http_method` UInt8 +`http_user_agent` String +`quota_key` String +`revision` UInt32 +`thread_ids` Array(UInt64) +`ProfileEvents.Names` Array(String) +`ProfileEvents.Values` Array(UInt64) +`Settings.Names` Array(String) +`Settings.Values` Array(String) diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/count.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/count.txt new file mode 100644 index 00000000000..9a037142aa3 --- /dev/null +++ b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/count.txt @@ -0,0 +1 @@ +10 \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/default_compression_codec.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/minmax_event_date.idx b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/minmax_event_date.idx new file mode 100644 index 00000000000..4b6c49f8d50 --- /dev/null +++ b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/partition.dat b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..b9ea5569e3a3b34e3a1b6a1c4cbcb46f5e2d3fac GIT binary patch literal 4 Lcmb0VWo7^X0I>iw literal 0 HcmV?d00001 diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/primary.idx b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/primary.idx new file mode 100644 index 00000000000..d599340cc82 --- /dev/null +++ b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/primary.idx @@ -0,0 +1 @@ +H!._Hm._ \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/checksums.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..747078ab876a6913b8e0a9a92d3d0a3e3a1deb0f GIT binary patch literal 262 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyWHkv{Wolw154dF1>;$$|v}nZZR-0 zTma&a_H3NV`K5U!dL%eM=a3btktj z6fTJR@=T@IyyV0s#)et!Y;O}d4yN1KT?$>ea}nQfR!#$k4|%yo*~S~V4GOsV4O}$! zZrdrRHA`0hWDw2G%*#!zh)*p`%`1rq8l0+^nNq=`zaq;cGh)rekNa9x-^dl4GVm58 z7L{a{09~#JRK)U=M_>H!p{7$hOFDXtB}~#8xC@Fha}$dyLArT1wS3n4_i1|9>D}5g J%+@!V7yzW3XQcoD literal 0 HcmV?d00001 diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/columns.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/columns.txt new file mode 100644 index 00000000000..5ea415161bf --- /dev/null +++ b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/columns.txt @@ -0,0 +1,47 @@ +columns format version: 1 +45 columns: +`type` Enum8('QueryStart' = 1, 'QueryFinish' = 2, 'ExceptionBeforeStart' = 3, 'ExceptionWhileProcessing' = 4) +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`query_start_time` DateTime +`query_start_time_microseconds` DateTime64(6) +`query_duration_ms` UInt64 +`read_rows` UInt64 +`read_bytes` UInt64 +`written_rows` UInt64 +`written_bytes` UInt64 +`result_rows` UInt64 +`result_bytes` UInt64 +`memory_usage` UInt64 +`current_database` String +`query` String +`exception_code` Int32 +`exception` String +`stack_trace` String +`is_initial_query` UInt8 +`user` String +`query_id` String +`address` IPv6 +`port` UInt16 +`initial_user` String +`initial_query_id` String +`initial_address` IPv6 +`initial_port` UInt16 +`interface` UInt8 +`os_user` String +`client_hostname` String +`client_name` String +`client_revision` UInt32 +`client_version_major` UInt32 +`client_version_minor` UInt32 +`client_version_patch` UInt32 +`http_method` UInt8 +`http_user_agent` String +`quota_key` String +`revision` UInt32 +`thread_ids` Array(UInt64) +`ProfileEvents.Names` Array(String) +`ProfileEvents.Values` Array(UInt64) +`Settings.Names` Array(String) +`Settings.Values` Array(String) diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/count.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/count.txt new file mode 100644 index 00000000000..780fea92d29 --- /dev/null +++ b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/count.txt @@ -0,0 +1 @@ +77 \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/default_compression_codec.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/minmax_event_date.idx b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/minmax_event_date.idx new file mode 100644 index 00000000000..4fedf1265b2 --- /dev/null +++ b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/partition.dat b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..7af4421fcb5ebc50aa06a5f4dee6e454020552a6 GIT binary patch literal 4 Lcmb0UWo7^X0J#7) literal 0 HcmV?d00001 diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/primary.idx b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/primary.idx new file mode 100644 index 00000000000..7f33001d913 --- /dev/null +++ b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/primary.idx @@ -0,0 +1 @@ +Hb_H_ \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202101_48_74_6/checksums.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202101_48_74_6/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..eec9f2828ad60100f834ee2ec7c2ecdd572e4d2f GIT binary patch literal 261 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyUI``tQA+<)x6f|4kELHWvPFy2-%6 za2|+1+Ocsa=a=S{=#^BIFika>Ad;2)__e!oc(_Su$x;T6l*E!my`;>%-kfLCXPdd* ze|I8LNK`6r-p_c(P3^3@vsGTkiX2IichpIIeqjR>r#{1nyxgK};|<&n9adSpqodSI zI&*}wm1=%7h~{SI(O)RPe>EV>iAT$c8rTFS^0FlRNAYdvChb+}S`mO_BfzWMOn+E@EkJqa$s) z<(i%4_5Su;$|AM?s{R3`Wi?hP8GT=Tu0P7G#$xv>0SPVu@NR8#YcsG56t3`xr|pUp z;{9}LW*LD002OU%Zf#+BUuAY>ZggK{VRU6KX=Hc=138%)U-7Z;v9?{s9d$U?01a?q za&&2QX>V>WWMOm!K^bOwf>Qhj>E&4HXupM*01I$(X>DP0c`j*Wcnn{AlpFuovTY~I Kn?YGm%iaJK*Kw-= literal 0 HcmV?d00001 diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/columns.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/columns.txt new file mode 100644 index 00000000000..5ea415161bf --- /dev/null +++ b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/columns.txt @@ -0,0 +1,47 @@ +columns format version: 1 +45 columns: +`type` Enum8('QueryStart' = 1, 'QueryFinish' = 2, 'ExceptionBeforeStart' = 3, 'ExceptionWhileProcessing' = 4) +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`query_start_time` DateTime +`query_start_time_microseconds` DateTime64(6) +`query_duration_ms` UInt64 +`read_rows` UInt64 +`read_bytes` UInt64 +`written_rows` UInt64 +`written_bytes` UInt64 +`result_rows` UInt64 +`result_bytes` UInt64 +`memory_usage` UInt64 +`current_database` String +`query` String +`exception_code` Int32 +`exception` String +`stack_trace` String +`is_initial_query` UInt8 +`user` String +`query_id` String +`address` IPv6 +`port` UInt16 +`initial_user` String +`initial_query_id` String +`initial_address` IPv6 +`initial_port` UInt16 +`interface` UInt8 +`os_user` String +`client_hostname` String +`client_name` String +`client_revision` UInt32 +`client_version_major` UInt32 +`client_version_minor` UInt32 +`client_version_patch` UInt32 +`http_method` UInt8 +`http_user_agent` String +`quota_key` String +`revision` UInt32 +`thread_ids` Array(UInt64) +`ProfileEvents.Names` Array(String) +`ProfileEvents.Values` Array(UInt64) +`Settings.Names` Array(String) +`Settings.Values` Array(String) diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/count.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/count.txt new file mode 100644 index 00000000000..9a037142aa3 --- /dev/null +++ b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/count.txt @@ -0,0 +1 @@ +10 \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/default_compression_codec.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/minmax_event_date.idx b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/minmax_event_date.idx new file mode 100644 index 00000000000..0e3b7fb75b8 --- /dev/null +++ b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/partition.dat b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..da540c2d1a825be3ef80d13baa834834b2248491 GIT binary patch literal 4 LcmXRDWo7^X0wDm8 literal 0 HcmV?d00001 diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/primary.idx b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/primary.idx new file mode 100644 index 00000000000..0b5fcd2fdea --- /dev/null +++ b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/primary.idx @@ -0,0 +1 @@ +HC!`HG"` \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_80_80_0/checksums.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_80_80_0/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..a466972c338851bb9e376b5fec0b27c54f763a01 GIT binary patch literal 260 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyUI6hQ;Hcxb2IaD6D#6V%Tn`7;(^Ad>Sd->up}O2Td;F;<|11GQ{zVyvz!@t3lfV; zGE0E&)&nYHagV>iAT$cnNbtE_VA_ox+gYO}M76zw zkN^Mxh5!Hn@jV0yV{dhCbS`vwbOCIfRlVt&lYFUDR9N&B1^)mDWMOn+E@EkJ&k!F~ zY}HzkJ8Te4L?36TX=VY?FDFqniJ!N25;D3H*NTaDs0l6r@Fs0?Ycucyl#~%aH>Tie q3B6siBLUYk01I$(X>DP0c`j*WcnlPm#22y_7RAN-7E)Vm7x4gDE=jup literal 0 HcmV?d00001 diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/columns.txt b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/columns.txt new file mode 100644 index 00000000000..725a526b957 --- /dev/null +++ b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/columns.txt @@ -0,0 +1,13 @@ +columns format version: 1 +11 columns: +`event_date` Date +`event_time` DateTime +`timestamp_ns` UInt64 +`signal` Int32 +`thread_id` UInt64 +`query_id` String +`trace` Array(UInt64) +`trace_full` Array(String) +`version` String +`revision` UInt32 +`build_id` String diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/count.txt b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/count.txt new file mode 100644 index 00000000000..56a6051ca2b --- /dev/null +++ b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/count.txt @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/default_compression_codec.txt b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/primary.idx b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/primary.idx new file mode 100644 index 00000000000..b3b53069cfb --- /dev/null +++ b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/primary.idx @@ -0,0 +1 @@ +H9 `H9 ` \ No newline at end of file diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/checksums.txt b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..41a34e68660fcf30f2ebf3886af70965d1c50d97 GIT binary patch literal 188 zcmV;t07L&{Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$cC)=R{W#rbM&U?0f^HAjYm zkN^Mxh5!Hn@jV0yV{dhCbS`vwbOCIfRlVt&lYFUDR9N&B1^)mDWMOn+E@EkJ&=4}7 zAlkfU=Me)DyD->zPV@oLFCA^8bN;KGdEy@&Pv!bL6$vf?@Fs0?Ycucy8(uAC5Vito q|AbpmqG_6t01I$(X>DP0c`j*WcnnVC4s8oxdoa!Z4je{MT>k*Ano9!! literal 0 HcmV?d00001 diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/columns.txt b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/columns.txt new file mode 100644 index 00000000000..725a526b957 --- /dev/null +++ b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/columns.txt @@ -0,0 +1,13 @@ +columns format version: 1 +11 columns: +`event_date` Date +`event_time` DateTime +`timestamp_ns` UInt64 +`signal` Int32 +`thread_id` UInt64 +`query_id` String +`trace` Array(UInt64) +`trace_full` Array(String) +`version` String +`revision` UInt32 +`build_id` String diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/count.txt b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/count.txt new file mode 100644 index 00000000000..56a6051ca2b --- /dev/null +++ b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/count.txt @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/default_compression_codec.txt b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/primary.idx b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/primary.idx new file mode 100644 index 00000000000..24c83f6002c --- /dev/null +++ b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/primary.idx @@ -0,0 +1 @@ +H `H ` \ No newline at end of file diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/format_version.txt b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/format_version.txt new file mode 100644 index 00000000000..56a6051ca2b --- /dev/null +++ b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/format_version.txt @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/checksums.txt b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..94103f5a76ccb7f25b7a74030c04dbc2413b0242 GIT binary patch literal 260 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyY$U>CqQtR5+I_knLZ-OHz2#4F(2= zb3pvjmW?wxzcjBzucV@cX`=X>wR6M7)Yk+*GdQ7`_n(0yC9xz?FDWyxQ#5noG1-=x zMe7?wHfev7dG?oaNwm=1jm!7WVfXwmTE9_VNPjb@9>a&c+@fsb3oO0Y#oSLXwp>@( z{bpjsq?khtqPdxQxrr6=sb#5oCGkLGQ}r@aDp&%G|HSG1ZQ1rGFrD>==$*L?yakCx zC7C5ack2NavHaxG7yo;x>6FfrjviwPlXM2|f}+ga#G*=&Zk~`GM+A);B?QWr7Vt)1 I5q`=5051<|M*si- literal 0 HcmV?d00001 diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/columns.txt b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/columns.txt new file mode 100644 index 00000000000..7775d6f8eb4 --- /dev/null +++ b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/columns.txt @@ -0,0 +1,20 @@ +columns format version: 1 +18 columns: +`event_type` Enum8('NewPart' = 1, 'MergeParts' = 2, 'DownloadPart' = 3, 'RemovePart' = 4, 'MutatePart' = 5, 'MovePart' = 6) +`event_date` Date +`event_time` DateTime +`duration_ms` UInt64 +`database` String +`table` String +`part_name` String +`partition_id` String +`path_on_disk` String +`rows` UInt64 +`size_in_bytes` UInt64 +`merged_from` Array(String) +`bytes_uncompressed` UInt64 +`read_rows` UInt64 +`read_bytes` UInt64 +`peak_memory_usage` UInt64 +`error` UInt16 +`exception` String diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/count.txt b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/count.txt new file mode 100644 index 00000000000..eb13855b7d7 --- /dev/null +++ b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/count.txt @@ -0,0 +1 @@ +79 \ No newline at end of file diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/default_compression_codec.txt b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/minmax_event_date.idx b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/minmax_event_date.idx new file mode 100644 index 00000000000..07abc658bdb --- /dev/null +++ b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/partition.dat b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..7af4421fcb5ebc50aa06a5f4dee6e454020552a6 GIT binary patch literal 4 Lcmb0UWo7^X0J#7) literal 0 HcmV?d00001 diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/primary.idx b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/primary.idx new file mode 100644 index 00000000000..16d6a0eea67 --- /dev/null +++ b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/primary.idx @@ -0,0 +1 @@ +He_H(|_ \ No newline at end of file diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/checksums.txt b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..2b7fff5ccb9418c7ab7aa25bd1fb7d93eceec517 GIT binary patch literal 260 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyZ>tGIK5Cl6r%;##xVQ61m%&ZZI$~ zoCD&Iwrrfq`K5U!dLfa6k3NUy@yiSHP=3yLyx6N@TAx_KfGyWeaxDL=3MNbdhr I+Xd$s04~;Rw*UYD literal 0 HcmV?d00001 diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/columns.txt b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/columns.txt new file mode 100644 index 00000000000..7775d6f8eb4 --- /dev/null +++ b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/columns.txt @@ -0,0 +1,20 @@ +columns format version: 1 +18 columns: +`event_type` Enum8('NewPart' = 1, 'MergeParts' = 2, 'DownloadPart' = 3, 'RemovePart' = 4, 'MutatePart' = 5, 'MovePart' = 6) +`event_date` Date +`event_time` DateTime +`duration_ms` UInt64 +`database` String +`table` String +`part_name` String +`partition_id` String +`path_on_disk` String +`rows` UInt64 +`size_in_bytes` UInt64 +`merged_from` Array(String) +`bytes_uncompressed` UInt64 +`read_rows` UInt64 +`read_bytes` UInt64 +`peak_memory_usage` UInt64 +`error` UInt16 +`exception` String diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/count.txt b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/count.txt new file mode 100644 index 00000000000..9a037142aa3 --- /dev/null +++ b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/count.txt @@ -0,0 +1 @@ +10 \ No newline at end of file diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/default_compression_codec.txt b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/minmax_event_date.idx b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/minmax_event_date.idx new file mode 100644 index 00000000000..7d66cd0aae2 --- /dev/null +++ b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/partition.dat b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..a713aecd075daae7071deceaa93147871ef28fbf GIT binary patch literal 4 LcmXRFWo7^X0v!O3 literal 0 HcmV?d00001 diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/primary.idx b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/primary.idx new file mode 100644 index 00000000000..3e5042c4bf2 --- /dev/null +++ b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/primary.idx @@ -0,0 +1 @@ +H_HE_ \ No newline at end of file diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/format_version.txt b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/format_version.txt new file mode 100644 index 00000000000..56a6051ca2b --- /dev/null +++ b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/format_version.txt @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/checksums.txt b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..322c6666b3efd30007e1786d71ebf7dbb6bc46f1 GIT binary patch literal 10792 zcmX|HcRbbq_kX?b`(FFno3hEsY|xNB%F0MWT<*m+?&Z2hG;B&%s8l3WDkCF18Ym-F zMwCdUVcmP}mEXI3zQ2F`aqoQ`uh)5<^E}V3N(6qi0e)R^Q=BoWMwR6U5K%rs`c zqebZ+C)gsghP0ebQHp1vhrPLI*c<=ZJ<`Lhf@PpljHCTEw>F?*11Nr)#cv=Ph4Sk{ zQLLo`hh*=scf4W)%=+F02SCT9rCf)2dF=i;=;5C>o-L#63GI$Q0ca@JhfltIz~o-- zVUp^>;#xklXaLv>(0EdQs&h*!P=04R%#`ObPt5>c0}44Tm_RWkhmfe0GfrS#Y_lu| zx|E`M@cx-EORtQYclOXES)l&BMwX0*vC@&#Tjz%O_zQrt*9PAq0OP>|%EJ6XqEKe|$sBK0qv> zd*fHs$uDQ1;-^d8BM$~`#{cxhaCw|pof<5$WkoJ)4d^_lS6&$lo*2oEywRSzc&(=y z@EQ6O2_&jfV}_%mZaSgmq`#_nod)CXsPIh1ssY%(%dE-G4-{53w~=WN`J~iO0qLO zTj>9Ihk_eeM7^mtzDyk%|8P&J&}n|p%YE2=06vL)t_WF_v&c$`FU?zELT>H33Zwz| zJ}0msRuQ!h*9loRNR=OjpIhTU2Gxu!L-~uthHtNoO-Y_qdX{85ruZvQPleM7@b5c? zoT@RrLus9trQ*%V{WXS!*a+a20e}zEI7KE2Atz%LK7ddQrDqnX52l{lGtE%9xZrMR z9xB*XMq81Zw$y2IKR4X|Nyi@g`viDEYVL9h_4M z;)hwLE=NifG>O4)0goY>L?U<&Lh3}4w>x67G^PnXG@XOz=$xF&3dtdE9P)b{uhSIro;(5EhGeoIkzh>1dH731 zbaZ73T8oE#RZblwzm_49+*bYxBs|NQn%t^Ej(yc<|WRl5jM z#-GooSiP z8d=$eAg`MK@;#jOM~4xwCm1NPN9p^qpgWnTz!J)~t$xqtfD!Fl`^RB}AwkKU`vHu6 z&e_Jh@cd2Ny4V5b;<`yAZv`M=WNm3k4hZxoPzi`fF;7>cVMl+zh2V!t2Xud&@MBNC zT<8J*pjxlKC1^{RKTLeyeQrNVieK;ufPDcSX6^UrU@OR-h+x}nm=XUu2}G4a4dOBq z)q;$}6Yylj=CAA(G}gaQ|A^G$fru9xc6G`>9LZn{H5}fKHPzkR*>Lz&-OHaNLt;h% z4O0P)c}>l%zT~(AbYs9E-HB@eXQ=n z*P3_L6vX6a5c+Gf!_F#Z`^LrVFLRPa3;+ra1axL?Z&3?1p3~L)U7q*cPz-AYJVs;^ zwHMMbBT{iFI%Y2Z38cVoi~W>_3s}V-AMUteAmM>pLyaVDU3$t=YZ*@5`Nz-xoaKL! z2Q$iy8su7D3N73@Wa#DARSvL@7!4yFbtm$;Mc4LoYVELtIHZ(oQ8|HqgI|Y_pqEe= zv%G7#&1Y-{&7Z!k=%^jve%yIza>8T2FN2A*t=`#2*Y$tHti!|2lHF+Mk1uABukB zn!Oxc2-r;=NW>t1ctLFR_6sz4dvB~&tng@kiXu91QI)~2O~x5O!qaTB3xx+pm3N!$wHG9geseT)!qb*6WqhacSj>6-94Yne;1 z$>-py+)A*5lFIA!d9(98d&TH-qu!#B*>>jB*)^AvV)y0NYW^@e^P%N*av9wI}np#%G1i#?H+fJ6J_wGv>DMX{2Qmq12TQjL3&mM+g0d-0QHuewu-)K^%JkTm|TCw+YqL^;O*`Hde34 z1@7NG1^7(~BmxEJ&xw{M1n0t=Vk2`gOdL|tV7QrBP@*}Ywz~%N6H2XaReBO7sRqZD zP*i@Yx#6N?uW!G?CM>fNmRssD9?uuGS?T?BEEb3=0h?fA1VLRM!KU&D7g3ClCIw7d zPL+Ae_O+|dVVxVOE!2Itf~au$^CC*m7yi2WJCr zQ!>s!*p`6v^daDV!K!H9oJzFeNbr6Ez1N8ey>InPx@Dn_8y#pone#{0Z(fz1^${`= zji-#a@pl!rcO5ZUVosefC?p*c8N9_01myvy7kSsx+6!KRrv_LmY5r~cyVZrT31@o{ z_()9g@07lGXD&}A>G{eBAS!eO0N4+K4?cEPde%_+)Qfqxl){o{VE}7NCgXM3z$?TU z<2U-kMbqz{`DOWVqkzSdrsb{ANWSNvW&W7Sr+G6V6AIzX*EzeL?+mo-FAtAW#)?#| z;-_*hyPR3TY!-#2z+7xUc4Pi>YqT%^T(s@=1F0clN-tp=%KnMxgsjEo9+|1IlfAFr z*KN}{2HW+#yP>?(VskE>&y9{xYnuS9nZ3P@F@y7y~kO8T5@KRYjd=9g7Z_4er)9Dlr2@-Ak7@xOa4-_9n+ zGYD{*+gK5(VPuM53v#lh%4O8Vk|!#)_$)qQ%!gC!=CE)JLTScwBMkIe><)7%svcBO zzU;9LYz*diyn6OTTl|YQE4BUm-D%T+&79;(rZLgc?_Qp}LeB3fo={v)6&mP9<>vd(<0X?c0jGJe5jl+HPsZt^>FB6xEQoX|&R;9>7Wgq3 zMjW)`a6v#Y(B$xH*4=%+zu0$pnjS4XybZI97@1`HSDKdAw_g%+tHCpbw}9O|_#G0h zKdVu092dXw?XiAJZF}VlP+sp&B0Yt4(^TQ9JG@Ogb-^r(hrav#MKTA3M-~+5=zuQ@ z;TQahm~P(c%tD+VY(XUXu_ED6)2=f-zW3YXyXVKNWi#Wwonq(g5w|zHPlbFIs_?#X zC~wTAE-ZZ8?F`n_yXE@XBRw7&a35QGszxXQ9Ol86WDjB(2OV9WyXTp?2$NrIcrpX8 z;iWy=tmS}2*2&k+lh^4<)26%5>ucM{wEh?OkiSp*o%UNxwT>g7ZpV1dTGT<&R%EIH z(o=a>vC=Uw<}#q%>c_J`Bfq|>3AHpbmhF1tjClT5gvnyP`A)~f*)Jf%JuQ|0JkPSf z^XMz>>BwhKyq9q;cMQ)1Zu4Mke>{bahhV@$N5?!t#hGj_|7Kfu%T_XS_E2hOMk!BQjMM(1Ok<{3a7k1WMb}5Zp zdGMVqg0itCdyuJZi2J`Z@uoko>8a2V`HuOO+}ilbDgxFtn#KaPY~LqaR%nVgiaMR>g*hxq32_xGt1u7OXv z7Ey*Pav?!F=Y#cx7mr>X^K{?#{t~lWf7}hV3;8{IF0ASEw!__k^C&J9ca@WluFG|J zS5l#L`)ZlP)R!;GfguVx&|+uiea0mGes((M;$j1e+eO`RWmfcP zWpaH-0qgEtwwNu{H`2V1sP}lN)&BeB`>%_42NJv#1x@MbL=`LgbwY9r8=G@Lh z|B+$_?9pz1@f)0v303inEI4@V7~rahWo&#Lc^GhRtS0StQa{%nnxUo<(}#>QfoP@` zRLL-LfA!%N;EF|&6jd^;qTVT02tWevTp7KOquZj}jymjZq{pl^wEi{~+$qCF`# z1N(Yb7c!G?a{5fIzPZ8o4(eLG%tE#-@y4XM=mq%=Tm7P6#j1x^~L8{TzMh|8{^5AP#vtW@atKR*{>hRf(%W0wfcrmk|fCIt%I3&LZCwSni2Ni@ZrWJTM zk1su4%aV#jEuz{R$4!|t%9$M4yEOdv$5a2S98e~QW=x1!)puc2VXl?(Y9~j4jzX+# z6?4aF{0c?yc-9;x-m$o2pz&@G3mqCUTbw4$K3g6fS8J$kiG1h?pzZpIi8UOrHLP8K zgcL9^Nofd-iKa!&$bYGy7DCy#7AJ(-mKM#sMvIPrhnY33qqcP;^myYK~ z6%T}>n62$B01abK;_OX-{Z=)F*s^pm+d~_0S_cYqNL%5U5p+oGW33)>$ZnxFUA18u z2;x~?t0h3oDEa*Q!`Wluwn7meRKp{pG5?GC`A?dkKU+n1ucXLa$5O9!0B%`e7fi)b z^;6NSqD^DWLG`^~(@N@!(G{&Xa*NSfP(@-9x+JAbU}ba90f|>H_#aT|@HW;gz@tWq zyIXzc4=)Nl6<2I1l=w@!VhnKsfjFW!i2$}lD}FF__0--B`1SH?U&l=EmokkT&{kq0 zI*qhI>==w(_6e%Uj_*sw{Kvi)QR!-U-`9*<@y|THDuy;6S%3-%u_I7IiJk;iXhCEE zTAOOfJG0$0Z1C@Dsiu? z2Y=t)d^Fm>Qd4>g@b3n;hyB)DaZBc~Y4rPnnRKMviRO~1bA1xAnZXnZOJDce6!KI;4iItp@opm=E ziI%r)0MB0LAj1#}g+THY5ndC0*uvb^6`TqlskV(=iL54OTz(1tnfbtx@cyc9jeO%X z64URrJ0~A?@*V&%4&fG8uN3m#G#c%QakKBA=KSCd_!Iz27D4O8B#f{y@7;bymZYt$ z)L&-3O@aPw8>mRlimbFj{UKrl>jCo@b>I2_X>xQ}!1CfzrTu)Wf^8}BmmL~U0d7UG z$C`qF%g+$0UqUH8(u?RWH)ogbn{T<%sZN2e_vWF0Y3BhK@6RHv)@e;cKjims0rWqu zg$_G(kADi}y*$otxti)I&Q=aM-GHVIg(%EQM>o9Y?w1P>%Lf@}L(PyWf z=->Lel!1_q3W^sm4+7;hjgJw|Xe6SyM4NK-4mop$TFDhy3g9n0y73M>K=*IlXKM1f za&AoTQ^yyY+fsG;n1-+(fRJ$8f+W{bhb@60Xfr_xUz`qc4FC|gCh~>I*Dd3a%$l`` zOKuu$5{37~C!#hWM#)F?*$bUJo%*avYQK5i+WXo@0|3AT@@zqmJLSU5q;x`EYC;Rv z4;=FPYe?&_q3H(kaF;t5-3aR>tbaxCNiH($MPFy=^x`7zt z*9yXHK_B?us+w@RQ=TVZiWmUP64gMC;O(qCOED^H4C;< z!>0F(ul7~kZV;$`S|~8!j~LqHkK9eeGgI!^1+feph1C|xcQ3B{BZl@OUkJD6Lk}B_ z3-4w2tIUtND8C=zEr&YC40Q>A*1Ifgp6O#-Gh8-OWd9mMe!mVE@w}NgYYS0iN!y{f z(=t6C>>Pt)PvHDR2u9?PSHfFjKmRPjZNu`CB|@JPuj#>$esHA*K%}zTnH`Vm9 zc0l{$8RO%&@G6vZs+l?SK!eO3=cuS6r)SUz3PxluKfqv<>l#&m{6ycn(M@KOlZn#Sb3HNMK$=IXfguXGf;3R88f@h+lmr zznziTVJt=gFcq>f(3O4paOhxVpap>2CMybHWQK9*O7eTTr{5?^cYK+?ivN ze4YUQr^+%|W+VoauDE?-Q(jW5<7sPT$b|FdsarVPNy>&()o0#f zfFqt3UlXHu{6tG7F0GaG+<4Y`F7KDdLjHbeIww~7>A}4PMxGYG*RG!tpYXq`x7|g~ z!{c9KW8}TpqmtK{j~G1tISRNGfCkd%-s43Yt-e-l)Gk5O&%~qIH~BYE%y$C zr-5}Tmopm0Uc$x;IGU$#v-1U10Hdgo2`yq`ER0h*FsQ}@;sMT<;El|Ja-n&#UkfD= z-XlJT2c0Z?I?j4(3OXSVHNCpbEj5|#DPcHkZ}SU#$H!4wm=6cg|K4>S?t3?wMhq>x z!dV+v6!~Jv5D4!Am}`Gq(zoMq`Hj zKw5kE|D;anPh3qj`Bn|K^8WO`Be8xWT_|Y^QYY!?^j8|MMpS(hE)PWu_w7|0WOUhI zf@#puweYsP0aY2xL%k~!)8-)^DCl3K#efMpb6?43*bhdhkLs`ZeoF!zlK@WOUlh%K zhFWB`IHo^vXk+}Xs?ia0>EypD%!Q9xX7`?1DWX;!+sG3fUE5<08uWpjer-fuqe0$_ zf0{il_037(9zdoN%x$z0;`1tb8rE`uJr;d?O5nTN_4Kae-!1>nMtEx~X|F9CC&@0C zU%ydQ%KP8UBqaSVG3sP6oo6UzQ0ai9DKayGswC)`rn|r3z>);A$+cbfoPQ3T>TUcu zyu`JD+N`iT*4q)#Ht@rJ+B#3N8kv}2sw8Mw(dt|AjpIu;4u=(k*&eT20Zv;&PzWKI zY84VtjcCk_+-T;KupE|fru1NuZzo$G$frnGBb|epqO9K8teD_3=g`75jmK47NMBR~ z7NG+(W^CpB{F*wBvZ+6Jy7ajKDSL`E8y%A|R|P3QsDDwp&_AaA8bp%|6h5{<3s9kn z;6X|1@mhs!iAR?-3{JWLfS7Ut3~tslrtJC8HO~v8P3`8W);568mJm$#51}FfW<&^_ z7oek8JN&u7*oe{h&c~VWx7QZ&4exhHW>!6tQi&zL*K*L0YLvhko*&z_dl~h^$(a;( zR8`{eC1!Ef*@0(35ga!o5WRiOkT#@wWCVKD{&SbZW&2iFw^zO?5(PeqY)dHo5%s1G z4RC9?c25#NU*DF1H-P^Muq)8slu9m9zn+PoE`>2Vmezc4fTkl6{|0ee;l*)?^~%Fa z&aOI|PKjiok9I<>H&}+c-$q{cOaDl3{eIP&P4UqBU4Z&GPoqIcrB|699_kpa$E0+7 z3yE_!V~D|lWKu9ef`^WIFc8b!H2>3~v>5Nc?`ieLhkU#z7|_gEZC z-s5*ebQ^ybV5#12>nyG8{!trWliyiOkw8cpaO^-Zzaxd2OM9=+m#rb`@?EhXUWjwQ zN6HlH4dc2kPGF^amVdq~^|98Pus%|>Nyua)k~92^i4sWIsEKmD^H4(=VDWs?Bmzy6 zj=omt6rpVK8RZcFwyvl2_;htYgh4H#f_Hm8eBacEN=wm7f93IAV_RKYVI4m1`@Q+{ z-GY1fuVvzou&x7MI|7d4=~F64M-Nv}Ulsb_`1R5HdH0WL6Ganl&TMEMWifA^J5YGS zJ0v~h4EaZ$9v?FB@$-_3Ele6@JkKWqPqaKOC_xZC(pa0t_cSH*7A%Q0hB zRWQ2(<=Q+)U7ckG^c6~f!{GA?faH24=4wJ!4pje!&SQBHYUmDDFxu$Df8Met8XaG4 zb8{y2DgaJ9!VCgB>+MUZP`##uukY8^IbCV2sA-CQGKF47x%R@(KUjqA4Ouaml|R1x z>2K`2025z&7K zx={26F=9eb?iWbk<)m(CbkneAbs-@!s!0liT+^k((!6o>xj#@fb0Q>=X~pIe>rL5p9JP2_7&x% zC(Gn&uA_Z_DP|gW3u?Tw952rt?oJX38dn^-F#L~V9zsGQIw=<#4xvgHRx#`)uh~7@ zyN1@uLv2A6T}NxjqPd8c48Qg2ugBv_h@o`E(1t3};lnS&Q?)(p&W0=XP@ z!7FM2U^ob@z}mH$l)887#g_hIByyLoFyL1Md^SL7Q#5giKd34H-5odo3zumKuu!25 zsI_1Kz3zD-R&2~s)}!4B^Cpd`wB08z!oc>GtK+Ljt}c(rMPZ^1j{xf$%n+Zu_{iEt zYw_4v-;Scs9(EpXGRWk*v-N8?>W(yL1Dz4DR(Vi!`LI=08`~a$wgU_nr|gX9Mkcvz z*m?ocJET>>Yfr%;3a>Ksx2eO7t}j5v1utYU4@+Fu?&`m9lh<>XLm=fdm)oqS*vCcl z+er!@O`UApn;Y{mTlTI_k1fw5zP5_vT)Daj0rLqoO?>rhr;9a7x;RuLbKGN)Z`V!U zI$LP=?$2)2N{`Mv8=FJ7G$kHXJ-ImcmPPaLro|eQ#hVP>?$W(RMbGU{?XWO&13XA> zG-21oewLO)wT?~`>sDzAe;&w+O9vlUgJ`YsT3xd-a?&CqE9N}+CP!3Q+_s6@8XWqG z{nFscMjPi2sWaBD>SQ3K05Cf^kg?<111Nz1QGaN+*H~XKr_0gifwUbQTc~$SKWjrh z;&0q-Y-{Wl-d+12hA%PoTYui=cb8&PxLV`Syy`^)SmaD4*d?) zI{D?L@k_%{$~(I0g|b7;r?{p?3fSk&u#a^*ww3h-rs42u_ba@ii!&=*4^GINZ~}J3 zQW2Jp`TSslXR_rQ58=tCW|o*g1o- z)|`ctT1wgzdRk@go4^@>GU9-4VV6bA>jI@pCxEsTau{;Yv1_> z`=aS%n{u4{LL{zi1JW`~{vHoP&RudA7Ae*_b}j$lU#SukiiAK;LJBnfVD$R)+QY?^ z*kA$1Z0BqPqEa3xZCSIMJI@k<-e@aLD zF+;YBwbO~L&)m!tntAvBCpN>K8krZIa*Qpk`4?HG_slQ=ryMwrOzsJ!PIkITW);`9 z0AHTTJIT)mp7xP1)^;aAC3%(TVRv_~oK6KB{`#wVu296qECBzT^fK5EeK*(AD+?N1 zb&FHJag=2Xi3od3OQe@G#nMFT<3)Md&^6dE;eI8LPjL5+5^h2B_Xg1^ccOVpRc{dF z5+;TooZ^hk0Pw%X3_2wKSY)qj(WKelueS(n8#<-J9^&M7+$ z=SKf=;YhNCZa(^fo{!Eb-`T0S%(Z#5G2n!jq|^Uwu})&q3y&@1b6Sd5M5=mZnvf8) zx3niv9)kr@S`oY6gO4hOjWM2MT}#L|fl?YYpI3^`WlN6Q=x7QsjW{wPd0=*6TUeiG zHP_HyE{yPu3YI-7yh{~hfLs$<@Y46jK}%wzSFS*_rlG+_dA?8X^=hOm_G=CP+dh*q zxSn^mYM(4w%d=3u4UON9{YBUzfZ5o@%-u&)FViZ1{|4soW|1Z`YEO`%W8QrjhNQIQ zm0qMod|8q6j^&Pv?0(6-j)FFuQ?+!@+2x_tLY>1RYPQj41q%5Xbz!V&Uq?&8SV4Wr zizWQuNjn}9*4ul%420xD72IpS9ypeMhDWfv!xLeR`ZAsEOh=bGl$4rCGwY|`c>?Gk z6{5lKe$BnxH}M37HNnkGNlY1lEg7J~5gGjZ&sK~}CL!zO+FIkyo{vwN7mh%A@I30y z>Do;TcGt4GnDj1r`~jzJ&4#lcfNl8%@ zvSjAZDsV^%U^arFCk{w*!@tEl&zC95sDCc8z; z8zQy;r}(eH!a!1-dcI9!>A?%`H~uW^^+y9ieO49pI-;o;OhNu?cCjk@htbp2 zNuAhj8nl4M>s#-u>uCx!Irhl&*4EJs!0ObjiELS=2*f$Ge zfg;>z`^kl!J};t5f6a$ZpJhWT&v8VwlMhyZS(P#G;Z$42;(2mx6tH^sY05Ku4`ls553pOZ%%@COh{0l09tZx}icvadr>CE~-;hIe)Dg_qIVf1GoZXDPDW z2e5%SWNiAKjmC+`QPCUw+YI-`B^e97(U5{P&44g)Ofl!CYbC OT{m@gI+HDZ!T$k}efzWk literal 0 HcmV?d00001 diff --git a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/columns.txt b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/columns.txt new file mode 100644 index 00000000000..abc4136c586 --- /dev/null +++ b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/columns.txt @@ -0,0 +1,135 @@ +columns format version: 1 +133 columns: +`WatchID` UInt64 +`JavaEnable` UInt8 +`Title` String +`GoodEvent` Int16 +`EventTime` DateTime +`EventDate` Date +`CounterID` UInt32 +`ClientIP` UInt32 +`ClientIP6` FixedString(16) +`RegionID` UInt32 +`UserID` UInt64 +`CounterClass` Int8 +`OS` UInt8 +`UserAgent` UInt8 +`URL` String +`Referer` String +`URLDomain` String +`RefererDomain` String +`Refresh` UInt8 +`IsRobot` UInt8 +`RefererCategories` Array(UInt16) +`URLCategories` Array(UInt16) +`URLRegions` Array(UInt32) +`RefererRegions` Array(UInt32) +`ResolutionWidth` UInt16 +`ResolutionHeight` UInt16 +`ResolutionDepth` UInt8 +`FlashMajor` UInt8 +`FlashMinor` UInt8 +`FlashMinor2` String +`NetMajor` UInt8 +`NetMinor` UInt8 +`UserAgentMajor` UInt16 +`UserAgentMinor` FixedString(2) +`CookieEnable` UInt8 +`JavascriptEnable` UInt8 +`IsMobile` UInt8 +`MobilePhone` UInt8 +`MobilePhoneModel` String +`Params` String +`IPNetworkID` UInt32 +`TraficSourceID` Int8 +`SearchEngineID` UInt16 +`SearchPhrase` String +`AdvEngineID` UInt8 +`IsArtifical` UInt8 +`WindowClientWidth` UInt16 +`WindowClientHeight` UInt16 +`ClientTimeZone` Int16 +`ClientEventTime` DateTime +`SilverlightVersion1` UInt8 +`SilverlightVersion2` UInt8 +`SilverlightVersion3` UInt32 +`SilverlightVersion4` UInt16 +`PageCharset` String +`CodeVersion` UInt32 +`IsLink` UInt8 +`IsDownload` UInt8 +`IsNotBounce` UInt8 +`FUniqID` UInt64 +`HID` UInt32 +`IsOldCounter` UInt8 +`IsEvent` UInt8 +`IsParameter` UInt8 +`DontCountHits` UInt8 +`WithHash` UInt8 +`HitColor` FixedString(1) +`UTCEventTime` DateTime +`Age` UInt8 +`Sex` UInt8 +`Income` UInt8 +`Interests` UInt16 +`Robotness` UInt8 +`GeneralInterests` Array(UInt16) +`RemoteIP` UInt32 +`RemoteIP6` FixedString(16) +`WindowName` Int32 +`OpenerName` Int32 +`HistoryLength` Int16 +`BrowserLanguage` FixedString(2) +`BrowserCountry` FixedString(2) +`SocialNetwork` String +`SocialAction` String +`HTTPError` UInt16 +`SendTiming` Int32 +`DNSTiming` Int32 +`ConnectTiming` Int32 +`ResponseStartTiming` Int32 +`ResponseEndTiming` Int32 +`FetchTiming` Int32 +`RedirectTiming` Int32 +`DOMInteractiveTiming` Int32 +`DOMContentLoadedTiming` Int32 +`DOMCompleteTiming` Int32 +`LoadEventStartTiming` Int32 +`LoadEventEndTiming` Int32 +`NSToDOMContentLoadedTiming` Int32 +`FirstPaintTiming` Int32 +`RedirectCount` Int8 +`SocialSourceNetworkID` UInt8 +`SocialSourcePage` String +`ParamPrice` Int64 +`ParamOrderID` String +`ParamCurrency` FixedString(3) +`ParamCurrencyID` UInt16 +`GoalsReached` Array(UInt32) +`OpenstatServiceName` String +`OpenstatCampaignID` String +`OpenstatAdID` String +`OpenstatSourceID` String +`UTMSource` String +`UTMMedium` String +`UTMCampaign` String +`UTMContent` String +`UTMTerm` String +`FromTag` String +`HasGCLID` UInt8 +`RefererHash` UInt64 +`URLHash` UInt64 +`CLID` UInt32 +`YCLID` UInt64 +`ShareService` String +`ShareURL` String +`ShareTitle` String +`ParsedParams.Key1` Array(String) +`ParsedParams.Key2` Array(String) +`ParsedParams.Key3` Array(String) +`ParsedParams.Key4` Array(String) +`ParsedParams.Key5` Array(String) +`ParsedParams.ValueDouble` Array(Float64) +`IslandID` FixedString(16) +`RequestNum` UInt32 +`RequestTry` UInt8 diff --git a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/count.txt b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/count.txt new file mode 100644 index 00000000000..9034d8fd0a3 --- /dev/null +++ b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/count.txt @@ -0,0 +1 @@ +8873898 \ No newline at end of file diff --git a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/default_compression_codec.txt b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/minmax_EventDate.idx b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/minmax_EventDate.idx new file mode 100644 index 00000000000..0cb239d932a --- /dev/null +++ b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/minmax_EventDate.idx @@ -0,0 +1 @@ +?? \ No newline at end of file diff --git a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/partition.dat b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..9c176950d3f897f5cc9714d7011107a21e47a229 GIT binary patch literal 4 Lcmdlj#LNHy11td1 literal 0 HcmV?d00001 diff --git a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/primary.idx b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/primary.idx new file mode 100644 index 0000000000000000000000000000000000000000..9a604fd108fd6ba7f48a9c05368431b9ac5fc126 GIT binary patch literal 15330 zcmX||cU+C%AHW-gRMK5din{yWL|M@jA{xX|T1r%;G*qNQi%LclMN5%UiH1s%G)1Yb zgpgDcz9kKf-{*O*`}_0#`pk2lb;Oq^&B6N71LTr@Km*K93|N|JcPLgIIDY7Cz7mSoC8YfJH5ob+pVQYGl< zWj#)-Tw}++htkoHCJ_-5cGQEsE8UGr5CwBEci#}s;}Ar{G*fPL=w9L?`U^a>l~rRZ zCKuW0%xe)5dF-yuG}WUMJk<6Dmz^$RK6X^+AkA((!ucenwaF&*f8Kbl1l1zXbe z`TUN{Y=WrRvK5o(pXqW5qG6m3bH+Er4Bj2XXKB^^`)`%2*vLs-6t0oXLFFsAzc5jw ztSH3uVAU+7s4~z~HBr#iUNF%pww;Zt7mC6)y6xlk@@y*uy;+89*3rzc0b>R#U4_fe zl7IMQDKk-}9!}LWeYeRMGYFz$*Ya7S$!ED}_8MGvbkSLCnKKjlnv25Sv*en?&=xZe zdbt~qHz(C(^W3aWDSn&$3UxU?7e!?!`}L z%wyG24*DI3XQrLv(UP-?fr{dBFOI(^yXAhdkY5TOVO>%a_sSVI3QQM;t0EOT8W70m zBa;lAAvt&iSBtKLMLarMUyMia zKAKQ`xs8Pw^>}6pmvZ`T2NkV~yab*jMx0~x7{!H6ur96Bu0%|W|3xTeL()rAHxnP^B) z492%I-hR)c95ymB!0Fm_XS*NT9Q4^1x67Z^{AMqci>e&OV3r(BmR#-VCjM4aN??R>o#2#AF3FgxM=uA4!~$QbWyw%L$5iy#U% z-DA0Z)^s*ORP2B9^ermy%y+=pMzi~2`DD3Eqr6a}&t|@ZN%InfE zF7go(huJ-R`{HjqTNZK^69<`A()jG}jXab$8<&|e#*G{!S*V024qoi)^FDvcr=#%& zIFYxm?f9+1KsigqA?w)>0%qRc!bVG0;dUZyCtunH7D4iu>i5FYBVK%@yg?jBG^Xl2 zQf~!MoDoj=HJWylfjklJQ{+Zj(fm?6YOus}Nk_)}+Yj-Ot~)MUZ*y*3n$AGo{y2%| znsgOwGthzr+>7E}O5ORBOr%_i%kWwHx}1(y)r!Mi5c2_BP&3R!9xrj9W}8GG9IfJ^ zn_qFeibuNcJN7V;+z3vbA$5!5cm~o~ISn5}N`rJ%01FxVPXk%pdNC=lBnDd1G7Y@E z_YNO&sN|tdo2NrOm-sr(eMji%bMbW09Mjmf=HpHV%I}#D^O7`>{_;>68_n^Qfcukv zS$zHAN_f9SNkC@ztp|cb;JusmLjvr&Bczx4ZDgXfD>ER%^XtvIap!o*Ky)T}Ik{OS zKMLM%{&p1@9!HLE@L8_g|i?RPqEq5iH(eZ#NP6F)#t533bE_3 z(^@K|N0C~giQV*nn%)nw>ywc}>iQtwXs`NzQtunN!xs<4s}z z=JDtxEJk*pr@Nt$zEvg0w#^XIroS#FvGCp%XyW(M4_84hWGu~ES9+|4Maallgw^0u zmxn@sArcx)(V|VhY*+WKVw=CwplLiDuLH_|9x0>7U^JSRErESWk$IDoXbX zyucSWVavw$Iw4KS)q8&T7gD0ug~VXQB_dFSM#HDrRxv4&v2q(J2JG+DZrfJ zM8k&WX^kRHc-P&)HLqFbs!48O5=6ls{q3L+>%&_SWHce52sCNfmmt5#aSsJDd2ISmUeP-ZO>~q$8#22jYao~DO#eqlN~ag) zrU~gnFJ|iXMM!0A50@R5LaMNs2&pR(Qi!eU<&fDziuVXEinuMLA5q57m#-Gm!IBm` zA^F>O=bF?BNmb!%{cIJX-JRxwE?r{*k+J^7qoQiCM!{!-j5YuCQ;CN4r+|`|o9foM z3uX07EG$`eLVEm4&8PK>kZv8fiJ9dsB>X8Ry%2f{BI{N)6$)kaa{CW{93hcA4~`W# z2s9~}U5>vpJDE*Hr(hZ%Li8IJ3F(b@qeG3JgiLU}|4cyb0`E-0H4XwtBE^K^}e zoP;v@-E-V$ED%z>Txe(QF!G`e6Uv^ZvPa_o$>cHdC%fj+=kSSm z^4Q83tqRX}m>09r^Y{4uLSfmAg#BQmWeQU892vVA$ObX#|0uNN^^WE66&1*;I%Ffw zV7(+D>m_%p11$tZ!rXQB#haH3ypS-zo^?j+<_JkmV`<Dt1GYM!EoZ2(Aef_t{DHuaLr* zUcYG3C!~q`dr!O9a|ulvrV_xKlg{P+BR77gEp}NbvtD@q@Z}pq!o8eV7HYO{j(=UM zC?I*Pt7*I1*l9lDMIQ6D@Y{X7NJyfum>DyZgjA|$*IhHtN3zTDD;8sCZ}0GlP7nz@ z9L7_90&f&tM#3aMwtr1N!z74|$b>06U(`RK%I{HnYQ{owU2l!cnV<8$_G$1LxYUR)GAA_Z%y zFy>|V>#CaQ^;&7rv|ikV>Nm1bZ>BV4J!T?qx!;#gkUSRJpH`!Pg^%uaO2gGWHrsqF ze<24oV=|!G7`k|Q*9aY*J|hDW-e@Xm9%XS+!$TSH`ONq1K37;LGiznQ4u2DkC$iDy zFq#D9r~^SD+rmM;0J{p zcxbhlB4}Qe`=UsfXZ)kJGnXv?IYs|R=PX;$$rvFm^q1PZsai-0tx1Qhl7%!~MKe-Y zQt0JoEG2cyOehlxZOfI0%7P%C;_pRe*XIbS#8a-20_johoB>_2`T9%{#dpT0Jq1E)1P*%O}VAp7^fXJBZShk2`5{s}Ce6<5ggygp4 z>9W{mLfVJY&Ar8hbYVnqU-54tneq_CE!gk+BFx~H&ND8n;r*d(No zqZtWiH9`uIzLh1NCM5hm+jU;(C4P-av&}Xk)i~yMROurh`aW`io`{l zXzfh=-o0Csekbq)1LZ~G&);8Tm#;z7V9%U_%XZ$J@^n2#1c!!;jLOYMtyagg1Z63l!^?dFjHsOe;FR)#mj`Q2K{ z^Om!ai;FTuxOFJ1W-Nw55DnYh)cm|KpNDQgQU<$|6SmL2B-toETm|g>)i2QdbdYDGt6?0Vd2V&JHILb-=`|lR+Y<5NXyQ@$22d7(7n`SQ!TE-C zG;cr)B9uoOWJQNF(SIsSz~}v1&z9P`T=e0+Hsqqwc#F$a<)JeMOTo_EU$&=iA=K+4 zRsii-sc_Q#I34X)TnUuFNJ*!`3`R?k#o^NjrJ))lppA8p3Nb7{+T$>;EmhsdvgRb3Z^L4LeYku8?H&gzLu`o<0-@WN3nNr6)u_(%Jv(7m~j6nAS#xl z@!a6D2ba*KVj&K<&yOw>lDF51Ps^kPM8mc`h}Ag{t8F|24WrcrU%BvJK=PQ3-XaML zSRvsud2BHjZREaE^B?_IxswbZ8ml@< z7iXZamijQ-KQ8?aTWgr;%sZSUu1{V$qr^ggbk=}fi&6N(x_KPr+_DyC@aX6-+L;ap zL1Zj3uhnDkFDCk=Y5~z=CP5jy#ZYJZ?W%USBG$r#lU)q7kD@6UN_XeSG5>I zgo4EEih)5UdbV&Q*uCA7bb4|l8x6K^0+QK!>TwUBjeIT}gXSkIg|`S>AU-d&8u0y^n_!K{YZEuBm!T-6z{SsEUH%$604@K^A z2Tl9@pfeRK;2CT30G}b^Gb9SNIB49`3rJ&Q?o%UWHp<&|5WL{0LZ{xc(3F8U*e$Yk zOX+|$N`{IrM6Wcd?&0(@(6_EbV7F|^UlmdU8@)c~2bv$r4GI!ab;pT<{aKq7^TU;m zZY}T!yYw-}cj+J|`dJYGG!$TC)#%AUep1K4E<5izm8{P|llyTB9cfQImq$m`@gSIa z?#243*G z&4ULlbm$XKYG{0mDpdWfl}k9ujZsp2!l&x?QG(V*M<@k~7 zex{Ji8?|V-3mzLwyYXl|bnZ(uL|7l-y5v0Uh;X7}jtLsik}q+P``j2Ha^s#NZD$T@ zToVhLf4 zG96?U2}zLkvm*{0BUUg7B4MEl_uIb&G6|v(&jWrsfNRQQHqpzHqGdJFt!&)xWq?A$ zE;x6>iHt@6kyc{D+Xg52{;hj%S)~s5H4qi+Ej<{$396Gg(Xgx)wBCiV?paxe%QS5g z-nq%anMV0#u&d`(Z{Wb!LNzZLG;0HnlIFvDsdIA*Xx`LJRMDRWdy>Qyi0&)5x+4!( zGj-LeAUonZrRoH$yxnbSkmr+;8PQv%VDGy<9c0tv-9pl=IVjjS9qblIto82>p%X;H zbi9pAx-cd|6zqM=1_k~on;k204ad2B+OO)b&jBSY2|$g_%5__VH?jkXnK zz|2=m-Wl;e%tVpfGl8h%JPmnxW8y@^uD88fPzU>WmE$PhJNv?JLTDjMVT@ z>+p5(;?;iA^QI#g{ms7tnvSc#-I9fqpSHrAK>OSVt1b<(P^(lnM9BVT7nZ?>)5j&* zkY_sYdT;t}CP9!(jNi)mFfJ;)oDH*NtbR&X3+@S=s07u+NpI4RZ1CBfm7s2r!a{Fj za)8XDE#@V@W+4xkT$mU9?6WGEjUFw%4PLgE++$3@sstwz_IvFgZ3S4n;Y7w9BDc|w zcC!egV4df$s5q7g?5LP&@oI7(R8??I8n#k7WO0}-k097<*BmRlUj?V!mvNtsqw61S z|II+jDYqfdkLGSx-=QYAxHb=>_eaJ$h%baYgmnkTeyO-(Q1U9+?Ys-odwA<-#lbAC zm&*q)2{&aL`%e^8sf4T( z4nHLwy2C^!_bb8Yr4?<<0-+*-6BVn7wmrKz9@f{Hk3o~Cka*^6B@3N3tpZIEIdA_@ zKj{QPe+W=E)Ykm`t010cy|c~UXeKIsSPfoYY|oILZNovoK2(F3OC`Y;GoG>tqG5*T zzSYkv;3A19H89tjbHCd#y16Jzt`73V&rsMSEOg~}JzQytJbOLWJAAb3QUgS&2x_=| z0^Y5M^`Aj57shNq4Xe?S-LGfh#b@TRI&~HgF}Y3P#m9e*6&<=hzLhnBtYjnauN|9* zuAO-fv`6Xb#mx`>u_gL*1rZ#Kc%{vbx?ua^zAi7 zNa-;fJkI7KbJsQ)QKL^-w-cQF{5|v*ymXcfmCS}}jAqPR7~ixJ$uT_+bdmtU8q7Cp zk>?8z%DD3ue10A}G}a7V7*9X8LuUH#80#bo=*WKGdyp+;B~R{$-Srx^50DGKLfQbU zAX{z+Xx{(b)ZRXmK@ijc+cI8Vgsu;7^A3pTP;s@^9M-fsb3OviJH__ea~Sq`2R?$A z!!bwJ_dkHUp!Xwq!S`btin9nA1@qlOj<9+v$p1%i*S43KkMfa~_GgIa()L&1h(kw_>`oY; z-Y|X0H4@Hme{_N@s~|1bPlHYn1#69qc~RUakjZ1EYl5vT*YnZdBVRzXb+scmz!;v} z8C@{fT8$~{y?{tB8(lRMPZVV7ks)eRJQT<@zHoY9_~(*v3- z=C+KcmcZ^>q!;ETcXn*1(g)}u>ch!r>}Lgigo~P6zrlz)MV5_Ene!3pMIYELDVbG3 z+Q~wFv%bTeos|D{#VeFS5ZLWzd6XXKAaHd)@+XCXJ0U!XZRD(@ zYH>g}9gWT!1TXXdE)Mzy`{@j~LAWYb0eKWFR}PAr7z8`KE5ap{jS8v1LDPcyb-f|f z*>Qr?|2N;&9z(xIrpj;7OsJk?FI~e%Z{mML1h0PYG3o<4TJYdE_`J}WH=R3=i_V4( zftM!rLyz-CSg3Q|AJCMXIu?9;E(1xt`~zNUO~+`)Ob$}~FbtYACZrQjRWs3^CnG@3 zmdl!2nNW4890j}866!ld&(H}XW1EL~-J7A-h09>|v?2T6>y_{}9U23gsmBU6hIa^K z`Z##Os{p-+SxAFB4ziszr@;AYJm`#?fIM4Tj3s=bt7UokBwVq=+Hd(Tj(qgYa0+PO z+1ThbrY3qnJ_TOdm{{QO6cZImh>F4)JKABBew%^nZ-|Lfup^;%zYj>j&Zbx#=$Fq> zeh7t)ux4>lD)v)V{ZAp(m(Ijb1NyMx-fA9)i^MCZi^^m7yhT2LQR5>cof#m*&w74u z=c2TM84zKa<(0H@sAQ}=GZV;+@zFYTSU_-A{zQE{S(!t~sF)~oARgN)AR6}Be(x_y zSnu8`nkh=gbWS{T?OF+IjpkXRa#-fw`nn74bb`njb1X2)vLD``AcM-foJ5*4^g-NK zmIO`lpkK#s!kz>tc`VsrPqwEvAN97+hFthxkFE^;1?%1aq`=Pkj^Qkm%^Va^E+b0A zMxO*r9)K=3s^=W=Qup%xGM|sImjR*>=eIKAY^0Dj2QvFo^660OX%;f}kOiN|A8H?5 zxPpn+M$ZMGH|ldz+AX+fX}uh1zFz-VqalEWPK1#mf|Acw>%=@R>iS552p4@NlIGUa z38E19wGOOl#YCwfJK$O{Qge)sC@X0|J@z(le?OulF|IuL>~@!kVCX^JMpgl21FPpo z;Po?=9t9ZfHRkxVM-lLq=~M(6b5n0+4fN@7Ta-YS`BLxk#VI(c->3pvH{@q)ErUw% z{ZtjmdiwNro-U~z^!La-813KTxMvg7Ip}oXe6YK2x994Ua6W32Qw5SroY&Q`h>z?o z)F2m?(vb0IuV9z_YXQif2koc4O6H>+8+C|&|KqRRpOtjvbyOYV*$3*X9lyvx&t7Uk zJV(YF%ct!e)agZsc=f!v3J+-xN_A%dE&Ao0{KyD;HT#+1bJehE@K5L;)KFuC%vw$F zV|*&C{9D)%!GHX!RW%RRgZ3QA<%d@HC(dIyQF+XPxlUDjU$Ny5tZ6ws(4@FpH-3<$ zquFJA$i>0oai%d;2pp)3K+`j9Zs>Ry7rk7h1rh3Ah@LKa3jJ;qTHrI2GIC41n~7v< z7XxL#?x^#-$wGf^w86`b;ZLO*P^I!W(}4(;12roRTId9ku{x!u?QNim%cxkG5puJK z&H$X`v1rA&AEm$WQGOq8cOd+&!e!Vq<3z%^_K7w>;9kNB<|VyxMmb^=1S_kKL38nN zE$7&7roNOV6<*} zJ=5`8-ikV$q*iggr7D>O3A)1`!j33Wd<}Rhs20n5Ig5|Bty%+KW}a7g`V{U7@e6BV zcDtgnh`Ev+bZ?IVzf;NF>oBg%=?ou9-%rSzQ-@e@3 zAhVQ#)V7(x>}qLv(08OzDMP8`jJp= z+P7dEcu9@?NtV}RqC+MKsBQUO{`TceWcn3BF6&>AckVjFM~~KZX$s=Ym6?}X8|9Z()yTh2nK0_{NaeMg^j?mGtZ zklh92>-G2hay}UD7fA;oFPD%-hB|bDNSH19LsAN6{v(oaT}y@w^r?a- z^n;Cx``g1#11Aa=DWbP&Q;5I|6&tMco?8Q7AY4Yn@XEeDsRq z4Dm7(mL~j3p`$+gy)bsXFMTQ4Ex5W5znY!W=P$tPUU~qCffUhTCnpad@enfM}TGz==N-$Doh8@c=~U$t>No&X14Q z*SLe`#o-nAHT>cGSKiA^iOy>AA-yh#LiFJJcp`-#$nLR z(R$Y$v5t%OF7SgfT;T4pN`aHm>mQGR-D9nx=GrCDhimB%@gmC=zHfNXLz^oCK$f$^ zafj?E1C{R%1TSe_PCdby@N9k!gk1Dyj;-q%;iIA%$6*XX{T?x<|G_>$F$id6f9p?4 z2dGP_1%qb!{55sa%9`k@XE69|UsNXRVZ}tQDIq}k4rR$qCYnwUgXs8^lb+2*Q6*sz z4?n|P@`Hnvd6}9xpTpJ--x$(Yloemugph?04 z7oS+V^fGiX5R_*|zo5BLNHS8|!+wqeO)~Z|%g12p4CrnEO)@r=`zxWhS4gxP z^6i}xhmgTZ(^TY>X;6E@?cgas*>G=dGLIm64DYKHhrZwB>$t2@G{|r@^vp8qqaZrt z zY`?~XEbE21mlBzS%KZ~y=KZ*Rh8+m{=Y21LZ0nT=N9&+ZM`H@7_|0xf(Hl6ZwDclq zYRZNd?D@z+o?=Om=lu|R!@je8RCYQEB3N8vlxzrP5+si$+L(0yfqOph`z4U6))j6D z3x^X_AbIRytnS*z*?d&-@iNGEB=0+G^q7G*olXH#7d6g^>4!6-fmDdckz3Of3bmQT z!)cIVH$ zs;VYK*BmZ`9sBcbDjUaG1W~a)oWV_bhq(mNu(XEh6_Hn=%D6TYqIXlvM?Y=npn8w1 zK-#K9f2To({M6_*@PePx?16Ji+k06+?5W0!S=*pKF`5O@t<1N-dXr8^H`}j6W_Zt# z2y{j0th)hOfA;UR{ys|+g{a*GyC=>g*k$OV3Aut3{r3r!cHX7XC8O z$p+F`k$7bPO*WdbD+jz3Y4CJzTJTZF$y|sq9Ef4tp#o|7F&Fa0_fGA{V5NHVHprY! zSC$XM&PH}y9?0T+$J9u_IVk+Y9gq#Z+ntNA-PR@C1=+@nerg@sP!~?f2fACJJ0_;X zMQwWvfG*nA%-jY2y8?>(Yj=KXl0y*eLay!CMbWS;2TdB$@12vwLq{$aK%P|Td7Am~ z*GB*A0vN+^TAW<#H&`dD6~Zh{ZNB3j_KS`}*o9zcwRz%XPc;{v7%KvLQl56FZ3Pd# z2)GA?zXwxod1!`pG30VJT&Jlaii6_klz^9Om_xtkB|1Sg?BhZoXKn-!4cL}Kykya~ zixKCbYiIL)7;RiT=hvwO2J)hmf!(VYuk}piV66`X&wKWx#gAvR2!g*8CYKtmX=4)v zt5~HHBi(ckK{QNyQhiI02#+AB1bfnUwZa#@AhHZH^D;ZGoe~6fKEHDCdBNeklP~PK zR%$+g=zB*eZFAP%MpclRduT~loiP`gJg9=JvfBQhqFx3c9XeeNJ}*g!y_Q@Ot2<5l zV0Q*Y!w%Y}s(jweL-FP{;1geo$7#R`&f;3o?EJJOjv_|JN|~rv_Tzy7l?!{`NPZ={#$oVj6T1Sqc)4ivsOZ!M77B9z0J61Z1D_tO z4Im#yn#p^<7_p-d=R@It{&rgNa{Y=0zjE%iEshAZAz>*rjP_%gtT{_u!7N zFotfq9r-c-OjL5S2jbm}MwX7oJhXGN2V}Df-?=#XFbRSR0mb=9-FG%YG@{ecAD$dp z^ zUmR4vfS@l3H2YHp1gBS^xo#<(&GWy(OgxOKjEX(ZMjo|&5S=d>wBQx=N9I-ZgRE;4@HoD*c1Fi$!r?W4Lk1{^|1kLUzQZ7gDvygt_FNkNg`0I($TO368`42SxzOUp& zJ>jF!=>tFq=5jPwUWTus=>TLs+wSb!U2p;Z_rIj+wc?zvIc=}De7-EuV*9b!$Ht=;x?w+!XFl1D~7Wr3!=kL1I{h(!q>2&0Jn=W zNG=Fp%to;XM`omLoXcXf8cCvYLasvxp>l%fz<8`UwwQ!2~XAG$2Wnsa3 zX+Fx9`3vKlrtY6O*#duX>HY;f{11y;1x&Q6_b-e=Zjnv8E9}m4NB%-)7p+6OKAzy9 zXW`>O>yxsU|IXl`3z_4P3*NcX0Au(_nE)@wy~|p|*Rqi3-3f@WiL_5w@+7>bDU5*8PU4iqHu$jQ5C{Da>=Cba literal 0 HcmV?d00001 diff --git a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/format_version.txt b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/format_version.txt new file mode 100644 index 00000000000..56a6051ca2b --- /dev/null +++ b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/format_version.txt @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/checksums.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..a72e2bdb3706bcc08ec2f7d6ff976c10e56f75bd GIT binary patch literal 260 zcmV+f0sH=AXk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$cMSjn%S#tikUGF#tP8}kE# z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOBB}>nT0?1>@KmfG%gdU6%j|WMOn+E@EkJxd)

ZggK{VRU6KX=Hc=Yhv{9L4)i+mY_Q2(jfMG01a?q za&&2QX>V>WWMOm!_Mvet<#01I$(X>DP0c`j*Wcnq1^RmKF(8EcDh KC_ym>w$1=Amu)2g literal 0 HcmV?d00001 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/columns.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/columns.txt new file mode 100644 index 00000000000..193d3123896 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/columns.txt @@ -0,0 +1,12 @@ +columns format version: 1 +10 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`timestamp_ns` UInt64 +`revision` UInt32 +`trace_type` Enum8('Real' = 0, 'CPU' = 1, 'Memory' = 2, 'MemorySample' = 3) +`thread_id` UInt64 +`query_id` String +`trace` Array(UInt64) +`size` Int64 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/count.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/count.txt new file mode 100644 index 00000000000..7813681f5b4 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/count.txt @@ -0,0 +1 @@ +5 \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/default_compression_codec.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/minmax_event_date.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/minmax_event_date.idx new file mode 100644 index 00000000000..4b6c49f8d50 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/partition.dat b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..b9ea5569e3a3b34e3a1b6a1c4cbcb46f5e2d3fac GIT binary patch literal 4 Lcmb0VWo7^X0I>iw literal 0 HcmV?d00001 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/primary.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/primary.idx new file mode 100644 index 00000000000..99ab3ce7835 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/primary.idx @@ -0,0 +1 @@ +H-_H!._ \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/checksums.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..49d83aebd914bce7c8ebb35018eb910ea5337b95 GIT binary patch literal 260 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyUJ56aG_hdC|uFP3IrAaDDV{y1~G} za1Mw++Olyb=a=S{=#^BIFy_n&-TQLZLq37-DI9oG#gHUryHuyxGq-0z*~@5 zRFYW&bhjQ*5z9{=eeu7Cnoj8~>F6<*FiB_NE-1>(O)RPe>E>AzQ2A(+8nc=5L>~VI Iw&B?f0H_RU;s5{u literal 0 HcmV?d00001 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/columns.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/columns.txt new file mode 100644 index 00000000000..193d3123896 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/columns.txt @@ -0,0 +1,12 @@ +columns format version: 1 +10 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`timestamp_ns` UInt64 +`revision` UInt32 +`trace_type` Enum8('Real' = 0, 'CPU' = 1, 'Memory' = 2, 'MemorySample' = 3) +`thread_id` UInt64 +`query_id` String +`trace` Array(UInt64) +`size` Int64 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/count.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/count.txt new file mode 100644 index 00000000000..56a6051ca2b --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/count.txt @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/default_compression_codec.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/minmax_event_date.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/minmax_event_date.idx new file mode 100644 index 00000000000..d3715e50d46 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/partition.dat b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..7af4421fcb5ebc50aa06a5f4dee6e454020552a6 GIT binary patch literal 4 Lcmb0UWo7^X0J#7) literal 0 HcmV?d00001 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/primary.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/primary.idx new file mode 100644 index 00000000000..0156d831c6a --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/primary.idx @@ -0,0 +1 @@ +H_H_ \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/checksums.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..f45270c50bd34e0f4400456a9a0ac4304ebae307 GIT binary patch literal 261 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyXp3#UiEg{lBXdkG?GYv@f%}=_UgM z!+9Y7XvfBxoL`z(qE}K;!W{Rh;=-DOtwxG>m6dsRwjN>NNJ%V7)Jw|DJCdSa{OCi# zm(z|B#fI`%l-4CPUb)Y{Vnx12R!7vt_w^?Z&Do&ziBq59LtbuCw($j~;%~BCO{aYv z_O55y$Dp`Sl0h^#GcPx>B0jY&HLoNdXl|-rW=aK%{)#M*%!oA;KkjQ)eIr+F%D`KY zSX7c(0(7_@P!Y>d9)0n@hni05Ea~VmmM}?Y;4Uc2%uOt+1nK5c-7Q?0BT~9Ltm4Vj Jxg9xO3;;OHZSw#C literal 0 HcmV?d00001 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/columns.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/columns.txt new file mode 100644 index 00000000000..193d3123896 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/columns.txt @@ -0,0 +1,12 @@ +columns format version: 1 +10 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`timestamp_ns` UInt64 +`revision` UInt32 +`trace_type` Enum8('Real' = 0, 'CPU' = 1, 'Memory' = 2, 'MemorySample' = 3) +`thread_id` UInt64 +`query_id` String +`trace` Array(UInt64) +`size` Int64 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/count.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/count.txt new file mode 100644 index 00000000000..a21cae3648e --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/count.txt @@ -0,0 +1 @@ +454 \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/default_compression_codec.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/minmax_event_date.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/minmax_event_date.idx new file mode 100644 index 00000000000..4fedf1265b2 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/partition.dat b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..7af4421fcb5ebc50aa06a5f4dee6e454020552a6 GIT binary patch literal 4 Lcmb0UWo7^X0J#7) literal 0 HcmV?d00001 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/primary.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/primary.idx new file mode 100644 index 00000000000..698b75ec43a --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/primary.idx @@ -0,0 +1 @@ +H_Hƥ_ \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/checksums.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..5d9d47463d2dc3c588c2a68beb24126184e5955d GIT binary patch literal 262 zcmV+h0r~!8Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$cvWrlG#xSW(BO;NigHQ+OX z+5i9m&;S4c@jnI$V{dhCbS`vwbOb;rV4Zj{6Wu0P+wm(Y%$fiQWMOn+E@EkJsoeoZ z83XkA_OM)y9%i<(>8`Z_;I@6CZZ2eDbOfQp0V$_ea0t>|RE;3Dk?sHsaB^vFVRCscX=Hc|1aaTY#PfTb M%D$$}R*TMQ0I^wb#{d8T literal 0 HcmV?d00001 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/columns.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/columns.txt new file mode 100644 index 00000000000..193d3123896 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/columns.txt @@ -0,0 +1,12 @@ +columns format version: 1 +10 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`timestamp_ns` UInt64 +`revision` UInt32 +`trace_type` Enum8('Real' = 0, 'CPU' = 1, 'Memory' = 2, 'MemorySample' = 3) +`thread_id` UInt64 +`query_id` String +`trace` Array(UInt64) +`size` Int64 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/count.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/count.txt new file mode 100644 index 00000000000..0ac440f96a1 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/count.txt @@ -0,0 +1 @@ +1378 \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/default_compression_codec.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/minmax_event_date.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/minmax_event_date.idx new file mode 100644 index 00000000000..4c89b29a856 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/partition.dat b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..a713aecd075daae7071deceaa93147871ef28fbf GIT binary patch literal 4 LcmXRFWo7^X0v!O3 literal 0 HcmV?d00001 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/primary.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/primary.idx new file mode 100644 index 00000000000..6180f063fac --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/primary.idx @@ -0,0 +1 @@ +H_Hj ` \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/checksums.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..b91b7faa4f06a1664ebb74109586cac5fd23a1bf GIT binary patch literal 261 zcmV+g0s8)9Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$bE+#qTQ)Lu{YCP+gXf%IE~ z*#H0l&j0`b@jeC#V{dhCbS`vwbOU(kK>xQg&QB-oQa%ok*L?s8WMOn+E@EkJi8YXH zVhIrlaZ>%T4I&riEH(kO-vz`k)u>G8-l<@lxx5?vo?Zzr0Pt>Ya%(ft0?3Luk)Hm4 z)H$e;_u%sC>HrmOX>M&{cwc39Wo~p|WMOn=E@@2I=Kk=xD!%mjDZJa%pX0a(OOkWOxk4<8|vG32}9| Lb!HVccCGgS;Baoy literal 0 HcmV?d00001 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/columns.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/columns.txt new file mode 100644 index 00000000000..193d3123896 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/columns.txt @@ -0,0 +1,12 @@ +columns format version: 1 +10 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`timestamp_ns` UInt64 +`revision` UInt32 +`trace_type` Enum8('Real' = 0, 'CPU' = 1, 'Memory' = 2, 'MemorySample' = 3) +`thread_id` UInt64 +`query_id` String +`trace` Array(UInt64) +`size` Int64 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/count.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/count.txt new file mode 100644 index 00000000000..0e92c3c0fc9 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/count.txt @@ -0,0 +1 @@ +306 \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/default_compression_codec.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/minmax_event_date.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/minmax_event_date.idx new file mode 100644 index 00000000000..0e3b7fb75b8 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/partition.dat b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..da540c2d1a825be3ef80d13baa834834b2248491 GIT binary patch literal 4 LcmXRDWo7^X0wDm8 literal 0 HcmV?d00001 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/primary.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/primary.idx new file mode 100644 index 00000000000..b8d049dbb19 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/primary.idx @@ -0,0 +1 @@ +HC!`H"` \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/checksums.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..f448a75b6063fba410c431980059309d4b9a6aa1 GIT binary patch literal 260 zcmV+f0sH=AXk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$a$Hb&?2dEW#ESpcY+$ktzi z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOCIfRlVt&lYFUDR9N&B1^)mDWMOn+E@EkJ_5<}3 z3}5ITzipTtZggK{VRU6KX=Hc=VZjEVxwC1aJrFfB;*n}Y01a?q za&&2QX>V>WWMOm!K^bOwf>Qhj>E&4HXupM*01I$(X>DP0c`j*WcnsN(E!5h!GNyDe K$;I8+;Fkb~3T$=& literal 0 HcmV?d00001 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/columns.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/columns.txt new file mode 100644 index 00000000000..193d3123896 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/columns.txt @@ -0,0 +1,12 @@ +columns format version: 1 +10 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`timestamp_ns` UInt64 +`revision` UInt32 +`trace_type` Enum8('Real' = 0, 'CPU' = 1, 'Memory' = 2, 'MemorySample' = 3) +`thread_id` UInt64 +`query_id` String +`trace` Array(UInt64) +`size` Int64 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/count.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/count.txt new file mode 100644 index 00000000000..56a6051ca2b --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/count.txt @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/default_compression_codec.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/minmax_event_date.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/minmax_event_date.idx new file mode 100644 index 00000000000..16cc6680505 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/partition.dat b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..da540c2d1a825be3ef80d13baa834834b2248491 GIT binary patch literal 4 LcmXRDWo7^X0wDm8 literal 0 HcmV?d00001 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/primary.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/primary.idx new file mode 100644 index 00000000000..9f363c02b40 --- /dev/null +++ b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/primary.idx @@ -0,0 +1 @@ +HG"`HG"` \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_129_129_0/checksums.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_129_129_0/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..d331cb1fcc1616a320c584208bbbe060c20c1947 GIT binary patch literal 261 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyV5)|94mWJa#3U_`UDuPj{Tybd!OB z;XDw3v}5B;&M(a?(JQGaVLEni-X{0LThekT`8{LRP8u?Bq$HLk>Lq37bqTze&kSbS z^7D+CCQnYxcHy^-OLj7TynlAv5BGeAS4nH{ichgE;?!sOke6GOZG3^LKys6eD>t{J z*eAbZOH(qk8ANk4^Kug_;#12~^Gf1@=BDaprc|&b9%Ngvb93e*TLDw!M-#K08F&j4 zi%K#}fDYFKDq?YzOsi}P`os3}X~c_+{jJj(xC@Fha}$dyLArVRV>iAT$d33qg;0jf$7<_1PzbQF>&8 z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOPvZe4NH(<2?+h%j6?;@(ut9WMOn+E@EkJgdK+V z5ZhW2+=>to#{T20Lq-9{RA`+*_6V{cl3!Qme$Qq=nF%fc@NR8#Ycuc(P`_DAmgVQ_ z!Oxz{GfIzl02OU%Zf#+BUuAY>ZggK{VRU6KX=Hc=Yhv{9L4)i+mY_Q2(jfMG01a?q za&&2QX>V>WWMOm!_Mvet<#01I$(X>DP0c`j*Wcns^T1OQK`i$9(N KipbM-MdtvM<#0p* literal 0 HcmV?d00001 diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/columns.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/columns.txt new file mode 100644 index 00000000000..532baa6cba6 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/columns.txt @@ -0,0 +1,41 @@ +columns format version: 1 +39 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`query_start_time` DateTime +`query_start_time_microseconds` DateTime64(6) +`query_duration_ms` UInt64 +`read_rows` UInt64 +`read_bytes` UInt64 +`written_rows` UInt64 +`written_bytes` UInt64 +`memory_usage` Int64 +`peak_memory_usage` Int64 +`thread_name` String +`thread_id` UInt64 +`master_thread_id` UInt64 +`query` String +`is_initial_query` UInt8 +`user` String +`query_id` String +`address` IPv6 +`port` UInt16 +`initial_user` String +`initial_query_id` String +`initial_address` IPv6 +`initial_port` UInt16 +`interface` UInt8 +`os_user` String +`client_hostname` String +`client_name` String +`client_revision` UInt32 +`client_version_major` UInt32 +`client_version_minor` UInt32 +`client_version_patch` UInt32 +`http_method` UInt8 +`http_user_agent` String +`quota_key` String +`revision` UInt32 +`ProfileEvents.Names` Array(String) +`ProfileEvents.Values` Array(UInt64) diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/count.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/count.txt new file mode 100644 index 00000000000..3cacc0b93c9 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/count.txt @@ -0,0 +1 @@ +12 \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/default_compression_codec.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/minmax_event_date.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/minmax_event_date.idx new file mode 100644 index 00000000000..4b6c49f8d50 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/partition.dat b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..b9ea5569e3a3b34e3a1b6a1c4cbcb46f5e2d3fac GIT binary patch literal 4 Lcmb0VWo7^X0I>iw literal 0 HcmV?d00001 diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/primary.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/primary.idx new file mode 100644 index 00000000000..d599340cc82 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/primary.idx @@ -0,0 +1 @@ +H!._Hm._ \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/checksums.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..4398dc6a7abbe10202dcc9ff464b250d9453cfcf GIT binary patch literal 262 zcmV+h0r~!8Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$aI-o22|e$OHz$TackS;rWH z+5i9m&;S4c@jnI$V{dhCbS`vwbOTsTaBK6JI5}pr1~GpWy)OU=WMOn+E@EkJsmK8g z4bfp!g8FEMWeVZj)V}Wlv(^fr`M!nXj-ryTzn4HBVbGchFaYpwZE|Zf@CkwPt9t01 zsfP|9Ri`pFb>9FLZE0?8VR&C@6CZZ2eDbOiYfFBkp7g32tUh>SBBG-m({aB^vFVRCscX=Hc|vV`+0{_>ZK M%eyNa13TFQ07qJGTL1t6 literal 0 HcmV?d00001 diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/columns.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/columns.txt new file mode 100644 index 00000000000..532baa6cba6 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/columns.txt @@ -0,0 +1,41 @@ +columns format version: 1 +39 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`query_start_time` DateTime +`query_start_time_microseconds` DateTime64(6) +`query_duration_ms` UInt64 +`read_rows` UInt64 +`read_bytes` UInt64 +`written_rows` UInt64 +`written_bytes` UInt64 +`memory_usage` Int64 +`peak_memory_usage` Int64 +`thread_name` String +`thread_id` UInt64 +`master_thread_id` UInt64 +`query` String +`is_initial_query` UInt8 +`user` String +`query_id` String +`address` IPv6 +`port` UInt16 +`initial_user` String +`initial_query_id` String +`initial_address` IPv6 +`initial_port` UInt16 +`interface` UInt8 +`os_user` String +`client_hostname` String +`client_name` String +`client_revision` UInt32 +`client_version_major` UInt32 +`client_version_minor` UInt32 +`client_version_patch` UInt32 +`http_method` UInt8 +`http_user_agent` String +`quota_key` String +`revision` UInt32 +`ProfileEvents.Names` Array(String) +`ProfileEvents.Values` Array(UInt64) diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/count.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/count.txt new file mode 100644 index 00000000000..cbd6012bc6f --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/count.txt @@ -0,0 +1 @@ +204 \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/default_compression_codec.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/minmax_event_date.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/minmax_event_date.idx new file mode 100644 index 00000000000..4fedf1265b2 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/partition.dat b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..7af4421fcb5ebc50aa06a5f4dee6e454020552a6 GIT binary patch literal 4 Lcmb0UWo7^X0J#7) literal 0 HcmV?d00001 diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/primary.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/primary.idx new file mode 100644 index 00000000000..7f33001d913 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/primary.idx @@ -0,0 +1 @@ +Hb_H_ \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/checksums.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..cc8c216e1b7494e413c0da9a5b31486c5d90c403 GIT binary patch literal 262 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyWf3AW-@%fo-?_t?&Cpwzzz3y2Zf2 zZ~=%v+Ou&c=a=S{=#^BIFgFFyXHxB3fAoMC=byk-#h(luDTyVCdP$ji9rGB|rzA|U zJX$kL_GQ{`uZTS%j8ms`sB}#X6EH4R>Y^4rRJ5y0}W2q%S@?YIU{wz_k+a5P6o-x>)Eq|k1_BT zBo>uqmH=I@2UNte@Gzt1@~{GqOEDq63R@?>W8f|*%FInHss!ogDczYWTxYG5;(xo& KrZ}f&2Lk}qd}`$Y literal 0 HcmV?d00001 diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/columns.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/columns.txt new file mode 100644 index 00000000000..532baa6cba6 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/columns.txt @@ -0,0 +1,41 @@ +columns format version: 1 +39 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`query_start_time` DateTime +`query_start_time_microseconds` DateTime64(6) +`query_duration_ms` UInt64 +`read_rows` UInt64 +`read_bytes` UInt64 +`written_rows` UInt64 +`written_bytes` UInt64 +`memory_usage` Int64 +`peak_memory_usage` Int64 +`thread_name` String +`thread_id` UInt64 +`master_thread_id` UInt64 +`query` String +`is_initial_query` UInt8 +`user` String +`query_id` String +`address` IPv6 +`port` UInt16 +`initial_user` String +`initial_query_id` String +`initial_address` IPv6 +`initial_port` UInt16 +`interface` UInt8 +`os_user` String +`client_hostname` String +`client_name` String +`client_revision` UInt32 +`client_version_major` UInt32 +`client_version_minor` UInt32 +`client_version_patch` UInt32 +`http_method` UInt8 +`http_user_agent` String +`quota_key` String +`revision` UInt32 +`ProfileEvents.Names` Array(String) +`ProfileEvents.Values` Array(UInt64) diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/count.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/count.txt new file mode 100644 index 00000000000..aa59885c836 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/count.txt @@ -0,0 +1 @@ +143 \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/default_compression_codec.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/minmax_event_date.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/minmax_event_date.idx new file mode 100644 index 00000000000..4c89b29a856 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/partition.dat b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..a713aecd075daae7071deceaa93147871ef28fbf GIT binary patch literal 4 LcmXRFWo7^X0v!O3 literal 0 HcmV?d00001 diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/primary.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/primary.idx new file mode 100644 index 00000000000..f166214418a --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/primary.idx @@ -0,0 +1 @@ +H_H ` \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/checksums.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..0921ff99748c1a04a7fd95cb8943ecfec846c1a0 GIT binary patch literal 261 zcmV+g0s8)9Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$aAz*8X&k@J-6kO<{X=LkxI z*#H0l&j0`b@jeC#V{dhCbS`vwbOJ{BwXp-jZa9E;ck*GPUV8utWMOn+E@EkJ+%Aqj zHZie?gISC4ZtT~N;D7=A*a4?auCebz5f>&NuopMmmh}lQ0Pt>Ya%(g23Fi<6sEDe~ zu$pGM;@RW~KL8bNX>M&{cwc39Wo~p|WMOn=E@@2I=Kk=xD!%mjDZJa%pX0a(OOkWOxh%C+Pf(wa}}N LsJMgcdn4umo7ixJ literal 0 HcmV?d00001 diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/columns.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/columns.txt new file mode 100644 index 00000000000..532baa6cba6 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/columns.txt @@ -0,0 +1,41 @@ +columns format version: 1 +39 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`query_start_time` DateTime +`query_start_time_microseconds` DateTime64(6) +`query_duration_ms` UInt64 +`read_rows` UInt64 +`read_bytes` UInt64 +`written_rows` UInt64 +`written_bytes` UInt64 +`memory_usage` Int64 +`peak_memory_usage` Int64 +`thread_name` String +`thread_id` UInt64 +`master_thread_id` UInt64 +`query` String +`is_initial_query` UInt8 +`user` String +`query_id` String +`address` IPv6 +`port` UInt16 +`initial_user` String +`initial_query_id` String +`initial_address` IPv6 +`initial_port` UInt16 +`interface` UInt8 +`os_user` String +`client_hostname` String +`client_name` String +`client_revision` UInt32 +`client_version_major` UInt32 +`client_version_minor` UInt32 +`client_version_patch` UInt32 +`http_method` UInt8 +`http_user_agent` String +`quota_key` String +`revision` UInt32 +`ProfileEvents.Names` Array(String) +`ProfileEvents.Values` Array(UInt64) diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/count.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/count.txt new file mode 100644 index 00000000000..978b4e8e518 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/count.txt @@ -0,0 +1 @@ +26 \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/default_compression_codec.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/minmax_event_date.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/minmax_event_date.idx new file mode 100644 index 00000000000..0e3b7fb75b8 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/partition.dat b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..da540c2d1a825be3ef80d13baa834834b2248491 GIT binary patch literal 4 LcmXRDWo7^X0wDm8 literal 0 HcmV?d00001 diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/primary.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/primary.idx new file mode 100644 index 00000000000..67f45d29597 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/primary.idx @@ -0,0 +1 @@ +HC!`HG"` \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/checksums.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..2c30b88770877a5ddaa75c2f3daf2dbc321be19f GIT binary patch literal 260 zcmV+f0sH=AXk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$cbNRM_*!6JF+5+jxa^3W%O z*Z=?k&Hw-a@jV6!V{dhCbS`vwbON#VDF3f}{FN>b+}S`mO_BfzWMOn+E@EkJz#hw3 zOT%v^;N`ZYj_KCHG`s=1SwsOGv?YnRsHJA*@;B-H;|VSR@NR8#Ycuc(Kw~dMnFgB~ zmXR7wn|S>302OU%Zf#+BUuAY>ZggK{VRU6KX=Hc=VZjEVxwC1aJrFfB;*n}Y01a?q za&&2QX>V>WWMOm!K^bOwf>Qhj>E&4HXupM*01I$(X>DP0c`j*Wcnl8Xz`TJd7NP!7 K53t!rdd~o;%5H-I literal 0 HcmV?d00001 diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/columns.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/columns.txt new file mode 100644 index 00000000000..532baa6cba6 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/columns.txt @@ -0,0 +1,41 @@ +columns format version: 1 +39 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`query_start_time` DateTime +`query_start_time_microseconds` DateTime64(6) +`query_duration_ms` UInt64 +`read_rows` UInt64 +`read_bytes` UInt64 +`written_rows` UInt64 +`written_bytes` UInt64 +`memory_usage` Int64 +`peak_memory_usage` Int64 +`thread_name` String +`thread_id` UInt64 +`master_thread_id` UInt64 +`query` String +`is_initial_query` UInt8 +`user` String +`query_id` String +`address` IPv6 +`port` UInt16 +`initial_user` String +`initial_query_id` String +`initial_address` IPv6 +`initial_port` UInt16 +`interface` UInt8 +`os_user` String +`client_hostname` String +`client_name` String +`client_revision` UInt32 +`client_version_major` UInt32 +`client_version_minor` UInt32 +`client_version_patch` UInt32 +`http_method` UInt8 +`http_user_agent` String +`quota_key` String +`revision` UInt32 +`ProfileEvents.Names` Array(String) +`ProfileEvents.Values` Array(UInt64) diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/count.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/count.txt new file mode 100644 index 00000000000..9a037142aa3 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/count.txt @@ -0,0 +1 @@ +10 \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/default_compression_codec.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/minmax_event_date.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/minmax_event_date.idx new file mode 100644 index 00000000000..16cc6680505 --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/partition.dat b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..da540c2d1a825be3ef80d13baa834834b2248491 GIT binary patch literal 4 LcmXRDWo7^X0wDm8 literal 0 HcmV?d00001 diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/primary.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/primary.idx new file mode 100644 index 00000000000..8f3c7c8809e --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/primary.idx @@ -0,0 +1 @@ +H"`H"` \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/format_version.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/format_version.txt new file mode 100644 index 00000000000..56a6051ca2b --- /dev/null +++ b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/format_version.txt @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/checksums.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..e9f3f5dbfb1363c5743ef536fa11535cbb312446 GIT binary patch literal 262 zcmV+h0r~!8Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$c>P9c|kkD-grFaYpwZE|Zfumc7>wrI~> z!G@D=hNwi5yodl5ZE0?8VR&C_3*EI_1(J_Im&g zaA9(EX>@6CZZ2eDbOiRJGt`jAL&ph4JQlLni7WsMaB^vFVRCscX=Hc|qQj~+&$gS+ M9BY!1)str&0ESs^mH+?% literal 0 HcmV?d00001 diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/columns.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/columns.txt new file mode 100644 index 00000000000..37cb59062b4 --- /dev/null +++ b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/columns.txt @@ -0,0 +1,15 @@ +columns format version: 1 +13 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`microseconds` UInt32 +`thread_name` LowCardinality(String) +`thread_id` UInt64 +`level` Enum8('Fatal' = 1, 'Critical' = 2, 'Error' = 3, 'Warning' = 4, 'Notice' = 5, 'Information' = 6, 'Debug' = 7, 'Trace' = 8) +`query_id` String +`logger_name` LowCardinality(String) +`message` String +`revision` UInt32 +`source_file` LowCardinality(String) +`source_line` UInt64 diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/count.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/count.txt new file mode 100644 index 00000000000..19afd4ebd92 --- /dev/null +++ b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/count.txt @@ -0,0 +1 @@ +527 \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/default_compression_codec.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/minmax_event_date.idx b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/minmax_event_date.idx new file mode 100644 index 00000000000..4b6c49f8d50 --- /dev/null +++ b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/partition.dat b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..b9ea5569e3a3b34e3a1b6a1c4cbcb46f5e2d3fac GIT binary patch literal 4 Lcmb0VWo7^X0I>iw literal 0 HcmV?d00001 diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/primary.idx b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/primary.idx new file mode 100644 index 00000000000..714ba773856 --- /dev/null +++ b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/primary.idx @@ -0,0 +1 @@ +H-_H._ \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_2358_2361_1/checksums.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_2358_2361_1/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..e0dd83d152b7ad470cb7e6dc01063da79250d71e GIT binary patch literal 260 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyYGj&^g1ob@95*pY+^0H=Su~y1~G} za1Mw++Olyb=a=S{=#^BIFijMHvvzKnnEIOFX9gz}^Zqk%q$HLk>Lq37{ZYO&OJI4M zjCAP3lOg(izXaSEH$*n5CK&i9%nMq!Y`X9HpGOaI>M?xC%Pq<_-oU(R)d$Ag!s`2H zEpFbZu;!i*0UP(OA*i^mDlnNGyw=9*XUwBkGJZ(#GEfAi{z*~@5 zRFYW&bhjQ*5z9{=eeu7Cnoj8~>F6<*FiB_NE-1>(O)RPe>E?-ZOk)-2I#_?b=Uju7Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$b_qY0(p<%vUw$Fz9=x*t@6 z+W-In(EtDd@jwO%V{dhCbS`vwbOp+-iJ0^6C!0o57CJpSfd~KyWMOn+E@EkJ)Phz1 z%k1aRq;w4zWD2{8chZf$aFGnfz4lqGhN z^F@@yAM#diPMbsk6>Vv5ZDDv{Wp-t5bYEm)bY(7SWOxKobNpT`{e-ssQD+6%72KTw z4RB#{bZK;HZ*DGRVRQuf3@;b`!h*^yq=<|&7&K=93vhC2ZDDeGE@@u7Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$aUySV8>?*YX%1ymm|vajoc z+W-In(EtDd@jwO%V{dhCbS`vwbOk!)l;t<$46HHJYX_ISYDxeIWMOn+E@EkJvza1} zQyBilEl5*9x)JNQ3ivDm(!!zvxK+z2v32q!r+DVSDoo(32{8chZf$aFGvEe@okB*~ zbTR%3Y-@*T4j5|y6>Vv5ZDDv{Wp-t5bYEm)bY(7SWOxM38o*8P7?Ftp8RM@9Yg5Jm z4RB#{bZK;HZ*DGRVRQtc!vQI$R&WT?TvUx9wUO=s3vhC2ZDDeGE@@ literal 0 HcmV?d00001 diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/checksums.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c01d242d7ed0b91099869d3ec8d9393edff1e66 GIT binary patch literal 260 zcmV+f0sH=AXk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$b;Du41;5Zw_V+h#FX$e?_J z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOOj1`)FV*dCjdq%k=Y^lq&!TWMOn+E@EkJ%pskK z4+T*^fU(%s4!?yZDEI--N?tRe=#`t*XPo$DrKeG@MhPwe@NR8#YcsF|50od+F4&OI z%2q0?M&%lG02OU%Zf#+BUuAY>ZggK{VRU6KX=Hc=%oGj0*JiNNvOS&DmLLRW01a?q za&&2QX>V>WWMOm!p~C?wr&e$X(p*%HAhnV101I$(X>DP0c`j*WcntT(8?D{|1xXDg K$kqa`ii-e@x@&s? literal 0 HcmV?d00001 diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/columns.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/columns.txt new file mode 100644 index 00000000000..37cb59062b4 --- /dev/null +++ b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/columns.txt @@ -0,0 +1,15 @@ +columns format version: 1 +13 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`microseconds` UInt32 +`thread_name` LowCardinality(String) +`thread_id` UInt64 +`level` Enum8('Fatal' = 1, 'Critical' = 2, 'Error' = 3, 'Warning' = 4, 'Notice' = 5, 'Information' = 6, 'Debug' = 7, 'Trace' = 8) +`query_id` String +`logger_name` LowCardinality(String) +`message` String +`revision` UInt32 +`source_file` LowCardinality(String) +`source_line` UInt64 diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/count.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/count.txt new file mode 100644 index 00000000000..b2412e34dff --- /dev/null +++ b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/count.txt @@ -0,0 +1 @@ +62 \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/default_compression_codec.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/minmax_event_date.idx b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/minmax_event_date.idx new file mode 100644 index 00000000000..76b5229cbee --- /dev/null +++ b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/partition.dat b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..a713aecd075daae7071deceaa93147871ef28fbf GIT binary patch literal 4 LcmXRFWo7^X0v!O3 literal 0 HcmV?d00001 diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/primary.idx b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/primary.idx new file mode 100644 index 00000000000..e19de0711bb --- /dev/null +++ b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/primary.idx @@ -0,0 +1 @@ +H `H ` \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_3540_4637_415/checksums.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_3540_4637_415/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..6ff207deeb3ac1e0093ae21d37c1b81c3a9d2d5b GIT binary patch literal 263 zcmV+i0r>u7Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$cnp1>605;0weWScx&VIRYS z+W-In(EtDd@jwO%V{dhCbS`vwbOlE0NoHh|IjQ9=nBu&yL8kx+WMOn+E@EkJ-sK!V z{>Uu-LH|6^^@SRDj@mx~`{{rIlayO075KqB0u)m?sefCr2{8chZf$aFGvEeBYTdjB z7Ei**<+~!{Vky7?6>Vv5ZDDv{Wp-t5bYEm)bY(7SWOxJvIhh$>@v-l*wq3;?bvV`l z4RB#{bZK;HZ*DGRVRQsR8D@EcQv3$#iBAM}yn N|2B1HzutRv6aZJdZV>iAT$b7Fo=lt*g5R6KtA;xU-PVj z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOH&t)oZggK{VRU6KX=Hc=VZjEVxwC1aJrFfB;*n}Y01a?q za&&2QX>V>WWMOm!K^bOwf>Qhj>E&4HXupM*01I$(X>DP0c`j*WcnlzQ8dORAdLOmq K{L?d@g`fZn8gaG& literal 0 HcmV?d00001 diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/columns.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/columns.txt new file mode 100644 index 00000000000..37cb59062b4 --- /dev/null +++ b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/columns.txt @@ -0,0 +1,15 @@ +columns format version: 1 +13 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`microseconds` UInt32 +`thread_name` LowCardinality(String) +`thread_id` UInt64 +`level` Enum8('Fatal' = 1, 'Critical' = 2, 'Error' = 3, 'Warning' = 4, 'Notice' = 5, 'Information' = 6, 'Debug' = 7, 'Trace' = 8) +`query_id` String +`logger_name` LowCardinality(String) +`message` String +`revision` UInt32 +`source_file` LowCardinality(String) +`source_line` UInt64 diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/count.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/count.txt new file mode 100644 index 00000000000..dce6588ca14 --- /dev/null +++ b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/count.txt @@ -0,0 +1 @@ +36 \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/default_compression_codec.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/minmax_event_date.idx b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/minmax_event_date.idx new file mode 100644 index 00000000000..16cc6680505 --- /dev/null +++ b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/partition.dat b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..da540c2d1a825be3ef80d13baa834834b2248491 GIT binary patch literal 4 LcmXRDWo7^X0wDm8 literal 0 HcmV?d00001 diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/primary.idx b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/primary.idx new file mode 100644 index 00000000000..3b0e472e2cc --- /dev/null +++ b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/primary.idx @@ -0,0 +1 @@ +H"`H"` \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/format_version.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/format_version.txt new file mode 100644 index 00000000000..56a6051ca2b --- /dev/null +++ b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/format_version.txt @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/asynchronous_metric_log.sql b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/asynchronous_metric_log.sql new file mode 100644 index 00000000000..43ff67fb63d --- /dev/null +++ b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/asynchronous_metric_log.sql @@ -0,0 +1,12 @@ +ATTACH TABLE _ UUID 'e6e8fcae-2f46-4616-a056-74fe935c46b1' +( + `event_date` Date, + `event_time` DateTime, + `event_time_microseconds` DateTime64(6), + `name` String, + `value` Float64 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(event_date) +ORDER BY (event_date, event_time) +SETTINGS index_granularity = 8192 diff --git a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/crash_log.sql b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/crash_log.sql new file mode 100644 index 00000000000..6cb4dcdc376 --- /dev/null +++ b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/crash_log.sql @@ -0,0 +1,17 @@ +ATTACH TABLE _ UUID '5ec3f88a-cd1b-467a-9769-7ff65a09c665' +( + `event_date` Date, + `event_time` DateTime, + `timestamp_ns` UInt64, + `signal` Int32, + `thread_id` UInt64, + `query_id` String, + `trace` Array(UInt64), + `trace_full` Array(String), + `version` String, + `revision` UInt32, + `build_id` String +) +ENGINE = MergeTree +ORDER BY (event_date, event_time) +SETTINGS index_granularity = 8192 diff --git a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/metric_log.sql b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/metric_log.sql new file mode 100644 index 00000000000..fbdbe3c0592 --- /dev/null +++ b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/metric_log.sql @@ -0,0 +1,264 @@ +ATTACH TABLE _ UUID '057dfcea-531d-4525-a19a-6720aa9ea4b0' +( + `event_date` Date, + `event_time` DateTime, + `event_time_microseconds` DateTime64(6), + `milliseconds` UInt64, + `ProfileEvent_Query` UInt64, + `ProfileEvent_SelectQuery` UInt64, + `ProfileEvent_InsertQuery` UInt64, + `ProfileEvent_FailedQuery` UInt64, + `ProfileEvent_FailedSelectQuery` UInt64, + `ProfileEvent_FailedInsertQuery` UInt64, + `ProfileEvent_QueryTimeMicroseconds` UInt64, + `ProfileEvent_SelectQueryTimeMicroseconds` UInt64, + `ProfileEvent_InsertQueryTimeMicroseconds` UInt64, + `ProfileEvent_FileOpen` UInt64, + `ProfileEvent_Seek` UInt64, + `ProfileEvent_ReadBufferFromFileDescriptorRead` UInt64, + `ProfileEvent_ReadBufferFromFileDescriptorReadFailed` UInt64, + `ProfileEvent_ReadBufferFromFileDescriptorReadBytes` UInt64, + `ProfileEvent_WriteBufferFromFileDescriptorWrite` UInt64, + `ProfileEvent_WriteBufferFromFileDescriptorWriteFailed` UInt64, + `ProfileEvent_WriteBufferFromFileDescriptorWriteBytes` UInt64, + `ProfileEvent_ReadBufferAIORead` UInt64, + `ProfileEvent_ReadBufferAIOReadBytes` UInt64, + `ProfileEvent_WriteBufferAIOWrite` UInt64, + `ProfileEvent_WriteBufferAIOWriteBytes` UInt64, + `ProfileEvent_ReadCompressedBytes` UInt64, + `ProfileEvent_CompressedReadBufferBlocks` UInt64, + `ProfileEvent_CompressedReadBufferBytes` UInt64, + `ProfileEvent_UncompressedCacheHits` UInt64, + `ProfileEvent_UncompressedCacheMisses` UInt64, + `ProfileEvent_UncompressedCacheWeightLost` UInt64, + `ProfileEvent_IOBufferAllocs` UInt64, + `ProfileEvent_IOBufferAllocBytes` UInt64, + `ProfileEvent_ArenaAllocChunks` UInt64, + `ProfileEvent_ArenaAllocBytes` UInt64, + `ProfileEvent_FunctionExecute` UInt64, + `ProfileEvent_TableFunctionExecute` UInt64, + `ProfileEvent_MarkCacheHits` UInt64, + `ProfileEvent_MarkCacheMisses` UInt64, + `ProfileEvent_CreatedReadBufferOrdinary` UInt64, + `ProfileEvent_CreatedReadBufferAIO` UInt64, + `ProfileEvent_CreatedReadBufferAIOFailed` UInt64, + `ProfileEvent_CreatedReadBufferMMap` UInt64, + `ProfileEvent_CreatedReadBufferMMapFailed` UInt64, + `ProfileEvent_CreatedWriteBufferOrdinary` UInt64, + `ProfileEvent_CreatedWriteBufferAIO` UInt64, + `ProfileEvent_CreatedWriteBufferAIOFailed` UInt64, + `ProfileEvent_DiskReadElapsedMicroseconds` UInt64, + `ProfileEvent_DiskWriteElapsedMicroseconds` UInt64, + `ProfileEvent_NetworkReceiveElapsedMicroseconds` UInt64, + `ProfileEvent_NetworkSendElapsedMicroseconds` UInt64, + `ProfileEvent_ThrottlerSleepMicroseconds` UInt64, + `ProfileEvent_QueryMaskingRulesMatch` UInt64, + `ProfileEvent_ReplicatedPartFetches` UInt64, + `ProfileEvent_ReplicatedPartFailedFetches` UInt64, + `ProfileEvent_ObsoleteReplicatedParts` UInt64, + `ProfileEvent_ReplicatedPartMerges` UInt64, + `ProfileEvent_ReplicatedPartFetchesOfMerged` UInt64, + `ProfileEvent_ReplicatedPartMutations` UInt64, + `ProfileEvent_ReplicatedPartChecks` UInt64, + `ProfileEvent_ReplicatedPartChecksFailed` UInt64, + `ProfileEvent_ReplicatedDataLoss` UInt64, + `ProfileEvent_InsertedRows` UInt64, + `ProfileEvent_InsertedBytes` UInt64, + `ProfileEvent_DelayedInserts` UInt64, + `ProfileEvent_RejectedInserts` UInt64, + `ProfileEvent_DelayedInsertsMilliseconds` UInt64, + `ProfileEvent_DuplicatedInsertedBlocks` UInt64, + `ProfileEvent_ZooKeeperInit` UInt64, + `ProfileEvent_ZooKeeperTransactions` UInt64, + `ProfileEvent_ZooKeeperList` UInt64, + `ProfileEvent_ZooKeeperCreate` UInt64, + `ProfileEvent_ZooKeeperRemove` UInt64, + `ProfileEvent_ZooKeeperExists` UInt64, + `ProfileEvent_ZooKeeperGet` UInt64, + `ProfileEvent_ZooKeeperSet` UInt64, + `ProfileEvent_ZooKeeperMulti` UInt64, + `ProfileEvent_ZooKeeperCheck` UInt64, + `ProfileEvent_ZooKeeperClose` UInt64, + `ProfileEvent_ZooKeeperWatchResponse` UInt64, + `ProfileEvent_ZooKeeperUserExceptions` UInt64, + `ProfileEvent_ZooKeeperHardwareExceptions` UInt64, + `ProfileEvent_ZooKeeperOtherExceptions` UInt64, + `ProfileEvent_ZooKeeperWaitMicroseconds` UInt64, + `ProfileEvent_ZooKeeperBytesSent` UInt64, + `ProfileEvent_ZooKeeperBytesReceived` UInt64, + `ProfileEvent_DistributedConnectionFailTry` UInt64, + `ProfileEvent_DistributedConnectionMissingTable` UInt64, + `ProfileEvent_DistributedConnectionStaleReplica` UInt64, + `ProfileEvent_DistributedConnectionFailAtAll` UInt64, + `ProfileEvent_CompileAttempt` UInt64, + `ProfileEvent_CompileSuccess` UInt64, + `ProfileEvent_CompileFunction` UInt64, + `ProfileEvent_CompiledFunctionExecute` UInt64, + `ProfileEvent_CompileExpressionsMicroseconds` UInt64, + `ProfileEvent_CompileExpressionsBytes` UInt64, + `ProfileEvent_ExternalSortWritePart` UInt64, + `ProfileEvent_ExternalSortMerge` UInt64, + `ProfileEvent_ExternalAggregationWritePart` UInt64, + `ProfileEvent_ExternalAggregationMerge` UInt64, + `ProfileEvent_ExternalAggregationCompressedBytes` UInt64, + `ProfileEvent_ExternalAggregationUncompressedBytes` UInt64, + `ProfileEvent_SlowRead` UInt64, + `ProfileEvent_ReadBackoff` UInt64, + `ProfileEvent_ReplicaPartialShutdown` UInt64, + `ProfileEvent_SelectedParts` UInt64, + `ProfileEvent_SelectedRanges` UInt64, + `ProfileEvent_SelectedMarks` UInt64, + `ProfileEvent_SelectedRows` UInt64, + `ProfileEvent_SelectedBytes` UInt64, + `ProfileEvent_Merge` UInt64, + `ProfileEvent_MergedRows` UInt64, + `ProfileEvent_MergedUncompressedBytes` UInt64, + `ProfileEvent_MergesTimeMilliseconds` UInt64, + `ProfileEvent_MergeTreeDataWriterRows` UInt64, + `ProfileEvent_MergeTreeDataWriterUncompressedBytes` UInt64, + `ProfileEvent_MergeTreeDataWriterCompressedBytes` UInt64, + `ProfileEvent_MergeTreeDataWriterBlocks` UInt64, + `ProfileEvent_MergeTreeDataWriterBlocksAlreadySorted` UInt64, + `ProfileEvent_CannotRemoveEphemeralNode` UInt64, + `ProfileEvent_RegexpCreated` UInt64, + `ProfileEvent_ContextLock` UInt64, + `ProfileEvent_StorageBufferFlush` UInt64, + `ProfileEvent_StorageBufferErrorOnFlush` UInt64, + `ProfileEvent_StorageBufferPassedAllMinThresholds` UInt64, + `ProfileEvent_StorageBufferPassedTimeMaxThreshold` UInt64, + `ProfileEvent_StorageBufferPassedRowsMaxThreshold` UInt64, + `ProfileEvent_StorageBufferPassedBytesMaxThreshold` UInt64, + `ProfileEvent_DictCacheKeysRequested` UInt64, + `ProfileEvent_DictCacheKeysRequestedMiss` UInt64, + `ProfileEvent_DictCacheKeysRequestedFound` UInt64, + `ProfileEvent_DictCacheKeysExpired` UInt64, + `ProfileEvent_DictCacheKeysNotFound` UInt64, + `ProfileEvent_DictCacheKeysHit` UInt64, + `ProfileEvent_DictCacheRequestTimeNs` UInt64, + `ProfileEvent_DictCacheRequests` UInt64, + `ProfileEvent_DictCacheLockWriteNs` UInt64, + `ProfileEvent_DictCacheLockReadNs` UInt64, + `ProfileEvent_DistributedSyncInsertionTimeoutExceeded` UInt64, + `ProfileEvent_DataAfterMergeDiffersFromReplica` UInt64, + `ProfileEvent_DataAfterMutationDiffersFromReplica` UInt64, + `ProfileEvent_PolygonsAddedToPool` UInt64, + `ProfileEvent_PolygonsInPoolAllocatedBytes` UInt64, + `ProfileEvent_RWLockAcquiredReadLocks` UInt64, + `ProfileEvent_RWLockAcquiredWriteLocks` UInt64, + `ProfileEvent_RWLockReadersWaitMilliseconds` UInt64, + `ProfileEvent_RWLockWritersWaitMilliseconds` UInt64, + `ProfileEvent_DNSError` UInt64, + `ProfileEvent_RealTimeMicroseconds` UInt64, + `ProfileEvent_UserTimeMicroseconds` UInt64, + `ProfileEvent_SystemTimeMicroseconds` UInt64, + `ProfileEvent_SoftPageFaults` UInt64, + `ProfileEvent_HardPageFaults` UInt64, + `ProfileEvent_VoluntaryContextSwitches` UInt64, + `ProfileEvent_InvoluntaryContextSwitches` UInt64, + `ProfileEvent_OSIOWaitMicroseconds` UInt64, + `ProfileEvent_OSCPUWaitMicroseconds` UInt64, + `ProfileEvent_OSCPUVirtualTimeMicroseconds` UInt64, + `ProfileEvent_OSReadBytes` UInt64, + `ProfileEvent_OSWriteBytes` UInt64, + `ProfileEvent_OSReadChars` UInt64, + `ProfileEvent_OSWriteChars` UInt64, + `ProfileEvent_PerfCpuCycles` UInt64, + `ProfileEvent_PerfInstructions` UInt64, + `ProfileEvent_PerfCacheReferences` UInt64, + `ProfileEvent_PerfCacheMisses` UInt64, + `ProfileEvent_PerfBranchInstructions` UInt64, + `ProfileEvent_PerfBranchMisses` UInt64, + `ProfileEvent_PerfBusCycles` UInt64, + `ProfileEvent_PerfStalledCyclesFrontend` UInt64, + `ProfileEvent_PerfStalledCyclesBackend` UInt64, + `ProfileEvent_PerfRefCpuCycles` UInt64, + `ProfileEvent_PerfCpuClock` UInt64, + `ProfileEvent_PerfTaskClock` UInt64, + `ProfileEvent_PerfContextSwitches` UInt64, + `ProfileEvent_PerfCpuMigrations` UInt64, + `ProfileEvent_PerfAlignmentFaults` UInt64, + `ProfileEvent_PerfEmulationFaults` UInt64, + `ProfileEvent_PerfMinEnabledTime` UInt64, + `ProfileEvent_PerfMinEnabledRunningTime` UInt64, + `ProfileEvent_PerfDataTLBReferences` UInt64, + `ProfileEvent_PerfDataTLBMisses` UInt64, + `ProfileEvent_PerfInstructionTLBReferences` UInt64, + `ProfileEvent_PerfInstructionTLBMisses` UInt64, + `ProfileEvent_PerfLocalMemoryReferences` UInt64, + `ProfileEvent_PerfLocalMemoryMisses` UInt64, + `ProfileEvent_CreatedHTTPConnections` UInt64, + `ProfileEvent_CannotWriteToWriteBufferDiscard` UInt64, + `ProfileEvent_QueryProfilerSignalOverruns` UInt64, + `ProfileEvent_CreatedLogEntryForMerge` UInt64, + `ProfileEvent_NotCreatedLogEntryForMerge` UInt64, + `ProfileEvent_CreatedLogEntryForMutation` UInt64, + `ProfileEvent_NotCreatedLogEntryForMutation` UInt64, + `ProfileEvent_S3ReadMicroseconds` UInt64, + `ProfileEvent_S3ReadBytes` UInt64, + `ProfileEvent_S3ReadRequestsCount` UInt64, + `ProfileEvent_S3ReadRequestsErrors` UInt64, + `ProfileEvent_S3ReadRequestsThrottling` UInt64, + `ProfileEvent_S3ReadRequestsRedirects` UInt64, + `ProfileEvent_S3WriteMicroseconds` UInt64, + `ProfileEvent_S3WriteBytes` UInt64, + `ProfileEvent_S3WriteRequestsCount` UInt64, + `ProfileEvent_S3WriteRequestsErrors` UInt64, + `ProfileEvent_S3WriteRequestsThrottling` UInt64, + `ProfileEvent_S3WriteRequestsRedirects` UInt64, + `ProfileEvent_QueryMemoryLimitExceeded` UInt64, + `CurrentMetric_Query` Int64, + `CurrentMetric_Merge` Int64, + `CurrentMetric_PartMutation` Int64, + `CurrentMetric_ReplicatedFetch` Int64, + `CurrentMetric_ReplicatedSend` Int64, + `CurrentMetric_ReplicatedChecks` Int64, + `CurrentMetric_BackgroundPoolTask` Int64, + `CurrentMetric_BackgroundMovePoolTask` Int64, + `CurrentMetric_BackgroundSchedulePoolTask` Int64, + `CurrentMetric_BackgroundBufferFlushSchedulePoolTask` Int64, + `CurrentMetric_BackgroundDistributedSchedulePoolTask` Int64, + `CurrentMetric_BackgroundMessageBrokerSchedulePoolTask` Int64, + `CurrentMetric_CacheDictionaryUpdateQueueBatches` Int64, + `CurrentMetric_CacheDictionaryUpdateQueueKeys` Int64, + `CurrentMetric_DiskSpaceReservedForMerge` Int64, + `CurrentMetric_DistributedSend` Int64, + `CurrentMetric_QueryPreempted` Int64, + `CurrentMetric_TCPConnection` Int64, + `CurrentMetric_MySQLConnection` Int64, + `CurrentMetric_HTTPConnection` Int64, + `CurrentMetric_InterserverConnection` Int64, + `CurrentMetric_PostgreSQLConnection` Int64, + `CurrentMetric_OpenFileForRead` Int64, + `CurrentMetric_OpenFileForWrite` Int64, + `CurrentMetric_Read` Int64, + `CurrentMetric_Write` Int64, + `CurrentMetric_SendScalars` Int64, + `CurrentMetric_SendExternalTables` Int64, + `CurrentMetric_QueryThread` Int64, + `CurrentMetric_ReadonlyReplica` Int64, + `CurrentMetric_MemoryTracking` Int64, + `CurrentMetric_EphemeralNode` Int64, + `CurrentMetric_ZooKeeperSession` Int64, + `CurrentMetric_ZooKeeperWatch` Int64, + `CurrentMetric_ZooKeeperRequest` Int64, + `CurrentMetric_DelayedInserts` Int64, + `CurrentMetric_ContextLockWait` Int64, + `CurrentMetric_StorageBufferRows` Int64, + `CurrentMetric_StorageBufferBytes` Int64, + `CurrentMetric_DictCacheRequests` Int64, + `CurrentMetric_Revision` Int64, + `CurrentMetric_VersionInteger` Int64, + `CurrentMetric_RWLockWaitingReaders` Int64, + `CurrentMetric_RWLockWaitingWriters` Int64, + `CurrentMetric_RWLockActiveReaders` Int64, + `CurrentMetric_RWLockActiveWriters` Int64, + `CurrentMetric_GlobalThread` Int64, + `CurrentMetric_GlobalThreadActive` Int64, + `CurrentMetric_LocalThread` Int64, + `CurrentMetric_LocalThreadActive` Int64, + `CurrentMetric_DistributedFilesToInsert` Int64 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(event_date) +ORDER BY (event_date, event_time) +SETTINGS index_granularity = 8192 diff --git a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/part_log.sql b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/part_log.sql new file mode 100644 index 00000000000..ea3aabd51e4 --- /dev/null +++ b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/part_log.sql @@ -0,0 +1,25 @@ +ATTACH TABLE _ UUID '721d6adf-4b1c-49bc-8fbe-1599ac4e048d' +( + `event_type` Enum8('NewPart' = 1, 'MergeParts' = 2, 'DownloadPart' = 3, 'RemovePart' = 4, 'MutatePart' = 5, 'MovePart' = 6), + `event_date` Date, + `event_time` DateTime, + `duration_ms` UInt64, + `database` String, + `table` String, + `part_name` String, + `partition_id` String, + `path_on_disk` String, + `rows` UInt64, + `size_in_bytes` UInt64, + `merged_from` Array(String), + `bytes_uncompressed` UInt64, + `read_rows` UInt64, + `read_bytes` UInt64, + `peak_memory_usage` UInt64, + `error` UInt16, + `exception` String +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(event_date) +ORDER BY (event_date, event_time) +SETTINGS index_granularity = 8192 diff --git a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/query_log.sql b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/query_log.sql new file mode 100644 index 00000000000..bcafc4b0530 --- /dev/null +++ b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/query_log.sql @@ -0,0 +1,52 @@ +ATTACH TABLE _ UUID '3f558312-ca2a-49dc-a6bf-a5daea1869df' +( + `type` Enum8('QueryStart' = 1, 'QueryFinish' = 2, 'ExceptionBeforeStart' = 3, 'ExceptionWhileProcessing' = 4), + `event_date` Date, + `event_time` DateTime, + `event_time_microseconds` DateTime64(6), + `query_start_time` DateTime, + `query_start_time_microseconds` DateTime64(6), + `query_duration_ms` UInt64, + `read_rows` UInt64, + `read_bytes` UInt64, + `written_rows` UInt64, + `written_bytes` UInt64, + `result_rows` UInt64, + `result_bytes` UInt64, + `memory_usage` UInt64, + `current_database` String, + `query` String, + `exception_code` Int32, + `exception` String, + `stack_trace` String, + `is_initial_query` UInt8, + `user` String, + `query_id` String, + `address` IPv6, + `port` UInt16, + `initial_user` String, + `initial_query_id` String, + `initial_address` IPv6, + `initial_port` UInt16, + `interface` UInt8, + `os_user` String, + `client_hostname` String, + `client_name` String, + `client_revision` UInt32, + `client_version_major` UInt32, + `client_version_minor` UInt32, + `client_version_patch` UInt32, + `http_method` UInt8, + `http_user_agent` String, + `quota_key` String, + `revision` UInt32, + `thread_ids` Array(UInt64), + `ProfileEvents.Names` Array(String), + `ProfileEvents.Values` Array(UInt64), + `Settings.Names` Array(String), + `Settings.Values` Array(String) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(event_date) +ORDER BY (event_date, event_time) +SETTINGS index_granularity = 8192 diff --git a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/query_thread_log.sql b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/query_thread_log.sql new file mode 100644 index 00000000000..56ff3366702 --- /dev/null +++ b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/query_thread_log.sql @@ -0,0 +1,46 @@ +ATTACH TABLE _ UUID '8b2738c3-4bea-49f7-b547-0a92673830cc' +( + `event_date` Date, + `event_time` DateTime, + `event_time_microseconds` DateTime64(6), + `query_start_time` DateTime, + `query_start_time_microseconds` DateTime64(6), + `query_duration_ms` UInt64, + `read_rows` UInt64, + `read_bytes` UInt64, + `written_rows` UInt64, + `written_bytes` UInt64, + `memory_usage` Int64, + `peak_memory_usage` Int64, + `thread_name` String, + `thread_id` UInt64, + `master_thread_id` UInt64, + `query` String, + `is_initial_query` UInt8, + `user` String, + `query_id` String, + `address` IPv6, + `port` UInt16, + `initial_user` String, + `initial_query_id` String, + `initial_address` IPv6, + `initial_port` UInt16, + `interface` UInt8, + `os_user` String, + `client_hostname` String, + `client_name` String, + `client_revision` UInt32, + `client_version_major` UInt32, + `client_version_minor` UInt32, + `client_version_patch` UInt32, + `http_method` UInt8, + `http_user_agent` String, + `quota_key` String, + `revision` UInt32, + `ProfileEvents.Names` Array(String), + `ProfileEvents.Values` Array(UInt64) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(event_date) +ORDER BY (event_date, event_time) +SETTINGS index_granularity = 8192 diff --git a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/text_log.sql b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/text_log.sql new file mode 100644 index 00000000000..874466c610f --- /dev/null +++ b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/text_log.sql @@ -0,0 +1,20 @@ +ATTACH TABLE _ UUID 'b619f0a8-9836-4824-9594-a722b0a7d500' +( + `event_date` Date, + `event_time` DateTime, + `event_time_microseconds` DateTime64(6), + `microseconds` UInt32, + `thread_name` LowCardinality(String), + `thread_id` UInt64, + `level` Enum8('Fatal' = 1, 'Critical' = 2, 'Error' = 3, 'Warning' = 4, 'Notice' = 5, 'Information' = 6, 'Debug' = 7, 'Trace' = 8), + `query_id` String, + `logger_name` LowCardinality(String), + `message` String, + `revision` UInt32, + `source_file` LowCardinality(String), + `source_line` UInt64 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(event_date) +ORDER BY (event_date, event_time) +SETTINGS index_granularity = 8192 diff --git a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/trace_log.sql b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/trace_log.sql new file mode 100644 index 00000000000..04517208a2e --- /dev/null +++ b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/trace_log.sql @@ -0,0 +1,17 @@ +ATTACH TABLE _ UUID '72dc35e3-6e64-44bf-af3f-4d66fb70834a' +( + `event_date` Date, + `event_time` DateTime, + `event_time_microseconds` DateTime64(6), + `timestamp_ns` UInt64, + `revision` UInt32, + `trace_type` Enum8('Real' = 0, 'CPU' = 1, 'Memory' = 2, 'MemorySample' = 3), + `thread_id` UInt64, + `query_id` String, + `trace` Array(UInt64), + `size` Int64 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(event_date) +ORDER BY (event_date, event_time) +SETTINGS index_granularity = 8192 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/checksums.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..15f590ec5a9f148c51c40e5f8643b93ed6312e48 GIT binary patch literal 260 zcmV+f0sH=AXk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$bUKSS==f{FYCNyEFtWbCto z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOT5}}ui?1Ks zvg>S#d$7rg02OU%Zf#+BUuAY>ZggK{VRU6KX=Hc=Yhv{9L4)i+mY_Q2(jfMG01a?q za&&2QX>V>WWMOm!_Mvet<#01I$(X>DP0c`j*Wcnm83Pgz_C>>6$e Kar3Qw+64edhHlyb literal 0 HcmV?d00001 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/columns.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/columns.txt new file mode 100644 index 00000000000..5bfe4600a4e --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/columns.txt @@ -0,0 +1,7 @@ +columns format version: 1 +5 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`name` String +`value` Float64 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/count.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/count.txt new file mode 100644 index 00000000000..afbe847262c --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/count.txt @@ -0,0 +1 @@ +126 \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/default_compression_codec.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/minmax_event_date.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/minmax_event_date.idx new file mode 100644 index 00000000000..4b6c49f8d50 --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/partition.dat b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..b9ea5569e3a3b34e3a1b6a1c4cbcb46f5e2d3fac GIT binary patch literal 4 Lcmb0VWo7^X0I>iw literal 0 HcmV?d00001 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/primary.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/primary.idx new file mode 100644 index 00000000000..38e2c642aeb --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/primary.idx @@ -0,0 +1 @@ +H-_HN._ \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/checksums.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..288379f59b4d0506587d2b915da29b5429f3c32c GIT binary patch literal 262 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyVUf?3K)m;9pkOXWjEYr54|4y2Zf2 zZ~=%v+Ou&c=a=S{=#^BIupV9eg5{F*3qhl-g^$@cNz7y5NJ%V7)Jw|DJHCbKd3ghe zFgEolo@6;#*(z-(X1n4kgRx0RBc}nwhrHaPY~v25bs?^=WxQQ@ znPqo7Se%>sj6pOvGcPx>B0jY&HLoNdXmF}tW=aK1VDXg_AS<=yCEMbz)z+F(3nVVQt3DPa}a>cj&7s0hFg}*8) KC3BoV%>V$TlWN`o literal 0 HcmV?d00001 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/columns.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/columns.txt new file mode 100644 index 00000000000..5bfe4600a4e --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/columns.txt @@ -0,0 +1,7 @@ +columns format version: 1 +5 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`name` String +`value` Float64 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/count.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/count.txt new file mode 100644 index 00000000000..23644ad9685 --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/count.txt @@ -0,0 +1 @@ +12390 \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/default_compression_codec.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/minmax_event_date.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/minmax_event_date.idx new file mode 100644 index 00000000000..07abc658bdb --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/partition.dat b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..7af4421fcb5ebc50aa06a5f4dee6e454020552a6 GIT binary patch literal 4 Lcmb0UWo7^X0J#7) literal 0 HcmV?d00001 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/primary.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/primary.idx new file mode 100644 index 00000000000..d57044faf98 --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/primary.idx @@ -0,0 +1 @@ +H>_H_H}_ \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/checksums.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..d702cbf79d2a5a91c07163c9680526ae0c503ceb GIT binary patch literal 262 zcmV+h0r~!8Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$bUW>V4jhooyz_~f{Za(pN8Q&6FaYpwZE|ZfumSZKokr7~ z|403vu$n{)-iQDdZE0?8VR&C@6CZZ2eDbOfQp0V$_ea0t>|RE;3Dk?sHsaB^vFVRCscX=Hc|m=Z;zemg0s MudjjE3#Lvc0M|-wEC2ui literal 0 HcmV?d00001 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/columns.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/columns.txt new file mode 100644 index 00000000000..5bfe4600a4e --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/columns.txt @@ -0,0 +1,7 @@ +columns format version: 1 +5 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`name` String +`value` Float64 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/count.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/count.txt new file mode 100644 index 00000000000..721ca6f552a --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/count.txt @@ -0,0 +1 @@ +6048 \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/default_compression_codec.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/minmax_event_date.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/minmax_event_date.idx new file mode 100644 index 00000000000..4c89b29a856 --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/partition.dat b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..a713aecd075daae7071deceaa93147871ef28fbf GIT binary patch literal 4 LcmXRFWo7^X0v!O3 literal 0 HcmV?d00001 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/primary.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/primary.idx new file mode 100644 index 00000000000..8e5b4736172 --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/primary.idx @@ -0,0 +1 @@ +H_H ` \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_440_574_27/checksums.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_440_574_27/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..318b8a518b960630dc1e834f27ce63e27b6593ab GIT binary patch literal 262 zcmV+h0r~!8Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$c7oqGLyZE0?8VR&C@6CZZ2eDbOb>eW_f~A{08aeSmV>iAT$cC2%NmSkKqG{?=Jq@?bnHd z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOJne3tc?*x$p?c>F_{T(_H`vWMOn+E@EkJfCsQb zD@h%ngTaZ|pyvL@;Y0zh4$4=AdlcKPVg5TpMxB@3V+k$*@NR8#YcsF`21}@_f-!EI z40m1$`G2OI02OU%Zf#+BUuAY>ZggK{VRU6KX=Hc=VZjEVxwC1aJrFfB;*n}Y01a?q za&&2QX>V>WWMOm!K^bOwf>Qhj>E&4HXupM*01I$(X>DP0c`j*Wcnrlp=b;Ym@z8Dp Kp(2L^K0p9kBW<++ literal 0 HcmV?d00001 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/columns.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/columns.txt new file mode 100644 index 00000000000..5bfe4600a4e --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/columns.txt @@ -0,0 +1,7 @@ +columns format version: 1 +5 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`name` String +`value` Float64 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/count.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/count.txt new file mode 100644 index 00000000000..f70d7bba4ae --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/count.txt @@ -0,0 +1 @@ +42 \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/default_compression_codec.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/minmax_event_date.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/minmax_event_date.idx new file mode 100644 index 00000000000..16cc6680505 --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/partition.dat b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..da540c2d1a825be3ef80d13baa834834b2248491 GIT binary patch literal 4 LcmXRDWo7^X0wDm8 literal 0 HcmV?d00001 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/primary.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/primary.idx new file mode 100644 index 00000000000..47460b95be9 --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/primary.idx @@ -0,0 +1 @@ +H"`H"` \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/checksums.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..705c91fa699e37af9a5bc17585f859123272e27b GIT binary patch literal 260 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyZ=8e&PS-lZH>%6zMI~Ocl~>y1~G} za1Mw++Olyb=a=S{=#^BIFxixH$J%_|`GMoa%MT9Wmtz??QW8rN^^!94dfBVyP4w8E z^d;5x=sNDHm&BhluH*BS+J9T=m`+Tsp_!zbhWTwyJ%$f?xkcH=8yJhzv;sK_cCKB% z@k(c3X6Zc!(cH|u+{B9b)UwpPl6au8sd||y6)cGd*%s{FoVmzWz|{EB#4Kk9-h#xU zlFSmIyY+yISR5tODw~4-u)Ta5@gie?>vRV0f}+ga#G*=&Zk}DNanDuX@+;a)C2mzy I?byly0Nr6}0ssI2 literal 0 HcmV?d00001 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/columns.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/columns.txt new file mode 100644 index 00000000000..5bfe4600a4e --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/columns.txt @@ -0,0 +1,7 @@ +columns format version: 1 +5 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`name` String +`value` Float64 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/count.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/count.txt new file mode 100644 index 00000000000..f70d7bba4ae --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/count.txt @@ -0,0 +1 @@ +42 \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/default_compression_codec.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/minmax_event_date.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/minmax_event_date.idx new file mode 100644 index 00000000000..16cc6680505 --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/partition.dat b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..da540c2d1a825be3ef80d13baa834834b2248491 GIT binary patch literal 4 LcmXRDWo7^X0wDm8 literal 0 HcmV?d00001 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/primary.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/primary.idx new file mode 100644 index 00000000000..dc2b3df8c14 --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/primary.idx @@ -0,0 +1 @@ +H"`H"` \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/checksums.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/checksums.txt new file mode 100644 index 0000000000000000000000000000000000000000..a79136dea2ae4694404058e307f30c58f2c95216 GIT binary patch literal 260 zcmV+f0sH=AXk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$b+KNoPO=+Yv2+Llc&Eg)Ee z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOJne3tc?*x$p?c>F_{T(_H`vWMOn+E@EkJk_VW( zJ28oFC&TW$V{^28W&8oI4&bJ;l7ZNHLvKrW15Q;okqIsU@NR8#YcsF`wn)byrXbCF zZggK{VRU6KX=Hc=VZjEVxwC1aJrFfB;*n}Y01a?q za&&2QX>V>WWMOm!K^bOwf>Qhj>E&4HXupM*01I$(X>DP0c`j*WcnoZhRO7$)udRQM Kfs2=!gM9!_8*V`W literal 0 HcmV?d00001 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/columns.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/columns.txt new file mode 100644 index 00000000000..5bfe4600a4e --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/columns.txt @@ -0,0 +1,7 @@ +columns format version: 1 +5 columns: +`event_date` Date +`event_time` DateTime +`event_time_microseconds` DateTime64(6) +`name` String +`value` Float64 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/count.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/count.txt new file mode 100644 index 00000000000..f70d7bba4ae --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/count.txt @@ -0,0 +1 @@ +42 \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/default_compression_codec.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/default_compression_codec.txt new file mode 100644 index 00000000000..061d1280b89 --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/default_compression_codec.txt @@ -0,0 +1 @@ +CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/minmax_event_date.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/minmax_event_date.idx new file mode 100644 index 00000000000..16cc6680505 --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/minmax_event_date.idx @@ -0,0 +1 @@ +HH \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/partition.dat b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/partition.dat new file mode 100644 index 0000000000000000000000000000000000000000..da540c2d1a825be3ef80d13baa834834b2248491 GIT binary patch literal 4 LcmXRDWo7^X0wDm8 literal 0 HcmV?d00001 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/primary.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/primary.idx new file mode 100644 index 00000000000..bd60e38174b --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/primary.idx @@ -0,0 +1 @@ +H""`H""` \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/format_version.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/format_version.txt new file mode 100644 index 00000000000..56a6051ca2b --- /dev/null +++ b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/format_version.txt @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 7049dfc03f6..f015ee73ec5 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -49,6 +49,14 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include @@ -960,7 +968,8 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

QueryProcessingStage::WithMergeableState && - !query.group_by_with_totals && !query.group_by_with_rollup && !query.group_by_with_cube; + !query.group_by_with_totals && !query.group_by_with_rollup && !query.group_by_with_cube && + !query.group_by_with_grouping_sets; if (query_info.projection && query_info.projection->desc->type == ProjectionDescription::Type::Aggregate) { @@ -1242,7 +1251,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

aggregationKeys()) + keys.push_back(header_before_transform.getPositionByName(key.name)); + + const Settings & settings = context->getSettingsRef(); + + Aggregator::Params params(header_before_transform, keys, query_analyzer->aggregates(), + false, settings.max_rows_to_group_by, settings.group_by_overflow_mode, 0, 0, + settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, + context->getTemporaryVolume(), settings.max_threads, settings.min_free_disk_space_for_temporary_data, + settings.compile_aggregate_expressions, settings.min_count_to_compile_aggregate_expression); + + auto transform_params = std::make_shared(params, true); + + QueryPlanStepPtr step; + step = std::make_unique(query_plan.getCurrentDataStream(), std::move(transform_params)); + + query_plan.addStep(std::move(step)); +} void InterpreterSelectQuery::executeExpression(QueryPlan & query_plan, const ActionsDAGPtr & expression, const std::string & description) { diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 4298cbbb794..8ea04cf7b27 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -160,6 +160,7 @@ private: }; void executeRollupOrCube(QueryPlan & query_plan, Modificator modificator); + void executeGroupingSets(QueryPlan & query_plan); /** If there is a SETTINGS section in the SELECT query, then apply settings from it. * diff --git a/src/Parsers/ASTSelectQuery.cpp b/src/Parsers/ASTSelectQuery.cpp index 1c5a4310f1b..f07a4a328e2 100644 --- a/src/Parsers/ASTSelectQuery.cpp +++ b/src/Parsers/ASTSelectQuery.cpp @@ -125,6 +125,9 @@ void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, F if (group_by_with_cube) s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << (s.one_line ? "" : " ") << "WITH CUBE" << (s.hilite ? hilite_none : ""); + if (group_by_with_grouping_sets) + s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << (s.one_line ? "" : " ") << "WITH GROUPING SETS" << (s.hilite ? hilite_none : ""); + if (group_by_with_totals) s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << (s.one_line ? "" : " ") << "WITH TOTALS" << (s.hilite ? hilite_none : ""); diff --git a/src/Parsers/ASTSelectQuery.h b/src/Parsers/ASTSelectQuery.h index 1c631783fdb..d1cfd1c4b1c 100644 --- a/src/Parsers/ASTSelectQuery.h +++ b/src/Parsers/ASTSelectQuery.h @@ -83,6 +83,7 @@ public: bool group_by_with_rollup = false; bool group_by_with_cube = false; bool group_by_with_constant_keys = false; + bool group_by_with_grouping_sets = false; bool limit_with_ties = false; ASTPtr & refSelect() { return getExpression(Expression::SELECT); } diff --git a/src/Parsers/ParserSelectQuery.cpp b/src/Parsers/ParserSelectQuery.cpp index 90ab5911d6b..f77fff658d6 100644 --- a/src/Parsers/ParserSelectQuery.cpp +++ b/src/Parsers/ParserSelectQuery.cpp @@ -50,6 +50,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserKeyword s_by("BY"); ParserKeyword s_rollup("ROLLUP"); ParserKeyword s_cube("CUBE"); + ParserKeyword s_grouping_sets("GROUPING SETS"); ParserKeyword s_top("TOP"); ParserKeyword s_with_ties("WITH TIES"); ParserKeyword s_offset("OFFSET"); @@ -184,18 +185,22 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) select_query->group_by_with_rollup = true; else if (s_cube.ignore(pos, expected)) select_query->group_by_with_cube = true; + else if (s_grouping_sets.ignore(pos, expected)) + select_query->group_by_with_grouping_sets = true; - if ((select_query->group_by_with_rollup || select_query->group_by_with_cube) && !open_bracket.ignore(pos, expected)) + if ((select_query->group_by_with_rollup || select_query->group_by_with_cube || select_query->group_by_with_grouping_sets) && + !open_bracket.ignore(pos, expected)) return false; if (!exp_list.parse(pos, group_expression_list, expected)) return false; - if ((select_query->group_by_with_rollup || select_query->group_by_with_cube) && !close_bracket.ignore(pos, expected)) + if ((select_query->group_by_with_rollup || select_query->group_by_with_cube || select_query->group_by_with_grouping_sets) && + !close_bracket.ignore(pos, expected)) return false; } - /// WITH ROLLUP, CUBE or TOTALS + /// WITH ROLLUP, CUBE, GROU PING SETS or TOTALS if (s_with.ignore(pos, expected)) { if (s_rollup.ignore(pos, expected)) diff --git a/src/Processors/QueryPlan/GroupingSetsStep.cpp b/src/Processors/QueryPlan/GroupingSetsStep.cpp new file mode 100644 index 00000000000..d409e0e061a --- /dev/null +++ b/src/Processors/QueryPlan/GroupingSetsStep.cpp @@ -0,0 +1,46 @@ +#include +#include +#include + +namespace DB +{ + +static ITransformingStep::Traits getTraits() +{ + return ITransformingStep::Traits + { + { + .preserves_distinct_columns = false, + .returns_single_stream = true, + .preserves_number_of_streams = false, + .preserves_sorting = false, + }, + { + .preserves_number_of_rows = false, + } + }; +} + +GroupingSetsStep::GroupingSetsStep(const DataStream & input_stream_, AggregatingTransformParamsPtr params_) + : ITransformingStep(input_stream_, params_->getHeader(), getTraits()) + , params(std::move(params_)) +{ + /// Aggregation keys are distinct + for (auto key : params->params.keys) + output_stream->distinct_columns.insert(params->params.src_header.getByPosition(key).name); +} + +void GroupingSetsStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & /*settings*/) +{ + pipeline.resize(1); + + pipeline.addSimpleTransform([&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr + { + if (stream_type == QueryPipelineBuilder::StreamType::Totals) + return nullptr; + + return std::make_shared(header, std::move(params)); + }); +} + +} diff --git a/src/Processors/QueryPlan/GroupingSetsStep.h b/src/Processors/QueryPlan/GroupingSetsStep.h new file mode 100644 index 00000000000..f27d7f1280e --- /dev/null +++ b/src/Processors/QueryPlan/GroupingSetsStep.h @@ -0,0 +1,25 @@ +#pragma once +#include +#include + +namespace DB +{ + +struct AggregatingTransformParams; +using AggregatingTransformParamsPtr = std::shared_ptr; + +/// WITH CUBE. See CubeTransform. +class GroupingSetsStep : public ITransformingStep +{ +public: + GroupingSetsStep(const DataStream & input_stream_, AggregatingTransformParamsPtr params_); + + String getName() const override { return "Grouping Sets"; } + + void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) override; + +private: + AggregatingTransformParamsPtr params; +}; + +} diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index 2a515fdf3be..c9d516f8a11 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -15,6 +15,7 @@ public: AggregatedArenasChunkInfo(Arenas arenas_) : arenas(std::move(arenas_)) {} + }; class AggregatedChunkInfo : public ChunkInfo diff --git a/src/Processors/Transforms/CubeTransform.cpp b/src/Processors/Transforms/CubeTransform.cpp index c64f39af5de..13e4963573c 100644 --- a/src/Processors/Transforms/CubeTransform.cpp +++ b/src/Processors/Transforms/CubeTransform.cpp @@ -45,7 +45,7 @@ Chunk CubeTransform::generate() consumed_chunks.clear(); auto num_rows = cube_chunk.getNumRows(); - mask = (UInt64(1) << keys.size()) - 1; + mask = (UInt64(1) << keys.size()); current_columns = cube_chunk.getColumns(); current_zero_columns.clear(); @@ -55,11 +55,11 @@ Chunk CubeTransform::generate() current_zero_columns.emplace_back(current_columns[key]->cloneEmpty()->cloneResized(num_rows)); } - auto gen_chunk = std::move(cube_chunk); + // auto gen_chunk = std::move(cube_chunk); - if (mask) + if (mask > 1) { - --mask; + mask = mask >> 1; auto columns = current_columns; auto size = keys.size(); @@ -72,6 +72,7 @@ Chunk CubeTransform::generate() chunks.emplace_back(std::move(columns), current_columns.front()->size()); cube_chunk = merge(std::move(chunks), false); } + auto gen_chunk = std::move(cube_chunk); finalizeChunk(gen_chunk); return gen_chunk; diff --git a/src/Processors/Transforms/GroupingSetsTransform.cpp b/src/Processors/Transforms/GroupingSetsTransform.cpp new file mode 100644 index 00000000000..637f244ff96 --- /dev/null +++ b/src/Processors/Transforms/GroupingSetsTransform.cpp @@ -0,0 +1,81 @@ +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +GroupingSetsTransform::GroupingSetsTransform(Block header, AggregatingTransformParamsPtr params_) + : IAccumulatingTransform(std::move(header), params_->getHeader()) + , params(std::move(params_)) + , keys(params->params.keys) +{ +// if (keys.size() >= 8 * sizeof(mask)) +// throw Exception("Too many keys are used for CubeTransform.", ErrorCodes::LOGICAL_ERROR); +} + +Chunk GroupingSetsTransform::merge(Chunks && chunks, bool final) +{ + BlocksList rollup_blocks; + for (auto & chunk : chunks) + rollup_blocks.emplace_back(getInputPort().getHeader().cloneWithColumns(chunk.detachColumns())); + + auto rollup_block = params->aggregator.mergeBlocks(rollup_blocks, final); + auto num_rows = rollup_block.rows(); + return Chunk(rollup_block.getColumns(), num_rows); +} + +void GroupingSetsTransform::consume(Chunk chunk) +{ + consumed_chunks.emplace_back(std::move(chunk)); +} + +Chunk GroupingSetsTransform::generate() +{ + if (!consumed_chunks.empty()) + { + if (consumed_chunks.size() > 1) + grouping_sets_chunk = merge(std::move(consumed_chunks), false); + else + grouping_sets_chunk = std::move(consumed_chunks.front()); + + consumed_chunks.clear(); + + auto num_rows = grouping_sets_chunk.getNumRows(); + mask = (UInt64(1) << keys.size()); + + current_columns = grouping_sets_chunk.getColumns(); + current_zero_columns.clear(); + current_zero_columns.reserve(keys.size()); + + for (auto key : keys) + current_zero_columns.emplace_back(current_columns[key]->cloneEmpty()->cloneResized(num_rows)); + } + + // auto gen_chunk = std::move(cube_chunk); + + if (mask > 1) + { + mask = mask >> 1; + + auto columns = current_columns; + auto size = keys.size(); + for (size_t i = 0; i < size; ++i) + /// Reverse bit order to support previous behaviour. + if ((mask & (UInt64(1) << (size - i - 1))) == 0) + columns[keys[i]] = current_zero_columns[i]; + + Chunks chunks; + chunks.emplace_back(std::move(columns), current_columns.front()->size()); + grouping_sets_chunk = merge(std::move(chunks), false); + } + auto gen_chunk = std::move(grouping_sets_chunk); + + finalizeChunk(gen_chunk); + return gen_chunk; +} + +} diff --git a/src/Processors/Transforms/GroupingSetsTransform.h b/src/Processors/Transforms/GroupingSetsTransform.h new file mode 100644 index 00000000000..0e300453215 --- /dev/null +++ b/src/Processors/Transforms/GroupingSetsTransform.h @@ -0,0 +1,35 @@ +#pragma once +#include +#include + + +namespace DB +{ + +/// Takes blocks after grouping, with non-finalized aggregate functions. +/// Calculates all subsets of columns and aggregates over them. +class GroupingSetsTransform : public IAccumulatingTransform +{ +public: + GroupingSetsTransform(Block header, AggregatingTransformParamsPtr params); + String getName() const override { return "GroupingSetsTransform"; } + +protected: + void consume(Chunk chunk) override; + Chunk generate() override; + +private: + AggregatingTransformParamsPtr params; + ColumnNumbers keys; + + Chunks consumed_chunks; + Chunk grouping_sets_chunk; + Columns current_columns; + Columns current_zero_columns; + + UInt64 mask = 0; + + Chunk merge(Chunks && chunks, bool final); +}; + +} diff --git a/src/Processors/ya.make b/src/Processors/ya.make new file mode 100644 index 00000000000..a1386acb6d0 --- /dev/null +++ b/src/Processors/ya.make @@ -0,0 +1,160 @@ +# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it. +LIBRARY() + +PEERDIR( + clickhouse/src/Common + contrib/libs/msgpack + contrib/libs/protobuf +) + +CFLAGS(-g0) + +SRCS( + Chunk.cpp + ConcatProcessor.cpp + DelayedPortsProcessor.cpp + Executors/ExecutingGraph.cpp + Executors/PipelineExecutingBlockInputStream.cpp + Executors/PipelineExecutor.cpp + Executors/PullingAsyncPipelineExecutor.cpp + Executors/PullingPipelineExecutor.cpp + ForkProcessor.cpp + Formats/IInputFormat.cpp + Formats/Impl/BinaryRowInputFormat.cpp + Formats/Impl/BinaryRowOutputFormat.cpp + Formats/Impl/ConstantExpressionTemplate.cpp + Formats/Impl/CSVRowInputFormat.cpp + Formats/Impl/CSVRowOutputFormat.cpp + Formats/Impl/JSONAsStringRowInputFormat.cpp + Formats/Impl/JSONCompactEachRowRowInputFormat.cpp + Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp + Formats/Impl/JSONCompactRowOutputFormat.cpp + Formats/Impl/JSONEachRowRowInputFormat.cpp + Formats/Impl/JSONEachRowRowOutputFormat.cpp + Formats/Impl/JSONEachRowWithProgressRowOutputFormat.cpp + Formats/Impl/JSONRowOutputFormat.cpp + Formats/Impl/LineAsStringRowInputFormat.cpp + Formats/Impl/MarkdownRowOutputFormat.cpp + Formats/Impl/MsgPackRowInputFormat.cpp + Formats/Impl/MsgPackRowOutputFormat.cpp + Formats/Impl/MySQLOutputFormat.cpp + Formats/Impl/NativeFormat.cpp + Formats/Impl/NullFormat.cpp + Formats/Impl/ODBCDriver2BlockOutputFormat.cpp + Formats/Impl/PostgreSQLOutputFormat.cpp + Formats/Impl/PrettyBlockOutputFormat.cpp + Formats/Impl/PrettyCompactBlockOutputFormat.cpp + Formats/Impl/PrettySpaceBlockOutputFormat.cpp + Formats/Impl/ProtobufRowInputFormat.cpp + Formats/Impl/ProtobufRowOutputFormat.cpp + Formats/Impl/RawBLOBRowInputFormat.cpp + Formats/Impl/RawBLOBRowOutputFormat.cpp + Formats/Impl/RegexpRowInputFormat.cpp + Formats/Impl/TabSeparatedRowInputFormat.cpp + Formats/Impl/TabSeparatedRowOutputFormat.cpp + Formats/Impl/TemplateBlockOutputFormat.cpp + Formats/Impl/TemplateRowInputFormat.cpp + Formats/Impl/TSKVRowInputFormat.cpp + Formats/Impl/TSKVRowOutputFormat.cpp + Formats/Impl/ValuesBlockInputFormat.cpp + Formats/Impl/ValuesRowOutputFormat.cpp + Formats/Impl/VerticalRowOutputFormat.cpp + Formats/Impl/XMLRowOutputFormat.cpp + Formats/IOutputFormat.cpp + Formats/IRowInputFormat.cpp + Formats/IRowOutputFormat.cpp + Formats/LazyOutputFormat.cpp + Formats/OutputStreamToOutputFormat.cpp + Formats/PullingOutputFormat.cpp + Formats/RowInputFormatWithDiagnosticInfo.cpp + IAccumulatingTransform.cpp + IInflatingTransform.cpp + IProcessor.cpp + ISimpleTransform.cpp + ISink.cpp + ISource.cpp + LimitTransform.cpp + Merges/Algorithms/AggregatingSortedAlgorithm.cpp + Merges/Algorithms/CollapsingSortedAlgorithm.cpp + Merges/Algorithms/GraphiteRollupSortedAlgorithm.cpp + Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.cpp + Merges/Algorithms/IMergingAlgorithmWithSharedChunks.cpp + Merges/Algorithms/MergingSortedAlgorithm.cpp + Merges/Algorithms/ReplacingSortedAlgorithm.cpp + Merges/Algorithms/SummingSortedAlgorithm.cpp + Merges/Algorithms/VersionedCollapsingAlgorithm.cpp + Merges/IMergingTransform.cpp + Merges/MergingSortedTransform.cpp + OffsetTransform.cpp + Pipe.cpp + Port.cpp + printPipeline.cpp + QueryPipeline.cpp + QueryPlan/AddingDelayedSourceStep.cpp + QueryPlan/AggregatingStep.cpp + QueryPlan/ArrayJoinStep.cpp + QueryPlan/ConvertingStep.cpp + QueryPlan/CreatingSetsStep.cpp + QueryPlan/CubeStep.cpp + QueryPlan/DistinctStep.cpp + QueryPlan/ExpressionStep.cpp + QueryPlan/ExtremesStep.cpp + QueryPlan/FillingStep.cpp + QueryPlan/FilterStep.cpp + QueryPlan/FinishSortingStep.cpp + QueryPlan/GroupingSetsStep.cpp + QueryPlan/IQueryPlanStep.cpp + QueryPlan/ISourceStep.cpp + QueryPlan/ITransformingStep.cpp + QueryPlan/LimitByStep.cpp + QueryPlan/LimitStep.cpp + QueryPlan/MergeSortingStep.cpp + QueryPlan/MergingAggregatedStep.cpp + QueryPlan/MergingSortedStep.cpp + QueryPlan/OffsetStep.cpp + QueryPlan/PartialSortingStep.cpp + QueryPlan/QueryPlan.cpp + QueryPlan/ReadFromPreparedSource.cpp + QueryPlan/ReadFromStorageStep.cpp + QueryPlan/ReadNothingStep.cpp + QueryPlan/RollupStep.cpp + QueryPlan/TotalsHavingStep.cpp + QueryPlan/UnionStep.cpp + ResizeProcessor.cpp + Sources/DelayedSource.cpp + Sources/RemoteSource.cpp + Sources/SinkToOutputStream.cpp + Sources/SourceFromInputStream.cpp + Sources/SourceWithProgress.cpp + Transforms/AddingMissedTransform.cpp + Transforms/AddingSelectorTransform.cpp + Transforms/AggregatingInOrderTransform.cpp + Transforms/AggregatingTransform.cpp + Transforms/ArrayJoinTransform.cpp + Transforms/ConvertingTransform.cpp + Transforms/CopyTransform.cpp + Transforms/CreatingSetsTransform.cpp + Transforms/CubeTransform.cpp + Transforms/DistinctTransform.cpp + Transforms/ExpressionTransform.cpp + Transforms/ExtremesTransform.cpp + Transforms/FillingTransform.cpp + Transforms/FilterTransform.cpp + Transforms/FinishSortingTransform.cpp + Transforms/GroupingSetsTransform.cpp + Transforms/JoiningTransform.cpp + Transforms/LimitByTransform.cpp + Transforms/LimitsCheckingTransform.cpp + Transforms/MaterializingTransform.cpp + Transforms/MergeSortingTransform.cpp + Transforms/MergingAggregatedMemoryEfficientTransform.cpp + Transforms/MergingAggregatedTransform.cpp + Transforms/PartialSortingTransform.cpp + Transforms/ReverseTransform.cpp + Transforms/RollupTransform.cpp + Transforms/SortingTransform.cpp + Transforms/TotalsHavingTransform.cpp + +) + +END() From 007969a74e17e8737e345a4e91cbbda72e2d2022 Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Tue, 9 Feb 2021 21:42:20 +0300 Subject: [PATCH 002/171] rm unnessesary data committed by mistake --- .../201403_1_6_1/checksums.txt | Bin 14305 -> 0 bytes .../201403_1_6_1/columns.txt | 183 ------------ .../201403_1_6_1/count.txt | 1 - .../default_compression_codec.txt | 1 - .../201403_1_6_1/minmax_StartDate.idx | 1 - .../201403_1_6_1/partition.dat | Bin 4 -> 0 bytes .../201403_1_6_1/primary.idx | Bin 3240 -> 0 bytes .../201403_7_9_1/checksums.txt | Bin 14273 -> 0 bytes .../201403_7_9_1/columns.txt | 183 ------------ .../201403_7_9_1/count.txt | 1 - .../default_compression_codec.txt | 1 - .../201403_7_9_1/minmax_StartDate.idx | 1 - .../201403_7_9_1/partition.dat | Bin 4 -> 0 bytes .../201403_7_9_1/primary.idx | Bin 1314 -> 0 bytes .../format_version.txt | 1 - .../202010_1_28_6/checksums.txt | Bin 262 -> 0 bytes .../202010_1_28_6/columns.txt | 259 ----------------- .../202010_1_28_6/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202010_1_28_6/minmax_event_date.idx | 1 - .../202010_1_28_6/partition.dat | Bin 4 -> 0 bytes .../202010_1_28_6/primary.idx | 1 - .../202012_29_2358_466/checksums.txt | Bin 11371 -> 0 bytes .../202012_29_2358_466/columns.txt | 259 ----------------- .../202012_29_2358_466/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202012_29_2358_466/minmax_event_date.idx | 1 - .../202012_29_2358_466/partition.dat | Bin 4 -> 0 bytes .../202012_29_2358_466/primary.idx | 1 - .../202101_2359_3529_235/checksums.txt | Bin 11145 -> 0 bytes .../202101_2359_3529_235/columns.txt | 259 ----------------- .../202101_2359_3529_235/count.txt | 1 - .../default_compression_codec.txt | 1 - .../minmax_event_date.idx | 1 - .../202101_2359_3529_235/partition.dat | Bin 4 -> 0 bytes .../202101_2359_3529_235/primary.idx | Bin 18 -> 0 bytes .../202102_3530_4606_216/checksums.txt | Bin 10969 -> 0 bytes .../202102_3530_4606_216/columns.txt | 259 ----------------- .../202102_3530_4606_216/count.txt | 1 - .../default_compression_codec.txt | 1 - .../minmax_event_date.idx | 1 - .../202102_3530_4606_216/partition.dat | Bin 4 -> 0 bytes .../202102_3530_4606_216/primary.idx | 1 - .../202102_4607_4607_0/checksums.txt | Bin 260 -> 0 bytes .../202102_4607_4607_0/columns.txt | 259 ----------------- .../202102_4607_4607_0/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202102_4607_4607_0/minmax_event_date.idx | 1 - .../202102_4607_4607_0/partition.dat | Bin 4 -> 0 bytes .../202102_4607_4607_0/primary.idx | 1 - .../202102_4608_4608_0/checksums.txt | Bin 260 -> 0 bytes .../202102_4608_4608_0/columns.txt | 259 ----------------- .../202102_4608_4608_0/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202102_4608_4608_0/minmax_event_date.idx | 1 - .../202102_4608_4608_0/partition.dat | Bin 4 -> 0 bytes .../202102_4608_4608_0/primary.idx | 1 - .../format_version.txt | 1 - .../hits.sql | 141 ---------- .../visits.sql | 189 ------------- .../202010_1_5_1/checksums.txt | Bin 260 -> 0 bytes .../202010_1_5_1/columns.txt | 47 ---- .../202010_1_5_1/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202010_1_5_1/minmax_event_date.idx | 1 - .../202010_1_5_1/partition.dat | Bin 4 -> 0 bytes .../202010_1_5_1/primary.idx | 1 - .../202012_6_47_9/checksums.txt | Bin 262 -> 0 bytes .../202012_6_47_9/columns.txt | 47 ---- .../202012_6_47_9/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202012_6_47_9/minmax_event_date.idx | 1 - .../202012_6_47_9/partition.dat | Bin 4 -> 0 bytes .../202012_6_47_9/primary.idx | 1 - .../202101_48_74_6/checksums.txt | Bin 261 -> 0 bytes .../202101_48_74_6/columns.txt | 47 ---- .../202101_48_74_6/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202101_48_74_6/minmax_event_date.idx | 1 - .../202101_48_74_6/partition.dat | Bin 4 -> 0 bytes .../202101_48_74_6/primary.idx | 1 - .../202102_75_79_1/checksums.txt | Bin 260 -> 0 bytes .../202102_75_79_1/columns.txt | 47 ---- .../202102_75_79_1/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202102_75_79_1/minmax_event_date.idx | 1 - .../202102_75_79_1/partition.dat | Bin 4 -> 0 bytes .../202102_75_79_1/primary.idx | 1 - .../202102_80_80_0/checksums.txt | Bin 260 -> 0 bytes .../202102_80_80_0/columns.txt | 47 ---- .../202102_80_80_0/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202102_80_80_0/minmax_event_date.idx | 1 - .../202102_80_80_0/partition.dat | Bin 4 -> 0 bytes .../202102_80_80_0/primary.idx | 1 - .../format_version.txt | 1 - .../all_1_1_0/checksums.txt | Bin 188 -> 0 bytes .../all_1_1_0/columns.txt | 13 - .../all_1_1_0/count.txt | 1 - .../all_1_1_0/default_compression_codec.txt | 1 - .../all_1_1_0/primary.idx | 1 - .../all_2_2_0/checksums.txt | Bin 188 -> 0 bytes .../all_2_2_0/columns.txt | 13 - .../all_2_2_0/count.txt | 1 - .../all_2_2_0/default_compression_codec.txt | 1 - .../all_2_2_0/primary.idx | 1 - .../format_version.txt | 1 - .../202012_1_18_4/checksums.txt | Bin 260 -> 0 bytes .../202012_1_18_4/columns.txt | 20 -- .../202012_1_18_4/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202012_1_18_4/minmax_event_date.idx | 1 - .../202012_1_18_4/partition.dat | Bin 4 -> 0 bytes .../202012_1_18_4/primary.idx | 1 - .../202101_19_22_1/checksums.txt | Bin 260 -> 0 bytes .../202101_19_22_1/columns.txt | 20 -- .../202101_19_22_1/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202101_19_22_1/minmax_event_date.idx | 1 - .../202101_19_22_1/partition.dat | Bin 4 -> 0 bytes .../202101_19_22_1/primary.idx | 1 - .../format_version.txt | 1 - .../201403_1_29_2/checksums.txt | Bin 10792 -> 0 bytes .../201403_1_29_2/columns.txt | 135 --------- .../201403_1_29_2/count.txt | 1 - .../default_compression_codec.txt | 1 - .../201403_1_29_2/minmax_EventDate.idx | 1 - .../201403_1_29_2/partition.dat | Bin 4 -> 0 bytes .../201403_1_29_2/primary.idx | Bin 15330 -> 0 bytes .../format_version.txt | 1 - .../202010_1_4_1/checksums.txt | Bin 260 -> 0 bytes .../202010_1_4_1/columns.txt | 12 - .../202010_1_4_1/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202010_1_4_1/minmax_event_date.idx | 1 - .../202010_1_4_1/partition.dat | Bin 4 -> 0 bytes .../202010_1_4_1/primary.idx | 1 - .../202012_35_35_0/checksums.txt | Bin 260 -> 0 bytes .../202012_35_35_0/columns.txt | 12 - .../202012_35_35_0/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202012_35_35_0/minmax_event_date.idx | 1 - .../202012_35_35_0/partition.dat | Bin 4 -> 0 bytes .../202012_35_35_0/primary.idx | 1 - .../202012_5_34_6/checksums.txt | Bin 261 -> 0 bytes .../202012_5_34_6/columns.txt | 12 - .../202012_5_34_6/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202012_5_34_6/minmax_event_date.idx | 1 - .../202012_5_34_6/partition.dat | Bin 4 -> 0 bytes .../202012_5_34_6/primary.idx | 1 - .../202101_36_115_17/checksums.txt | Bin 262 -> 0 bytes .../202101_36_115_17/columns.txt | 12 - .../202101_36_115_17/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202101_36_115_17/minmax_event_date.idx | 1 - .../202101_36_115_17/partition.dat | Bin 4 -> 0 bytes .../202101_36_115_17/primary.idx | 1 - .../202102_116_127_3/checksums.txt | Bin 261 -> 0 bytes .../202102_116_127_3/columns.txt | 12 - .../202102_116_127_3/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202102_116_127_3/minmax_event_date.idx | 1 - .../202102_116_127_3/partition.dat | Bin 4 -> 0 bytes .../202102_116_127_3/primary.idx | 1 - .../202102_128_128_0/checksums.txt | Bin 260 -> 0 bytes .../202102_128_128_0/columns.txt | 12 - .../202102_128_128_0/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202102_128_128_0/minmax_event_date.idx | 1 - .../202102_128_128_0/partition.dat | Bin 4 -> 0 bytes .../202102_128_128_0/primary.idx | 1 - .../202102_129_129_0/checksums.txt | Bin 261 -> 0 bytes .../202102_129_129_0/columns.txt | 12 - .../202102_129_129_0/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202102_129_129_0/minmax_event_date.idx | 1 - .../202102_129_129_0/partition.dat | Bin 4 -> 0 bytes .../202102_129_129_0/primary.idx | 1 - .../format_version.txt | 1 - .../202010_1_5_1/checksums.txt | Bin 260 -> 0 bytes .../202010_1_5_1/columns.txt | 41 --- .../202010_1_5_1/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202010_1_5_1/minmax_event_date.idx | 1 - .../202010_1_5_1/partition.dat | Bin 4 -> 0 bytes .../202010_1_5_1/primary.idx | 1 - .../202012_6_45_8/checksums.txt | Bin 262 -> 0 bytes .../202012_6_45_8/columns.txt | 41 --- .../202012_6_45_8/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202012_6_45_8/minmax_event_date.idx | 1 - .../202012_6_45_8/partition.dat | Bin 4 -> 0 bytes .../202012_6_45_8/primary.idx | 1 - .../202101_46_72_6/checksums.txt | Bin 262 -> 0 bytes .../202101_46_72_6/columns.txt | 41 --- .../202101_46_72_6/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202101_46_72_6/minmax_event_date.idx | 1 - .../202101_46_72_6/partition.dat | Bin 4 -> 0 bytes .../202101_46_72_6/primary.idx | 1 - .../202102_73_77_1/checksums.txt | Bin 261 -> 0 bytes .../202102_73_77_1/columns.txt | 41 --- .../202102_73_77_1/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202102_73_77_1/minmax_event_date.idx | 1 - .../202102_73_77_1/partition.dat | Bin 4 -> 0 bytes .../202102_73_77_1/primary.idx | 1 - .../202102_78_78_0/checksums.txt | Bin 260 -> 0 bytes .../202102_78_78_0/columns.txt | 41 --- .../202102_78_78_0/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202102_78_78_0/minmax_event_date.idx | 1 - .../202102_78_78_0/partition.dat | Bin 4 -> 0 bytes .../202102_78_78_0/primary.idx | 1 - .../format_version.txt | 1 - .../202010_1_29_6/checksums.txt | Bin 262 -> 0 bytes .../202010_1_29_6/columns.txt | 15 - .../202010_1_29_6/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202010_1_29_6/minmax_event_date.idx | 1 - .../202010_1_29_6/partition.dat | Bin 4 -> 0 bytes .../202010_1_29_6/primary.idx | 1 - .../202012_2358_2361_1/checksums.txt | Bin 260 -> 0 bytes .../202012_2358_2361_1/columns.txt | 15 - .../202012_2358_2361_1/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202012_2358_2361_1/minmax_event_date.idx | 1 - .../202012_2358_2361_1/partition.dat | Bin 4 -> 0 bytes .../202012_2358_2361_1/primary.idx | 1 - .../202012_30_2357_1895/checksums.txt | Bin 263 -> 0 bytes .../202012_30_2357_1895/columns.txt | 15 - .../202012_30_2357_1895/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202012_30_2357_1895/minmax_event_date.idx | 1 - .../202012_30_2357_1895/partition.dat | Bin 4 -> 0 bytes .../202012_30_2357_1895/primary.idx | 1 - .../202101_2362_3536_470/checksums.txt | Bin 263 -> 0 bytes .../202101_2362_3536_470/columns.txt | 15 - .../202101_2362_3536_470/count.txt | 1 - .../default_compression_codec.txt | 1 - .../minmax_event_date.idx | 1 - .../202101_2362_3536_470/partition.dat | Bin 4 -> 0 bytes .../202101_2362_3536_470/primary.idx | Bin 24 -> 0 bytes .../202101_3537_3539_1/checksums.txt | Bin 260 -> 0 bytes .../202101_3537_3539_1/columns.txt | 15 - .../202101_3537_3539_1/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202101_3537_3539_1/minmax_event_date.idx | 1 - .../202101_3537_3539_1/partition.dat | Bin 4 -> 0 bytes .../202101_3537_3539_1/primary.idx | 1 - .../202102_3540_4637_415/checksums.txt | Bin 263 -> 0 bytes .../202102_3540_4637_415/columns.txt | 15 - .../202102_3540_4637_415/count.txt | 1 - .../default_compression_codec.txt | 1 - .../minmax_event_date.idx | 1 - .../202102_3540_4637_415/partition.dat | Bin 4 -> 0 bytes .../202102_3540_4637_415/primary.idx | 1 - .../202102_4638_4638_0/checksums.txt | Bin 260 -> 0 bytes .../202102_4638_4638_0/columns.txt | 15 - .../202102_4638_4638_0/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202102_4638_4638_0/minmax_event_date.idx | 1 - .../202102_4638_4638_0/partition.dat | Bin 4 -> 0 bytes .../202102_4638_4638_0/primary.idx | 1 - .../format_version.txt | 1 - .../asynchronous_metric_log.sql | 12 - .../crash_log.sql | 17 -- .../metric_log.sql | 264 ------------------ .../part_log.sql | 25 -- .../query_log.sql | 52 ---- .../query_thread_log.sql | 46 --- .../text_log.sql | 20 -- .../trace_log.sql | 17 -- .../202010_1_3_1/checksums.txt | Bin 260 -> 0 bytes .../202010_1_3_1/columns.txt | 7 - .../202010_1_3_1/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202010_1_3_1/minmax_event_date.idx | 1 - .../202010_1_3_1/partition.dat | Bin 4 -> 0 bytes .../202010_1_3_1/primary.idx | 1 - .../202012_4_295_59/checksums.txt | Bin 262 -> 0 bytes .../202012_4_295_59/columns.txt | 7 - .../202012_4_295_59/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202012_4_295_59/minmax_event_date.idx | 1 - .../202012_4_295_59/partition.dat | Bin 4 -> 0 bytes .../202012_4_295_59/primary.idx | 1 - .../202101_296_439_30/checksums.txt | Bin 262 -> 0 bytes .../202101_296_439_30/columns.txt | 7 - .../202101_296_439_30/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202101_296_439_30/minmax_event_date.idx | 1 - .../202101_296_439_30/partition.dat | Bin 4 -> 0 bytes .../202101_296_439_30/primary.idx | 1 - .../202102_440_574_27/checksums.txt | Bin 262 -> 0 bytes .../202102_440_574_27/columns.txt | 7 - .../202102_440_574_27/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202102_440_574_27/minmax_event_date.idx | 1 - .../202102_440_574_27/partition.dat | Bin 4 -> 0 bytes .../202102_440_574_27/primary.idx | 1 - .../202102_575_575_0/checksums.txt | Bin 260 -> 0 bytes .../202102_575_575_0/columns.txt | 7 - .../202102_575_575_0/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202102_575_575_0/minmax_event_date.idx | 1 - .../202102_575_575_0/partition.dat | Bin 4 -> 0 bytes .../202102_575_575_0/primary.idx | 1 - .../202102_576_576_0/checksums.txt | Bin 260 -> 0 bytes .../202102_576_576_0/columns.txt | 7 - .../202102_576_576_0/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202102_576_576_0/minmax_event_date.idx | 1 - .../202102_576_576_0/partition.dat | Bin 4 -> 0 bytes .../202102_576_576_0/primary.idx | 1 - .../202102_577_577_0/checksums.txt | Bin 260 -> 0 bytes .../202102_577_577_0/columns.txt | 7 - .../202102_577_577_0/count.txt | 1 - .../default_compression_codec.txt | 1 - .../202102_577_577_0/minmax_event_date.idx | 1 - .../202102_577_577_0/partition.dat | Bin 4 -> 0 bytes .../202102_577_577_0/primary.idx | 1 - .../format_version.txt | 1 - 324 files changed, 3761 deletions(-) delete mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/checksums.txt delete mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/columns.txt delete mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/count.txt delete mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/default_compression_codec.txt delete mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/minmax_StartDate.idx delete mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/partition.dat delete mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/primary.idx delete mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/checksums.txt delete mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/columns.txt delete mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/count.txt delete mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/default_compression_codec.txt delete mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/minmax_StartDate.idx delete mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/partition.dat delete mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/primary.idx delete mode 100644 programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/format_version.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/checksums.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/columns.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/count.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/default_compression_codec.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/minmax_event_date.idx delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/partition.dat delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/primary.idx delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202012_29_2358_466/checksums.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202012_29_2358_466/columns.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202012_29_2358_466/count.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202012_29_2358_466/default_compression_codec.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202012_29_2358_466/minmax_event_date.idx delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202012_29_2358_466/partition.dat delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202012_29_2358_466/primary.idx delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202101_2359_3529_235/checksums.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202101_2359_3529_235/columns.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202101_2359_3529_235/count.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202101_2359_3529_235/default_compression_codec.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202101_2359_3529_235/minmax_event_date.idx delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202101_2359_3529_235/partition.dat delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202101_2359_3529_235/primary.idx delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_3530_4606_216/checksums.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_3530_4606_216/columns.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_3530_4606_216/count.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_3530_4606_216/default_compression_codec.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_3530_4606_216/minmax_event_date.idx delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_3530_4606_216/partition.dat delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_3530_4606_216/primary.idx delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/checksums.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/columns.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/count.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/default_compression_codec.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/minmax_event_date.idx delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/partition.dat delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/primary.idx delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/checksums.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/columns.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/count.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/default_compression_codec.txt delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/minmax_event_date.idx delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/partition.dat delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/primary.idx delete mode 100644 programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/format_version.txt delete mode 100644 programs/server/store/2c6/2c6da17b-121a-4775-b287-b0d80cd04080/hits.sql delete mode 100644 programs/server/store/2c6/2c6da17b-121a-4775-b287-b0d80cd04080/visits.sql delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/checksums.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/columns.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/count.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/default_compression_codec.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/minmax_event_date.idx delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/partition.dat delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/primary.idx delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/checksums.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/columns.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/count.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/default_compression_codec.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/minmax_event_date.idx delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/partition.dat delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/primary.idx delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202101_48_74_6/checksums.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202101_48_74_6/columns.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202101_48_74_6/count.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202101_48_74_6/default_compression_codec.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202101_48_74_6/minmax_event_date.idx delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202101_48_74_6/partition.dat delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202101_48_74_6/primary.idx delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/checksums.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/columns.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/count.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/default_compression_codec.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/minmax_event_date.idx delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/partition.dat delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/primary.idx delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_80_80_0/checksums.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_80_80_0/columns.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_80_80_0/count.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_80_80_0/default_compression_codec.txt delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_80_80_0/minmax_event_date.idx delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_80_80_0/partition.dat delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_80_80_0/primary.idx delete mode 100644 programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/format_version.txt delete mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/checksums.txt delete mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/columns.txt delete mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/count.txt delete mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/default_compression_codec.txt delete mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/primary.idx delete mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/checksums.txt delete mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/columns.txt delete mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/count.txt delete mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/default_compression_codec.txt delete mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/primary.idx delete mode 100644 programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/format_version.txt delete mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/checksums.txt delete mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/columns.txt delete mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/count.txt delete mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/default_compression_codec.txt delete mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/minmax_event_date.idx delete mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/partition.dat delete mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/primary.idx delete mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/checksums.txt delete mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/columns.txt delete mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/count.txt delete mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/default_compression_codec.txt delete mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/minmax_event_date.idx delete mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/partition.dat delete mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/primary.idx delete mode 100644 programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/format_version.txt delete mode 100644 programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/checksums.txt delete mode 100644 programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/columns.txt delete mode 100644 programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/count.txt delete mode 100644 programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/default_compression_codec.txt delete mode 100644 programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/minmax_EventDate.idx delete mode 100644 programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/partition.dat delete mode 100644 programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/primary.idx delete mode 100644 programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/format_version.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/checksums.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/columns.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/count.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/default_compression_codec.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/minmax_event_date.idx delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/partition.dat delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/primary.idx delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/checksums.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/columns.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/count.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/default_compression_codec.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/minmax_event_date.idx delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/partition.dat delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/primary.idx delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/checksums.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/columns.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/count.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/default_compression_codec.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/minmax_event_date.idx delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/partition.dat delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/primary.idx delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/checksums.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/columns.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/count.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/default_compression_codec.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/minmax_event_date.idx delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/partition.dat delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/primary.idx delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/checksums.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/columns.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/count.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/default_compression_codec.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/minmax_event_date.idx delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/partition.dat delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/primary.idx delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/checksums.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/columns.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/count.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/default_compression_codec.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/minmax_event_date.idx delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/partition.dat delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/primary.idx delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_129_129_0/checksums.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_129_129_0/columns.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_129_129_0/count.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_129_129_0/default_compression_codec.txt delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_129_129_0/minmax_event_date.idx delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_129_129_0/partition.dat delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_129_129_0/primary.idx delete mode 100644 programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/format_version.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/checksums.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/columns.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/count.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/default_compression_codec.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/minmax_event_date.idx delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/partition.dat delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/primary.idx delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/checksums.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/columns.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/count.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/default_compression_codec.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/minmax_event_date.idx delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/partition.dat delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/primary.idx delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/checksums.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/columns.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/count.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/default_compression_codec.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/minmax_event_date.idx delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/partition.dat delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/primary.idx delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/checksums.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/columns.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/count.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/default_compression_codec.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/minmax_event_date.idx delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/partition.dat delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/primary.idx delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/checksums.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/columns.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/count.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/default_compression_codec.txt delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/minmax_event_date.idx delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/partition.dat delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/primary.idx delete mode 100644 programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/format_version.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/checksums.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/columns.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/count.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/default_compression_codec.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/minmax_event_date.idx delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/partition.dat delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/primary.idx delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_2358_2361_1/checksums.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_2358_2361_1/columns.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_2358_2361_1/count.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_2358_2361_1/default_compression_codec.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_2358_2361_1/minmax_event_date.idx delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_2358_2361_1/partition.dat delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_2358_2361_1/primary.idx delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_30_2357_1895/checksums.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_30_2357_1895/columns.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_30_2357_1895/count.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_30_2357_1895/default_compression_codec.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_30_2357_1895/minmax_event_date.idx delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_30_2357_1895/partition.dat delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_30_2357_1895/primary.idx delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_2362_3536_470/checksums.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_2362_3536_470/columns.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_2362_3536_470/count.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_2362_3536_470/default_compression_codec.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_2362_3536_470/minmax_event_date.idx delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_2362_3536_470/partition.dat delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_2362_3536_470/primary.idx delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/checksums.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/columns.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/count.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/default_compression_codec.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/minmax_event_date.idx delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/partition.dat delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/primary.idx delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_3540_4637_415/checksums.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_3540_4637_415/columns.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_3540_4637_415/count.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_3540_4637_415/default_compression_codec.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_3540_4637_415/minmax_event_date.idx delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_3540_4637_415/partition.dat delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_3540_4637_415/primary.idx delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/checksums.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/columns.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/count.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/default_compression_codec.txt delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/minmax_event_date.idx delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/partition.dat delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/primary.idx delete mode 100644 programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/format_version.txt delete mode 100644 programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/asynchronous_metric_log.sql delete mode 100644 programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/crash_log.sql delete mode 100644 programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/metric_log.sql delete mode 100644 programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/part_log.sql delete mode 100644 programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/query_log.sql delete mode 100644 programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/query_thread_log.sql delete mode 100644 programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/text_log.sql delete mode 100644 programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/trace_log.sql delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/checksums.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/columns.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/count.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/default_compression_codec.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/minmax_event_date.idx delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/partition.dat delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/primary.idx delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/checksums.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/columns.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/count.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/default_compression_codec.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/minmax_event_date.idx delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/partition.dat delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/primary.idx delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/checksums.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/columns.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/count.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/default_compression_codec.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/minmax_event_date.idx delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/partition.dat delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/primary.idx delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_440_574_27/checksums.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_440_574_27/columns.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_440_574_27/count.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_440_574_27/default_compression_codec.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_440_574_27/minmax_event_date.idx delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_440_574_27/partition.dat delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_440_574_27/primary.idx delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/checksums.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/columns.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/count.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/default_compression_codec.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/minmax_event_date.idx delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/partition.dat delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/primary.idx delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/checksums.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/columns.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/count.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/default_compression_codec.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/minmax_event_date.idx delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/partition.dat delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/primary.idx delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/checksums.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/columns.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/count.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/default_compression_codec.txt delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/minmax_event_date.idx delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/partition.dat delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/primary.idx delete mode 100644 programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/format_version.txt diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/checksums.txt b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/checksums.txt deleted file mode 100644 index da44b12b708c635f7165a2c8b8437d5fd40cbb12..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14305 zcmX|IcOaGj_kTaneJ-xOcVuLhl@W=oGD2h*nHM+X%9e~oMo5Svqe3!DMOId1WklJO zQpzaz+GPBm^7;P$z6V{1KAsMK2Ltdvz67Gz@q_Zz z?VqOPubs%3R#X^Fo*H1>j;uTez!g0JHV#K&7&M#%PILz-SGmo243KC&ecX>u2_S-^7rXgRoMniX%9$dE9AAjJQ^BMy58h(CwFJ}iYr*mMP=l9({B-tiDE1Y5JYY;_7 zk4cc?B$St(gyTDrKvAQ3?5TC9HG9&IxhK7vpiEqa0V& zehL99ErTYA%-t4EU-;_2!DnhiYwsSj%fq=EjZjhhBUnt3QiFrSas7IPPGO|8F%?c0 zz(#<~dA#s!_;tI3=I*%fo8r{Tt$E6aix6o zF9EFIU#-6SHR#^{{#Q4&2XnBHZ*YXNODHGgHxAc$BRm=afxW>JCVbrY;)5_(PTx?a zouvF09AK0N_UOHMdizD9){3NGm7V37OZa5mJS#VDpQ~-_0yg2=2d~R3wgJpEZF-7L_(4?;P_v#CGL8|8^GG> z*FCs!T>#pFySQfi;=5ZjSc7f!9cCgp4)oCI-S`xmKlqGZ5x`dHR;9%` zowhcRpTFFu--iX$)E$5vwTYe%1g}ukR(~_HPDAa$q4Axfwp6|I*%R}J=y5|a`ja2Z z`^S09;zc)|al-2dp#L4*m64nW)ZvDu0?I0NL0bYe%J9Kc0aqcY5g{Y%o!ASI!Df)k z?Cp=0k}8@XY&7jqV|{{=QjzgLIM$DJrqqzFrrBXc8Ne6OTkX3YP1w_Ooc1Z`@Vb2D zY9ojO&>E`*kAKoV+E{a%_Dra9w)gI|Bhb(x5}gkNpoZET>A<8uJyQLCERz+wCd5{5 zwA!z<}9rwX#<7DHU>jI8o-y9k9@tO^lQ;@B6@rhy8Q~fe(fISYF>-Z4; zy}MvCC7SUz9zy3PxW2&y6Ka2W{unW1EpNMkJvh%`9HY0>W7L18ui*ZV(|nzQsSuvOpv}&8D#x5CA=V)PU&f2*}(M$vBHQH<+*IyAF9EWg640)3>1aN$rSM z%1PNSZpT_jE#*gtwGQDK9081m#Cbr*oK-eixS4>ze@n2#v+#^b6W|g8Ukx36-0^-| z4t{u7qE9d$Fth_UxF4a@1ZYgsbpf~mp_T#+0L#PA`vL$6eeMJ3AS`?fMt@*3K7}xh zp>O2F`}a*Z0AL(~hyW-aFmz+tVNDo^`lkz^YXeM-Yry-gw;zfXzTNN=XU{WC@7=rN zk(aWJU%ybap}d`lsZ=7HygI- zwM$$01bm!#Rdz~lNl>J0>|O>KbXhj&vK}JGKmSg%l%q?2;bx3y@x@|+FhGK#CC-9} zn7PWkW1+EZ>`I?fC|G(vm_T3p&kiMih)y46_Feh1$CUJ``qu=6MVIA@TY+Szq`;Z$ zf<+5rsp2#f-^c8LmYa`*@81klnT1V0EiegdRZnYw)*{?<)(WUcKb%iL4r0^%LViyZ zuzDguMcE!8|7ENy@uWeK*ItS}YD(Jefm)A(AuVGM2PZq!)@&Q5=H3x5zWcqh#%s4# z9-mIC`Wtt3Yr!5&Gi2?w$cf1Kw>vw*(EsA@AcKi1L)DexlSjF}JrT}-3fhryV;=(1 zhu}Aa;e~l9DgU7i-7bt`jq^FYFVWxUEnu+6u%W-Be`9b4)aIsm54=+$B?-Cx2%i0| zX|^)d`0GNiTy>ZA1;IkfCFt~9vyJ6Q1sUtG#vr#4I~ylJiN>iD8VBiC`Qx1vadlDe z0sp^MxeeN9|r5&Dm zb#1aEKj_$(3hE`Ogk|IpgGJzUYNb)DQh7kZ`@j1_cef0g%zos!q@?;jjw!e*dMc>R z4+Aa&xaY51zQfet*0!In$awXr?a`RNo?fmMyae@TkDnY&a;Hv-*Qpav60PyW?7I>f zW0QRNbLmSpOwWfu*L?UUya4S2SmHN7XYiLVHC=-$ISxpQ_?i}O(|F0ZQ%6QYwLN;D zxELIFKhHYPQN(^btXpG<#Q}K$n*%bGf@C(i9J%*v^T@EO>0&?u8zhSYRtY7-L`BA4 z>wE+g=YDw8?T!*Ml#pzRTjO-! zhT6w>&gam5lL}I*1=usxWbCi}J?K&1J*BJKUAf0=f8g{RZ;4+-4bAC14asPGlcnq- zX!oP>;{0m>qX1)x8lnwNj9=p_d>G;b>)yYcEUOa|Niz|)bZ|H;xGyNIVzj_A(*zHgsJnX0KDFC z7enmv$-vgBH;lZ?R8t|Hfg{d!yg3;%3JRZwD9?=;b4> z^AlL14VX;2t>x{N5WRLZXvik+;}qk+m4|SC{lgd7O*6jEDw6#5^e>sqh zv39yH`7U|Td59{ju5EHw4$zs=3*vn~qj;4#$NrU&ZNX0XshpTF+yCKhkyI55^A~9O zo}LQ1ID74QxoqatQQm!+Cs7t2i+&mBNqzI8^#p~rjrSU$)%N#sz*7Qx^d7qJ--N;* z543KVy}FB(HeUL%8&P5fR$<-+qlKCKdxu@F4f^lMQmoMZyHXOxJ%?b6&p#&nH%25! zh~2jKfclh|^Dey1hx+ZO#9hWmGj`7lMN;p+HG5mb^7$1vM1IgE5@ySNjT0%0iYx)l zl~}36G^pR2qV)W|k)Fysoh`kR$N2nGy5ND$2mi5iy3D3#^Ybb#LU;B!5^J5d!%=e{ z4!&-N4wr~Nlqk=Tr(KZ zty+T>h~e0oxck;uT=ebwZ_S^QF}DVS1TWMRk@hW)S{CM4JsS5j1GWoV4lK%bTBGTq z-&w)%ahgG&sA9MiN1yJTHGuwEB4hf&*;4B=l3ebuV`73SZmodnD!_EYWSrdJa9__6 z#k)=Gu1cPtuIx{@3Uj>II*lW&W4$x1hR+q`uG0;&1iy zzAN<3|2DP^>T*x<7n2RW%3#ks%$rQm$Bi&Cg>`3yLnLx7Tn&813>PtSCD9a z&hsl=DPOFL3uRVUwwh)DtU7p<&O3PMg<@p!zJ3YRJ3LP&5-^Q?yZp3;aXMkEHZ|(W zWIZT3^A~ngErzRaojQEXK+TwARPG&{8Va~z2U8O5c4x|O;*LVDrLF2WH)_Bw4vc*X zSMV~O=yt0ghtzaknn~#r#wiE7dKA6f(BnfZ7~FG<5}mY6I{~lHx8u7CCU~^a3dRSJ z8Tq~)nBa`Aew5W+IhhGpdje)1qJzgdyn~Y)-dDq`l$neyK4F~V)_vRlhY;2qv^#j4 zbKLD5<`1l1{_-pNX^;NLdm`&tvPst>b6iDU=;-#+IVlj z1(b__$)fm_buk9cau_UsACphWUQKicJ1|CFqT{RN1^>aU8;z|j2D)r${zgkBDAQf1 z#Uf<~|CRjk%$+=z(;WcggDhc+mzX$C-o#aG-jtDHe`PLo;VU0@0}76A2{(4N_Me$L z+XPSd-Rl5gKSN?>-Bb?=HHyoP$7=-p{iYrO>^H$n-$O@FY5Dv3;JuvAQ}1!zxm?XF zt)L-x{kBT6&)VhPCQGPe{WYX%c|7jr#Ob>z(IT;LQ3R z|K9xVWt<9F0VvzWv7NT5VSai*xQ;JtV+=}O+=SO-ZEFIy zp31|`^GDB%nD;Hg0Oj)kqJ0Ts_%239*xTLTF9#2vscfh)hJgsMH^uu996Ta1WaNFL zu<{G{FO@qxdd^vrl9AGdmk&a#Q0m=b>dqg`OfRZWw9Uuas_X#Fzgt~`&0hbKp=V4? z&z+MLeljFu0(2mXFQ}nf5~ucwyfF2q0KXw8j}M+UVV}{*Fli%?aPq9|*zt`o>Ssv$ z_ZheTqr)qd!?>QoQ}tt-kMvn25~!mu09rJA=rpmBkdjh}Z6n%b)ggv2K)u$g5 zL^OLyxKU;uJ}Yqk=-k0CTF-wTru#=RNHXfoE;qf6pH@@CECTZ_^4BSQ_3)8yUv1&bxxTW{HI->1LHAsyn< z5SY}U0}Oh`MtHwKqK|tROvVmx)IpWiw*^hIwt*q9$RKl-%$x<-fMrSAUcq%XhS2tb z7K3wDsf;K^N9zE}=G~mGr67sWyNq4T$B|%nK&|KHM6_ceArtq7`@k1ewyLr8QhkO? zx>+l*i)u1+>=J!BTH$h;507TJa%F1J90jlm*u;crPiS4YIv>%Tqmf(wOYbY7+e9g2 zNv?ike%|3D?hG#J_j46g4Sph6$~D-x>|WLqUYTF^tsxSwa!~NUTAk{g%A?DVRqLg= zs8c58p5x;It)8zY(LailgnWE*KuP7o8H(S`9H~czsh+E(NtU1p9++|Qp^D?9DBI3a z&u`uSxA#w*MAwxtfV;Tu_8~A-`r6B@d+C5)&sUom=;c9laIS_)$cHVsfNDj~{l)LE zm%MhcrlBp7mT2&hz3C}veH4mrVLj!(^NPpqqa+DB%cE!f>LfM8dj_jWA_slMQ=n-` zbR-N@laQ|;ndj4w56WJP0;3KgtO-YvJkF(*uyXW(395OfYL7dAy=*?bgB z)MB;gF681kcd1e7>MY7gH4v?VaY7xV^mszq^5#9Yd|+?w;N^@*Us~lHHVb5X;ZRZZ z`-`)=i$KkJ`t`HA-GO z#)TY{M-MC!%P)wY!5pC1cL;D0r6nOZ+jx(Lv(=sMf*zC~n6!G*67snWJ#o@PiF>Kl zoWJb6oa7C4!t8%1j-yMn}Jr05AJLpF{Fbax6VCLJ>15~}l>a3U;R?=8 zoEV52jk|2sL;7-MlNRL@=BPqw2z$}7y?1%t^k~5Eljm#_;F)8)iz1IU!3Xc;_nK>u zqX9BAdU2M(aZgo!sqp!qrNwM0?}*TkWU&}Atok9V-;N8`XB>Go02aOs{$Qke8t>-Y zF<=^O4woD{HNXS7WdL%7b(;%a+$)IgcSZ8Bx^Dck@6&$VrP*!NQbzQ39^|tT&k3hD z!e`5KI$$6IRz#5H-&?mJiq_X0mG*wu@XbNg$b!2c`j#AD$4%_Jv^RWu(?LbP z!35%eZy{_fp8!apfH^SR93vDyp9Nlhkat~hkm-AHB^+r$-#ld2=)Wn~eEByKs^}cl zWMNeX3=V*ig&2g{VBe!}2c~wz1LX;N%HKOYhnF^?FY;^zKKs)TRA{$_J{>M^)0J}o z5SqA9n8V~5sD>>i-;(6n4yL?DIk^ki;(##GIi6#ebLg;q{+q~^N85CX#c5iL@|}y& z{Yt2~@1gIJ^YIGrFYjbl@cyi4|0m>Z!9l4b;Bq`c?Q3`z;VyaD!SjI=X zwXiL2d-=?J%@=g-FUFuIt93=U%cpKz#i}*5STPxBU*K>7FlxsYO@QxT9#dTHw4LnE zpX>e3_I4d`9R?95{tjM#1ixT_U1QI{^ti-I|NiB02=S@d271!Kz{8WX4HVx2xzT`Y@KeOKmq+@{VvBb6#hTWJDp)3eX*b*qDR?Fk=Pr z&=$uOBy->UwQERq%{ZpL?p=wbFBD5dk<60((=ajKU!9mz+!QawjFJ^p8-wT9suc%5 zOg$wo+7;*tdKrUj00UG16GzA$usc`1{x@Gx@A-Ip4EO4%<8kU`=q1NT@}BFpfWxlO zHiN0tRMP(^j$!n278l05?n~gSVw*W%=HLLsS4v@Hw<|0pObpA388-g7S*z;3cQ_N} zPnc3AiVfKBk-(*qk=b(x1Y>e%JRZBEEgQ5ivW}ro{yZ?@!^#tPXSBe4nHmFFg@K0Q z-+G#6hx>k-Kx)kxvwLfo2xl8h2}#%fDfH{}EvWG{Rq>TK(!X7zw+ELi+Ku<=>nerq z^!N1UFoPx@c8%yS-^8i_EsEl;Oe7>@g0fzpU7XvK?sea@Lo9LYSqu#*o}4KcGvn{k zhdiV;gN;8dkpH6=EzU~cnB>c8Vwgx zoII`$RLxvn{XLFEQJyu;WL}}Q+q07T$l;pxG+SL~Cj9YDz~;HMEzvB{YW zUv=A_S3VJcfl9i`>)|AyJ9Mo@UybR(`L^R0ruThz(k;@oF8`H83c&{ERh!R?citHU zW)Cp19;C_$SPuat8m*7Ml98=4@#7cep3^e0zEBbXdgaV$1EqK>>vufW8;!d~hZjT= zj%@5Z*b1i9cIO8!xBHUh?xCl9%(kBayEs5D3&Bq)_qe7upaD^lx`(qr)tWi`7Kn#m z4?x|;`i7w=Jk(xnoUIztZSQw~JNjQivy18e@lc=7l1;2qJ*v5fAU+7V4+CTnEh)!8 zAYB~SK(k0r0ayBo#e_p z)AJX7>3Bdk@Iz^S0Kp00Ou51}Rt>r4DKHnwILTOY)k%r>QJse76AYepq+WW$z^C}Y zajc>E3C$v>7ob8@q&pfGKTQPGU*A8vUQt>b6W`GE5M%;MRQddbuC09)!`2XS94E&4 zVdI#uj5}Qe;S;(x>@2B?aZx{|eljyru&U_{uQ%EUK;6L)tU*`mzu<@q$7BAOON=Kq zNGTnJIZW+%S2*bD6rL@Bq$-(U{2UMCGA$h?|a(4S@r@Ol>G4D z_9Py4xr(=M9!*_Lyy6|xtXID6nGCQH1`nVV@Gph^E+i3p`(IJ!wVe^_l1Yh(Twqv(JHqcOG+UG^&EhgfW^F|+_G$JCRlDu) z?cf~64G!ufb>rtAHlvLX2OkH|R2Ea68MtqSg2ue4B}CP zK&@45?GWZ!pXEJ{m3^sZ6T_e#bI*gpcOV*wNdE%W^DEQ8K+fk0Pinf7$s8@{0S`{$sF%u%XLwT=G4 z15qr-Q_2s@#dhsXaL&NWK z!LuJ&3NtodpiKvL4$Jeex7yUBc=de3dIfo&{jceO!*tFtaA^U@Yj~yc19e*hfCD{r zU%az3*gJ%hzBBk>*}pnj_d8xGEA~^pDgGbYU$6VuQ|5a>Z*DHb-9JsU>}US8E?GaB zTgSNIIn@D#O82Sqh=I)(ACc!0yosZ6(~N7ATYN|QJ|Kf z3a&lQU*)tG0lD-Rv8_d-LDVMvwbnY&EwTwKn{Qtr&=VedlkL+}aje%J0HlU%58`Ai zAHf|~a?Sew@nLrjUeg^l(}mB`d zcoSmG8EukOjGm>EO>jEs@vLP4#-Qoz5~`%!egE)0AM2Vlw^xX6s?&xzpt;KMFGaj* z#~d%PvW*G7DRg6M>Ntplh_#k|zm^h%SfQ0Z&CS|yR)H!mIeFA&e3dU`{BJbP;S z`ufPHHf@~J^AvOaxnJQdV?6z~IhUJ%Nl!HRpI04&1fXNjV zAmHC1yWG8%_C}Fu!3r0QQ;Gh9M#7_Cu~5^>N6I3e_JRiU!YZFn9ieJ1MO*Bsy6FaX zF;juZDm*?ER$miz*v@Y~?E-jEsy>Ax)e&}q$^Gm(cj#sUpBz_5s?^sxx2TU>mPLu%0pe^o(mXJ74 z!V_y^siO9zP3CQ!nkQ8MrU9#R=9!F-S8hbd={rwk%07GiJ*?(Ks}Km3TYhpBcKcp+ zvufQTrv3zk4p<1U2{KBz6K(Xjz@HEwO_rqVaLf{sVKZkdHD7wr&nW5+#oYxNDuRyF^_Q`6=d;E8>o4^u0ZvhX zy^kK(e7aqo&%3pv88`q-4>tLb@atmo~Yc9x&XpdW55mx0wu?Gd8C5)@a zy49fZ9C7&5soKdy_5FZ03BW%o$;iv%urn96b(mLt(j<|R;fXrcp>WJ9d?(g3N&t6f zYV$|gpVz9+X8-Bu&^ZGTDls#>w*H~3#9O#R#sX-168-Sql1oTs(jv4eRA9EsaV4Kc zvb#4a?a2^o2`*)4(K(p&O6YJ!-3EU_#(sZh6;4|n8Ckfy!^9+Z3^e6GwM%PheRq4$oZRCrS@wi(^~-+MFN(DY0a(3 z$tzZH>ORxcKxz8PAFyPL9Q5#K6vVzbKtg(>1Py<4tfZCTY_g@|m2+}(6HV#1;rVga zlOjyKh;qW^;tOJaJbw}FHOfvW8{xoMCH}Kf2_I#$f9q`mF<_#LC%C$$qdH${mELWr zF+q4jam0}B-sz|KLaTIS4ZfwedD{LYzWK8cyQt8+CrfO2fcd9zZ$Q@$P_EX}rbYO^ z?-8OceH`2gj#?6&L-@!@Tc!S-o<*sXkiZC$zTtk&#up7d8liKJ;f^w`D`ECupNSfX zew2FnuQ+uEWSZyt$#bsK(;OlV6Mb{fv^4UKg+^>9dHGNR=u@?M+6KEIwWAdcGHt+}I#00ii_8S6A zn9D!jZG8MHuF)1E{vtH2{{&zZ1@ML_h(b1b))zNm-Pqx~QX>qRZf%{#*~PcsLt9NN z*e5zO&chR(_1`qgdtc~XDq=!I(GF$KxhcmwSiA;|+-=;bRHwE4{xRq4=+XB!6Nv(( zlbdC4R2COWGASZd6qT4A_;D5?ZhrUX}d5fbK$=1#?!zQ3s9d849*5Q}pHP3RCG+^kJJAc?I4 zRR%%}ch3Ig#G%7|27p9uoP8IOmKxuB=zKzAYnJjA;4me4pfiddV=yLH2x>tU$h2vG z9zC7Ql}OI%Gsy)x%?*EoZP0kW{H}CnH(<)wmpk)l zr~F@pZl>OYf*qx9Pn~wL6DHh0u+(_-&|jc$rbg$WI#f@!o*MdfuU(I4wqz`h!&zP& zy9fQs2(LGmWIGf-Hs>W6Hc9=LcgRH{d(dQ@sm7I0<5mcQ*@KdGYB_%;@^4z_eAN`P^e#hJ!-6hp2`4lWKpn7redaFf z$w^UqaXjEMB|4!E1X%{$9X=A$e;teexl<=zME~F>InSV2Sb9%;dymd zu`|@lVl*L8>PKTcGdAA1m~$E}o-3wLyf07eYma1%$N9>Ez;9cJ6wlk&gdl3QfKG}< zT^k)yqD2|Q${(gzBFbMDT#0x$FUT=@&bmht4Lz1`x}q1U(|;5ll^{;wJBI!@%qBY* zrx#}w%vY*EZrwW@FnlFd>4 zK(Rz2v10(*ne97t)V(p_38Ph@zthKlN25<{KLGgGEnmSL9E?9<&h)cE+=6Mn)>x(s zFk1q+n+lz)`oq2W({VSWuplDaN_U`{jQdG;rGi#Jr{hd+)SNT1r3>q#!MR}f7<%>- z!WMe;?UbRLwEjW%%y)O|YEA=oQGm>$F&wjU)`Gz+O1{=Cu>6gbX+q)EzByzMp82Ix zVVD2@Y}sbcz2U^PjsJ$t)@Y~=Kcp05l_C>x91S5o*S`a1GorTvJ^=60jn;-8A9??* zX4Iy9!Y$sln{tKB4Z2*=G=5w}PjHy4)}9qdBNc-L(U_{@A9TnJ7s7&o*IK&Of)DfYP6A5b*5dSF&# zwA#F#9b$0v8dA`cp;;(Jwph+aAPCSz95MV(VI{_|gL2X zHBra+MlD%?mEz*P5uIQ z;d9M-T&=`jf6xq%CqPi;#|K^2>U|Llx7C?M zVf#O6Nau<3KtLMM=cC18{Fdse8)39R6n|CE9{SBZMe-yDquDQdLC`IRU+9N}kIe3F z!-j>RKA?dkz#fKOMJBO_Yg>Aps!PmCsOKBtN8Om>4b>jcaO(u-jgdj9*>CcVWub1r znl-mT=u!0swQ|B4>3emek7RZ}j!P^)pggDtaBwKt#(bUMc> zRXMTwZ!*-4)XSZ)sx&Dvb_PPKfC@23J)t>?{x@%jcVWrRhc?4DY&uUz&OqN@$H3)v zs?EK%IfgatMs#BL5d~obKnF*mGsJq{T+Ek9qTP9}0yRS=n^K?z#tbw5t17bEU9W$4 zetKf2htsrV{MP1{x$qri1Gdus-qm|VEiNiMX6Ne#<_V5KuwS5TU?ka|1#~?o$CeHa zQIr(^+6K&r0c?yaWDNxZ?bLA(^J~rBRch;QHF96T-lzNnFBN~j@=QjL>)^cMb%n}F zI~Ek19l#dF&N*Wlwodt9PV$rnLpK!c024X}c8PI|yLCl+)$ncKO7b>O4_jAt-Odsh zl%KFbshF|tWc@lzTGlv4Kni3M{u>OzV520Wn?A;V7*#LZk-Qv-0URO#+eS}DqN>F2 z?da8*eLYpSIQ}@n+45}A5%k1G?}e7)S6{!J+rV3?c1zs(uWnwVAWL@EN2)k$XJvmF z4~Wi_0rcpKQt6>o+I8-qEdtq9*^~rn9{wb%odblfc?3$Ujm4-TOun@rlLXvq`h1z_ zx#9iZ`a0&bRpzrUzXN1F1)}spxq*MpDr^gyXKJTK<8GN=x-K;Zr9+JjKI zs~O9gfCyVs8AOV;zpbJ?S^SLl=+#@xqhk)=PyA0xA5H1rH0o={E(*Mzyi%Y^7E>GIdIK=T9$&@-D)WG04p>0^C$zF|g-{_l-C{x74%ZhfasLtk$j^wDg zy0tWTrPYd+GIQN05k>$#O1b}zoqHuPkU8tx5SEWWc;Bb{P!X7Dy`BBB!T9T~a|2Snme6pM^@EEEqp=aV~s}@HZMu?(fg#`+ExklO=#c_XWjwU9^nn0=`<1b*t(JHBZ-Y z^q#9mJtIEi;A6SNpF`xIgg&*pmO8ckcfuF?H*-OTgnopkaZQ~k-TU-eg{e$KwFQ~-&_U)?r}1^8ZjRJ z=AaxT*!oExs=RCH91&;ET&8PM#O_ofxY1!l`)|&Q3=JIHJer?%JH5(Ie*J?(tG)^b zm;&TEdZ#t(*xwn!+Y^`qL$mS%@A6ye3D41m^;BdWH7$(boCPh*)~<@eCotH5!;j0j z%}u{M6aMz#_`&*P7ri%x1rOaeJMK|0{_#*p*5u zO>Ai6?T-{n!WY*S%l!&!6|*ybQR3AE{8~(z%JaPXgal?xRNnH>Ri zTJT?{|Ck|iN`owa#ySmG6G@Y){%dr}5v>sW%<%&(#hVG)^K31H;dP_v3;W)RIr(T| z!(Pr}o>ijEGU2-bEufAAuq^cwvYZ`@=@-t`_qT5ycqjHG(;8HkPL zjpoP}v_mU!vR`a^D2-})`X5h-0A2mDs%Ym0D6`XkgamR7Z zC9-~3;ITw@hdB$HaZr_-vGV2b8snjU7p}FH3Yizz)B&p{-Vr5&L>~<&KSDsdD1-P3 zwgdYKolZ=Hl?83O`YSOypG6ofPO)I~{tdJ{0=n3fAJ{DkUe3h8{V(45vHiQZ%ETa_ zp1ig5iBIGdJNKT>>%7rThWB9Z#E|zV95>B=GE6VnMNc;VM{_qY&mwhWHq0vSXkIRO z#kO~w53r&59LxsZ(~>d0u6=L(dDJq4AK51@EqoR+#S+lDIoVjLDay?I9tVI7jCAIwswG*MrH&g_4 z-e`Z15Wqnak#ctML$;5-)l!U#J;l+&h>2}@E(U14eF&ZoKEYDx+eKWnZGXh}%008% Na{YAY`X7sc{{tR9x848% diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/columns.txt b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/columns.txt deleted file mode 100644 index 0d87e8d0e8d..00000000000 --- a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/columns.txt +++ /dev/null @@ -1,183 +0,0 @@ -columns format version: 1 -181 columns: -`CounterID` UInt32 -`StartDate` Date -`Sign` Int8 -`IsNew` UInt8 -`VisitID` UInt64 -`UserID` UInt64 -`StartTime` DateTime -`Duration` UInt32 -`UTCStartTime` DateTime -`PageViews` Int32 -`Hits` Int32 -`IsBounce` UInt8 -`Referer` String -`StartURL` String -`RefererDomain` String -`StartURLDomain` String -`EndURL` String -`LinkURL` String -`IsDownload` UInt8 -`TraficSourceID` Int8 -`SearchEngineID` UInt16 -`SearchPhrase` String -`AdvEngineID` UInt8 -`PlaceID` Int32 -`RefererCategories` Array(UInt16) -`URLCategories` Array(UInt16) -`URLRegions` Array(UInt32) -`RefererRegions` Array(UInt32) -`IsYandex` UInt8 -`GoalReachesDepth` Int32 -`GoalReachesURL` Int32 -`GoalReachesAny` Int32 -`SocialSourceNetworkID` UInt8 -`SocialSourcePage` String -`MobilePhoneModel` String -`ClientEventTime` DateTime -`RegionID` UInt32 -`ClientIP` UInt32 -`ClientIP6` FixedString(16) -`RemoteIP` UInt32 -`RemoteIP6` FixedString(16) -`IPNetworkID` UInt32 -`SilverlightVersion3` UInt32 -`CodeVersion` UInt32 -`ResolutionWidth` UInt16 -`ResolutionHeight` UInt16 -`UserAgentMajor` UInt16 -`UserAgentMinor` UInt16 -`WindowClientWidth` UInt16 -`WindowClientHeight` UInt16 -`SilverlightVersion2` UInt8 -`SilverlightVersion4` UInt16 -`FlashVersion3` UInt16 -`FlashVersion4` UInt16 -`ClientTimeZone` Int16 -`OS` UInt8 -`UserAgent` UInt8 -`ResolutionDepth` UInt8 -`FlashMajor` UInt8 -`FlashMinor` UInt8 -`NetMajor` UInt8 -`NetMinor` UInt8 -`MobilePhone` UInt8 -`SilverlightVersion1` UInt8 -`Age` UInt8 -`Sex` UInt8 -`Income` UInt8 -`JavaEnable` UInt8 -`CookieEnable` UInt8 -`JavascriptEnable` UInt8 -`IsMobile` UInt8 -`BrowserLanguage` UInt16 -`BrowserCountry` UInt16 -`Interests` UInt16 -`Robotness` UInt8 -`GeneralInterests` Array(UInt16) -`Params` Array(String) -`Goals.ID` Array(UInt32) -`Goals.Serial` Array(UInt32) -`Goals.EventTime` Array(DateTime) -`Goals.Price` Array(Int64) -`Goals.OrderID` Array(String) -`Goals.CurrencyID` Array(UInt32) -`WatchIDs` Array(UInt64) -`ParamSumPrice` Int64 -`ParamCurrency` FixedString(3) -`ParamCurrencyID` UInt16 -`ClickLogID` UInt64 -`ClickEventID` Int32 -`ClickGoodEvent` Int32 -`ClickEventTime` DateTime -`ClickPriorityID` Int32 -`ClickPhraseID` Int32 -`ClickPageID` Int32 -`ClickPlaceID` Int32 -`ClickTypeID` Int32 -`ClickResourceID` Int32 -`ClickCost` UInt32 -`ClickClientIP` UInt32 -`ClickDomainID` UInt32 -`ClickURL` String -`ClickAttempt` UInt8 -`ClickOrderID` UInt32 -`ClickBannerID` UInt32 -`ClickMarketCategoryID` UInt32 -`ClickMarketPP` UInt32 -`ClickMarketCategoryName` String -`ClickMarketPPName` String -`ClickAWAPSCampaignName` String -`ClickPageName` String -`ClickTargetType` UInt16 -`ClickTargetPhraseID` UInt64 -`ClickContextType` UInt8 -`ClickSelectType` Int8 -`ClickOptions` String -`ClickGroupBannerID` Int32 -`OpenstatServiceName` String -`OpenstatCampaignID` String -`OpenstatAdID` String -`OpenstatSourceID` String -`UTMSource` String -`UTMMedium` String -`UTMCampaign` String -`UTMContent` String -`UTMTerm` String -`FromTag` String -`HasGCLID` UInt8 -`FirstVisit` DateTime -`PredLastVisit` Date -`LastVisit` Date -`TotalVisits` UInt32 -`TraficSource.ID` Array(Int8) -`TraficSource.SearchEngineID` Array(UInt16) -`TraficSource.AdvEngineID` Array(UInt8) -`TraficSource.PlaceID` Array(UInt16) -`TraficSource.SocialSourceNetworkID` Array(UInt8) -`TraficSource.Domain` Array(String) -`TraficSource.SearchPhrase` Array(String) -`TraficSource.SocialSourcePage` Array(String) -`Attendance` FixedString(16) -`CLID` UInt32 -`YCLID` UInt64 -`NormalizedRefererHash` UInt64 -`SearchPhraseHash` UInt64 -`RefererDomainHash` UInt64 -`NormalizedStartURLHash` UInt64 -`StartURLDomainHash` UInt64 -`NormalizedEndURLHash` UInt64 -`TopLevelDomain` UInt64 -`URLScheme` UInt64 -`OpenstatServiceNameHash` UInt64 -`OpenstatCampaignIDHash` UInt64 -`OpenstatAdIDHash` UInt64 -`OpenstatSourceIDHash` UInt64 -`UTMSourceHash` UInt64 -`UTMMediumHash` UInt64 -`UTMCampaignHash` UInt64 -`UTMContentHash` UInt64 -`UTMTermHash` UInt64 -`FromHash` UInt64 -`WebVisorEnabled` UInt8 -`WebVisorActivity` UInt32 -`ParsedParams.Key1` Array(String) -`ParsedParams.Key2` Array(String) -`ParsedParams.Key3` Array(String) -`ParsedParams.Key4` Array(String) -`ParsedParams.Key5` Array(String) -`ParsedParams.ValueDouble` Array(Float64) -`Market.Type` Array(UInt8) -`Market.GoalID` Array(UInt32) -`Market.OrderID` Array(String) -`Market.OrderPrice` Array(Int64) -`Market.PP` Array(UInt32) -`Market.DirectPlaceID` Array(UInt32) -`Market.DirectOrderID` Array(UInt32) -`Market.DirectBannerID` Array(UInt32) -`Market.GoodID` Array(String) -`Market.GoodName` Array(String) -`Market.GoodQuantity` Array(Int32) -`Market.GoodPrice` Array(Int64) -`IslandID` FixedString(16) diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/count.txt b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/count.txt deleted file mode 100644 index 7821086ced1..00000000000 --- a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/count.txt +++ /dev/null @@ -1 +0,0 @@ -1194218 \ No newline at end of file diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/default_compression_codec.txt b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/minmax_StartDate.idx b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/minmax_StartDate.idx deleted file mode 100644 index 0cb239d932a..00000000000 --- a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/minmax_StartDate.idx +++ /dev/null @@ -1 +0,0 @@ -?? \ No newline at end of file diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/partition.dat b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/partition.dat deleted file mode 100644 index 9c176950d3f897f5cc9714d7011107a21e47a229..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 Lcmdlj#LNHy11td1 diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/primary.idx b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_1_6_1/primary.idx deleted file mode 100644 index 4731cba6d463bb6755b51e8dc1e552175657502e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3240 zcmXApc|4SR8^#TXtXXG}B_i1)yOZ-+24R?CG`6uPOGWl!?5||YQXwKsgpjgK2qC-3 z64{AJmV}cm#ru1Xf4`sWzOL)Of6wQUp`xPVfI_j^Ob7Yz8obxM9TZ|mW;$%TuYSIHA}GJR&WA=e{u6-x?-dP}2dM7}E>c4a3290!)z5`(S0Gda0ixqV+<0 zFhM~DtxX}yI=4DnrNeH)ga;zIK43b49!HIsrDJ3=;jBBkwo~{!AI~3Y$1;NY2!t z&(kvwQ*BS$lA5k&hqISb10P~t;a9Sy@BK%@rI>?%sew-hk-w?w^GFQseW|cWM;buB z_hzXw#gGaG8~5!Gb6}Eku-xNuQA92Fm8V7A4h?YLp2qbI9N(c?{3ph_bN?@}DL6sn*VmH|5JYWJ|Cs%|w>5C%1|E9d%3He{+>1f&k-qbw_qSFaM z^@OaAEslNNV4^aC2^aLhtBw%d6Wy09izlBDm6vku%OeZxQBi{=m>{9%M|KKh9oiOd z#pm2l2s3@^v-urvj(2qYelS5nX+LU^-U^ks`0=mIPKW`aCe{LjMWH8e2A+`8=JsFV z)yrM3lXI3Z;fDTM?+rbzF&!@cR_f{rS$7FL+!%^c_Vp_Wh6xW8Z%q|(v!5qeIW5EW zgb0f(9Zjrv_{$c&CBCs|0$W^OJdm$lWny8Apf~HDWCqu`@F)0ECO3x9$o3yJOpwrP z6W!vvdV|KZ?L#D(@Ibi=T6x~Qp4&_d!8XieEMRJAy1ybIXe7DCTH-ectPnLBnw zzPF%s`v2Ns0a;J^>~l!3MRlWZ_}R)H!&79;<=0W@B+W3K%Ur@?zkRQC|)dhX>4 z??&02kSv4l_NeA9+xwGujbOqFnXymFX1VK1R|keUoe=gE>nw|AUy&wne>_aMAl=z3 zh%dN`CyUv}rYA&nNzyCo;r`Mk5hJS;@_udN)0tC?rAY`UO_(5|FTs*%IeJwm&y-*G zCuHT=q*{q;PpMMff(#QB6fA@kdQ9ntWb(}Zn?gZ4#3_gZB z-+`}jErt1d?;8c_9bYy>-6eQ0MssOvhtKDSuSgAEq4cr>&C6mX#OL$-u1gmldt7_L z4zgCJjB)a@wfkDS7cF-5IKb3xmz7y}2CREjT@wZ-JkaO2^hGJudigujN``T?C@|%( zSg(jo&aSjCkqB+0;{h7`IVoY>B7d}E^h&6f3_r+^sp+x=ai-aFz({LO<+Td+GWqn0tpzGXgS zf?~~0xDGV>XJ+5bRho>2hQe?TaO5WwHw~+PjB$iMT5>CRydq&@xSjD~U%C85;Oz0HG{N8UYbG1{nxPUh z6=?SP7Z4LRrud+&4Y#SiG|+jW%=Och9~oonkR2w8N(UO#&^({RY*~h6QiT6i*DUZ} zNOXwHs&WptgMQP_At)OVPxQd@G-&0+6QWLaN8!0k17KJgmA-zXH4e&cMHLH9tqXmcHeR(xL#;EEn!gNpnj? zl9mi}0~-Kw-tSQf_ggg6++FfE5o!Wi%c)i8gxhbsvr+V}XlA_wq&)S~az;yToyxZ7 z?cVd9;9~Wsgmc!=1{p%3xp+m6KCqV;eyXZ_RV|#(n9VM(Y5=_8eqG&O%SIQ4#$Vh? zh;0~TDUu0U>@Rz=$X7^S{hc2{XS`q6ajV8|8w14(Dzc-FcFnJCl{a&Kl++4tQ8hQj5#JZY5 z(bpF((bgKkTSqYO`f_C%ln2L_X&RglsKUsM7vSeP`ZlQU8E zt#8)#+pOY?lmL+mUz^gd__>A0?PCQy)Iev$mQntIyvx;Q#=Zf&NG;%8Ozov>*;h0^ z^XP$rFXt7|kPzprZXQ7*&+Cp^^Vwm6ro@Tw>%9^tvL8O#!{iT4y4#8vrUaYn94;Jc ztH$EN?hzqpnSRK~?74Lf>Bx;*Avhs=_UTEalho_@k|KYzEj!S8WyA>`qlR2)pe?v5 zsNw*89Ql5isS!ELgIU|Hq&HkaL*{kcnvWcY73SqpL9yYUz&U|)T%;l|B5QIS5!P^I@$UsAUP3L>a-xB?^#1uU)5DBtesz*y@A8ejkcImwOJ2DoK zSaCzgI?L3=>V_)a^MOfV$^}K)B05@cUrSNNw-GXc^S?&*e@ymNvQ%+DEQ0Pm0aJR~ zXMPd*coR6X#jmzo#!EQBcWH zZ(8$`C=>MaX$5$T!g-9lnDqD?;cnt>lf_rSc};WXB2xMoQ!)^UN4vfT7t1TPOivDt zk~21T44FS{1e(O0#2u}MWZ#GSyFmuaZ6G~~Z^9UDy}r0{Dj@W?O9vnn#=Y?|TSm1< zGnd_0^}0d2BC|cMmmMj(UprDqL2Cf`AjB}M;*2Xx?$S9%Aw45NgJ*bzx>}%qIh_lM zA#_gw&5ProqD@_m{k)ebK}T$#L7xjaAL?U|2$i4jZ)1(tR=|5v+~VH4B9j-ex{Nvp zKHtG!qMJK~60EZF`|B?LqaE8o!$2vf>XS@Aqa2c@EiJtd2yco;qFOdDZd@}SV?%ok j(w$po#558Imm2ua8h8CU23d;MR=5E3ApT7w51;=7A{bPx diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/checksums.txt b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/checksums.txt deleted file mode 100644 index ab58ad06ceeb0d11e4723395c0f1b415399d055b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14273 zcmX|IcRbbq_kX?L?|bdNHz6e>Nk;Z6$|l-%ac!5b9hH%pB1(2e6j_m3vLcm~vOXj- zl9cYXvi)At_xFzn*L|ORUgMnS^E}VF4lYCocOO4bA2BB{vM0e;%%4d1arGh{6jNfr z$A+4pV9tNsH|Rtd_#W6Cep>|qR$~B`WUgYYx{m&aBxhF=(bz!V-j%eHfLa;ptFG`r zRWu$*OH3UcN=6di33DebTc)d3(ma_{vRqbR_{F9IC`_JYcg22YR&_RRnv+UjldwiP z&%DM_K&$Iav<4L1LN2?fgqp|6+X;^y*OB_{=7x3nGTNeFI2E<{+VRtF7G@p%ZaccY z06hw*OnaG1Cz#je+L+nPzdy9#V?s4(z8K*^?f0lT{UuL z{9~L40ZIV#sK=G5x8Pv!e*7NZ%b7{H*}yJfZ$cnB`w=)13T~~OVPr|A__YqUKbr3Y z5zlh$ls%Y34I1rvFO3GjcOOaX&};ntn#&&G)dA`NX7N97qeGlH@vs+OWhT-HTR))F zH+g|j`CmjJZF5seRusW9+LzZgS+v=uk>^kS&r;gouP-IJdtbd+!c6#GNUs1;Q$XdD zYqm$V@1W#{*t24=r;2DTeuo!lqzSWjU&|R>+ zFN#~6>zVayWD_-NsGQci`e}&p=Hk=Sj50R_bpa0b2D8}5?s99|hWqSCSAL2Ph*`7bvt>-`}TsM*A zYi#ixUcc3w*bVESPc-Q0R@8Bi)l~T>Kb!-zNEgTHsN`wOPxf>(sq4(|wMb+M(} ziVnq8(Nnz7a5(!R^87yQRc~$$fD*wRM(VIkx@%-Tk?%fgZ|wg5XjsR*+`0Rc?Cw%vjI$SM71CPsjx5 zU6#{9os=SV$Kj;5{C$r;hY-NFIVzS}m(s^k;lWWF{5+bk+W!vVi2=tBlfC@BU!oKm znDFBCOoWZhnG-{H{Hk&t-Fy7^q9fQut(wz>O}mqm7^!dVdH1HZ^J)Tsy1>;AZL*#B z=f7(CX=MoCmzJFh=ukxyFXu6YDv%q6zg9}+V16T}_J-p(b8?E$7_#*JCo(oSNq20e ztKne+J?<4YNyBOnP#6*?0Ts)ga(9f|DNO&#*>i->F251*OMp>R0@faEc1(01PDo=5~Nq;221NbpahnW%%+jv83SFetX)0aLt|P z0PbJD%Have-sg8J8b@D#bG_;-L?16dax-8(j&Z`j;s4+rWH@F`^dLGU)6VjJC_;EC7O#cHOVZ;m zn3stAT`<^6Yb-J}Uxw~pA1$&@sy7SfWp)6x+rWhuQU8#=4)z{;TmMpAH1ZV<2*nd^ z2xMoXZy7?Rg;7=X5P6nrsgRrZ8z1tp`(b46-{EpTz9C-rZ4F=R=cithjG(drXHPfFTS&gleCA8?J>;EWu-Ho`p`9jM z7#)E3dP#I(B5#g69XG9gQQEn+EV#HiB&i~zLHjfH65W$R+X})g)g1$;0os74V$;jW ziO3R{;-|l!-U8?42Sepn zdIN)(XXwCwfZ6~m(z%?R%$$4q>n$-S%@w9f5q4xBgsN0p+&V1A74In+K9xk9?2}KMg2|SB6r6iHhmWuIQ*qHV%5r8h|PR^ceK& z(&K|vMuzL>4$64ES86xVCMoAyQ&VAJ>Va(5*fk_h%5b&!~vI}Wmi zJe_K-o2F&CK90F{6=2Kgl@eA~J5fr+jjlK*u7kbJ+hm4{&C~IW?Ugo6nAUAvuS)As z0L=SXa9MDje>z!Al7nx@vU@d!e7UKrt1)%45&}CO*FJ`P7(JL5XVFnnM0Vc>b_(s~ zq(9t`0-7A3 zCB7#r`gi8?pDN)Vht@m@J}#yNH!m`V{NXQ%X4&JY zlsn9=Nmo>BZ}l?gX604}dL|oM>9aVnBUp;Bs(Qvnf7{$VGOz1qdqlR6PI_T6&105^ z<^<CBe_ZP31G(zy|JP{`hoxkzlLEyBtSPz?2}#_2WU=;oBW%7gc*+pRW==0g%6r zQL*P59+o@x!qrRcI(hpGnu-B?2*8?Aw*8gR+GHDxn3h>jU*N!~bs#p1TA%jVfvupo zOv-%B4=Pgg66?RO-l|^U_?zHYkb<@(b5~zqgMM3|ugg`o+UEe9(m80RqmAbHhQI#d z>9XRueQw_#--y;a2bPg?Q3}3HC#X)&)#lM9KOeDkfOG5ZjCU4_g;G9QgwMGP5UR*( zZvoRGvX`e3+%8BNa3gQfb}hMcvPm?RN7k~eK|deGfo1`{9+I6cVN9z#qIZZf=L}mQ zGbE0-fPyhl7COIeFfj=%9U2;_P89)+JAi@>!5IcWUv3_9j_4RM2g;7!Mt?icyw)v3 zUbl(fFR#r9zPuKRoK~vzT%lgEU@M2g(v30EoqRv6J+iC8c;T9vjc1t-eN5FxtX{?W^U3(6-o3|WfMP$cwi1zN8Dl~QQ_7C zs54l{E^|klC%LWWI)xKE6IuCZfPgfx@NvCJRCo=STUMx)ejuz~_mTkThx*FNGaZ)d zBCuk~YMb-s#4QDWKFINL)JkwtP#;z-WI$!J3N2l@siAt9rGAKRPrn-r;5h6>@US8h z99)P#x}LwF~GP5WRh;VT8^OVqug&)lm=b@wLj7&J4J%U2_2I z=#SaqOPg79ankj|t#)rFFK!nE%Scg=GuiTWw5}$v1$TUcIH?U_9*}c)=g~ck9?`q> z@^iI`(|mPrg9@DkSVEd^Hv9~Fe#3r0>&DFQGV)9t0Nc3{YyNTP^rqC|oQ|Fei=^^T zjesW(RQT*vG}QMalZhk;U4~7*CK>BJ!jBYMrRa1t6PNV$lY5cI_l9f^eY@1Xj(*2a z>7Op{vJU)5Y^i8}aQhqEV>&ZOtR99u6MG9;H2^Mv9tR27V)K5r&TgmOx#GR-e#80l zM_zHdfCc1m)JwauA$l*);a9%?N$kd-S%V^-WB9s8^rf!G!#Bw{6Ro$yoQn25!Dy#` zFx7rKHkJ4zNn9D2BcKt`YA{oI=Y`K?h@(tNFDYer@wbhie4|0G_OnYjT2MP!P{Tu7ENY$RU5p}=jYZtWqreeet zHIJY`3Y@nll3fWN7cmO%>2)^4)%>QxWE+K=AU&@c5L_yPEFcBD%~DoumfJ#>qQ%~i z&3LW??C+7zqCLk?3);JdC`g6U`1Ez8<^V&4rXQo?B4U~Lmv72T3Mj~&pR(c0q~+A_ z$Cgl$$U#raiI;O6uKKrY6BE?fAdfx`=4mfyPq!_;2qgpWZB9a%9jq_@16Nx86EmDu01udkNGcMnZ@fqXJ1({Hx(Z; za+^Dl21k1xmRTg(hc*^Z2AT3;L-n@|i z>ah)2LcPyaXaYO44++7`4gK{-tp6aKHer>=BumZ9F*{#BA&fu70x%ep9J~}bAyTfh z^mPqs?kxT`n_6^M1JQonxeB!x+zyUe&HU~%`;eRP#OE+GsMXMDoEYp zUrkOoIZ$iPksYjis^(e>$=W9xFd6&kdHI>}K}$7E=5rs(E4-Xcua4evW8YXQEQG*) zk|Z45|NbFkk1xUUzNP2bKj3EW99ZorXYsmJ$a#$S!H+%@z+&uU;1xjf@FF;t!)lHJu|LjVDo~f6;YZD}h-(u;tT5PlPzL~xdK#$_9Dtd|Pb$k*Kr`v7VVDX2P^yRT$`ho%+(nLHzr#xoDV`4a?^BQekl zg49EayHfG;ha5t+g)~1mzxQ`lwS-O$3bI~d{aDtgOtN8%=+Cp7+-pA@^sT9) z`e8m7SN}&X{Wm{eHUFjgcB0MUUG3WfSHNiEN^-wQJ1-3R;PYhT)VCWF-cDBP-WM6r zwNHVH$Z$snLgzKbea^bvz3-3sVdj^o028ExHc$z?GvH}SKM*CU6rykm?C%6XP?swe z0*-+zndsnqm4A~**Jb?F4{7P?H+HeitLHU>QjmG>=BnkW-G`KAZ2IOsViY+Rg>(QO z3X7}Qs`~Po47qUY6Zc-8cQ7bV0RjqucAaL8TYL9$u8`=Lx3yeQ8zcRD2*=I&8R%`10%x$syMfU>NNbW8&GoeWz!w%Zqk<| z_roL6{8~~z>0D}nl>pc{Ayr-CiJ)svJtKYZmxEaxk8+XRK2rvyRDiK|l#8A2l;WEw z|M4dGTE8Ix1mXk%{K2YsvUiVy(yE7}28V@jDyRbPSRm=;c#?O6w_bR3b~nu`ioTtZ zebCw)rW%1fbUEzC6U65nbNcj{&<&}P26-NMCx~DLZL~=VWced@pry=6BzeDWIM@Rv zWV&SmxcVu&xUOe0GLN;Q@5vThC0|Yf$ak4tS?}Me%qzTOYYABIeZKgE*9o9d_Hl;E z@4u8I-_pBINPQ`|6+NIxP5}He;DV(ef#mDzTgAA@la8LerZg8;E$Z<&GoQ*o5)%aX zWUh|s)K15Bb<&BUBef;pb`_woLcwbT)DtlGp)>>pmxM0zLvO}~QrCYdGh8|=uh8`E z_FcJAO{ChJqkN^s-XKuU#55}Wu(rS+9|Q+s&jHp&4di}N*(WsjOqZk@@X15GU=<+J z09^wMTT7e6oA7+4cD?;3RVvghd|UEt5?mU3%)~t72UTE?#1ouhV~9E*GeQfAhbv%K zL%3OG0ZQegAo2tI6kDJ7`5kYNs216MTSgvMhQ2i)%ZfH*jGb;UNLNbEGqSyy1!T^mN0~J>gg{(TKC4T2o3Hof>>_&eh;;0m+kStz`Sx4}f)mR>H#NB!sqSR65VM zDsi$R>2%`~2Tt4yHfi5SvxFuJXUo$EFedlf7*FkdT^PHKk$Mph@Iw|;Epc^8YW^B)hY!`mQZOB!+yX9a#HY!WJl?+hI$ z(`;$?K zY!A`M_07eVRqui=Rif?((;u~o`OovW2539ROBNKl}{IqirwUaXFp?Q+dO|4EE%r$1@{i7F44cP zH2-bfzmt_83a6+S%;`@~4aa(a^SV8PS4hRzq}>N{z}JfCL?jc*-hhg)DT_riFSNw)ug8H*DVk^a8Ya$V+umpQqWHO}ERB zcG(aE^1#m8mq7L%h4%i7NJj!p!|gd(;?CZvA`yzrSpvz>-hT13^rmm=ShAHK*nDCb zJ|##7^acR+0Tg`ZP317*$DB{syOx$z&i2v)8gpw;NHSl%;+Qv#G&~eE;H4drsFTmY z=#zuY;U1+jIp(rFTOv(N4wEaxb!pA5BRJ-m!gdCIx~!a+cOh3TO|w$e8L*jq6G=Y4 z1Yg|{nq~f%caT4Y<)51)kKYyamh`&Bz|4pIekO9*{Mj9*Dsc<5*XyK$7riv>kZ9(E zCG2CE{qy+Y_Jc%qyBXs8zy-jy6W}5dDsH|2lU&Rz_baOP*a{>LzQ~t>xi!APe|HZtw5vKzuh6x^w=@zj^d;j zlDa9vUKS;qvdL33ub|yT)Ojz&=LzUwjq3&jqhQp8tJ&*=yp}IK) zU8@*FCizkmXEt^&w7iXRm}R_lVhFm1DXzwbo2;%J#MWJd(^RkgmHOdz@CA#=?fgqo z!Vhii_xn6@-&LVpCW`V}?-yF&o%AmE8o=AW?#~NM$2QfLl3kIkLeZ16*REa-tsJ=J z@?YrjYs$&O=vMpD5#BhZV~$N0YV^DctmEn9cfzXy4#6A&aufWOaMLys@61GsOwGFe zfdi?nxm7~8J#Z3~^ZMAz1p7_ftTkw3guT#mP-TIP`x>lr`)HNf6Ay4BNUQ4jun)7s(C=*3jCFwIN1 zx5>*qU4d-<-FmCGwVTy8Yc;Qr|ALJT0-4|$!U+Lvh3~U6@ibG+5%pAF_M+R}XbWpH zQapKFgX13i;A1llIo0`7+CB|#9Pp~1h3`*QQZ9_8#BvVs9kO!ZN%{>qr2wjlQLz4< zj$FQzQ&Qh}d3Mzh9^18x4KuBwPIr$Nn#> z=rIMRcxZ`jKl}$DB@E$tAf9QhH+lETK&)NqOE*eee+AJ<~>@NLj`Ud3w6gE#D z+x7c-ju369&<}wa6eP~-)tRJB+?$)Ml-X={gJur6K{IQ#=~@LFJN$!r zCWSsfCjMy&Mc#eGj=#y&;9qEd_&DpU?UibGHE4$OfHty6o)S^IrG-=(or+xd?dn18hIz--y2c4C7!iN&VK0-U*~o z`Ua9!wPw*}q)t;aZrx^nkz*x&vL@=qQ?9kJSm>=SG~_wu}0m2dGg+zP5u(#R=} z&ksZ_F58!!po$6 zEoLk@YPa1F6Z;gyaeWO^ew7QSCXj)bF<9`^xmvp0G4uWGb9=w} zo}S==Dc%KQPe7fK<;7Lai?ywbvPs5{89;Ckp!tBKVuK3hR*?+R0T!t2KIcn$8%p6{ zAR7^gsVFZa7-U{V=s1izeeVbvz{2ixJG*m1@*|l=D`6HJu;La32%kekQEFfp7g~jH= zIbUq!uNBXYmh);LF#X^pJW~8JHvELtpPYje z*U}KpF&hnp_V4=UQK!DFFDLi=*dcZ*6nv12Ft`DftccEpU32{5VFF({6U48l@^|8^ zN7$Z_tppIt$1CaDoL7{c9&bZUO^r=ES7+XLo%yNbi z9p)7}?!gCo2)3L5gbt)wuV#~zcURg|YQXkCJ zgPG#PaibdEYOTj05?&C%>N`1^LKzpNj7i7xdbVvXph3DJzcY8-z135@>-ifN85ytx zASM!2{JrlHh?YkU&vce1d(T9E<1lUKxe%UtRB>nE!K*E#_f<63pLqhoXY3% z3(W0{pp)XG^y%=ub*2s3U!H(-tC8#Bx7DzZ5nkiJnZv^Q+v}nP@wez>PnJIuIhCMTczw^Tal4>~J0_>s?3CAsINE`tovWjT2;8qV(TvdI1H5_Mc-fbk>ZwP%?m$@sF^=x?Xlq<} zQ7grBbXa2X!99RN&l~`iQN-rCbp{iz#gzp%g+$H@0A?#MdoN!S(I*YMmQl~>zH+zA ze0sSebjIuyNpKhQPGk`sE;y=JD3ffgUc_D;=1kDsW)4gAMhn7xo`L~fA%E&dauO_0 z0S;>-f$ZQis|Y7$zOw0S3puqn+>m>t_E~@+eZHa=tfFT`JRAkj>YuzE*~c8~dX8mV z`k%!V_9*s>v&uf^JDy1&5tg3~*meO+i>(2G!<@^44_aO$?;qLTbftRi@dz-}+8vEB zAe(8i*ie_=Re=u*7i)(5X}=2=|8`_$#DR)WC> zxOU0m{SK~x*7^-U)b_5je!jfxzhE{WhP6F48<#QyrUMWP+Hm^$V<&xEZbP<**h5U( zaeLm5B0L@a?k2li^X0c((t2_{zSDrt8qSGrWts6^4g$Y(-nKJYHI25FdD13*hVyx- zR>z)=w9U_2otMW|b?9>J>EYa-37{}EB=#Q{i2m`EzI&F!bW_q2@LIci!0|(mJ`~$0 zg*=6-<3b+uLCb?}Z_3#C&Iv>t`+Q+fKMkXrFRP2exJ%G~pU8Kc!kDkQ|4$pb%3r1L z2|G2Zw|mif9yg*{&%D3_sdgY9@`D5)*^H)3J z`}GUj`(!UlH>dw+zEauq`0y5Kc*?Megzg-z%xY9w3KYVgW_F!a{pSxBDSlwD^yo}R zHNOClll{-!qE*)xViNJyv&W`my=JbcJ_c zm;k@Emjmn<2(Z$7!?1bPhMI>M4l8Ta20T`|bOq@XP}hS<@yfzLxVBx){8d$uK-8rT z83A252~-N9{-#sT&HED5yBzP~=3=1cri9vP(L2cxXoUb~Zq9uM1mAvG zWHw|ld|;LhxmB6q{eg7$zCHPC#sL_9Y1Qk$cSrDaE)v*FV|Ec1L#PC<40Kp&o z>!rieEZZ7J>;TRcM&6xye-$gkgZ*Z|eRcROs`@2`lougI&P?FJ7sfo;#VDhCeZ%5+ zyB$FOPP|dEc%WEQwBOsgOYCXuYl}e(z!U_qUcp&`rN#=HFKu}x@kdy1-07;4#MjKi zq0NdE*4y27H!Q=ev+ev-6fMgLQsV%?Ca)djF`@RIp9klfE%JAlPd)f7;tV+K0NO|g zT#!|PwNe${f!vFaMqb=IX+*i_Atk>9N%{5`3+GCq8DSiZ_`Wab2$vE)4JXW1_Fx60 zZ*EVyZ|1)9f&V+#bfx~6fM*xL$+E*v(8@uaVB}>PpOtGBY={9IHeTK)M1P`32P_Nw+8E^{ zSLG9u);kU<%8kf<_4wrkGgYUULRE8=t!bie6_+OOZq;qCsC2UxF==WEE=%)~&UbFt zM0o&K8!umihm++v2S)4|mZCC|N3D$VrsWG<_+1Rk9;g9}HOM@WbMN1N{48 zw-DJBSi7sR?mo5Ym?oUZGp9pZvMRShW~X1E0>k7#3~uW_*=vdfi^E@11@z#p-%^ef zZ?YyG#x_MAR7zx7N_%Pnw>&^~VUoG_Lp1!mZF7{<(B2w}5mQ#vK>rc2f$rQl`pn!7 zD~`Px6hv3=vc{tc*66k;mz96!w>xr;k_`vn9DiSXx(VC{N^l}ePhnNyQ4iYmS&`)Y zK=NU|#%pV7X2D=&aV-ylvnDEy{AV|m(QL4;eaUSQf?m?B2=yzwy$~ z_gC+!1v~ZvmJC=OmJEr~dG<57e(mTNtLZyEe>MGi5X^psjyp?4ySVQ3JG<9^Ff*o0FNP6z%6T{+GEwjYSO)Tj= zeaOT5QaoqC>?}dI_vy#-}3At6SzupJ2b-;Wqj3B-pJ4 zSQThs!)q}Kucq_j)vxmC4?Y1LtO%{ zyMm|hk`$Q-yix!!#!bb2=;M&n z`q}XTRc4MD(&4SnN)wQU8-6El!o9F}kLewnMib)hb9ec4VEya?W@(1Cd#>*+aLN8? z`9(f!eRB}7K##7nAdUDx0=Yj}<{!yt53@b;_N#nV(Xy*YADMp|i3AFz-DvtM-j>UH z=AfRQiU%vys!zgBL?Z`vtVw%WH0zzi?t)9HK3gp+d<#r9na?hSlB+$=@R)W)iAU{$BV5zvexq-`(O~0aD1$DW?-q8Tw=m$175H zQE{uO=sgDKV`yG>Y@A2YRBqC@BXFbzG5SoebP!1%kUt_7$+nU5U=d(2H6I65nwO1- zhzlc52E9bi>Ce-3vw-cGjlL@_5<&O3I9#0bk7v)co=RCfU2{(ToV1=F?*Pp*`naIZ z|A6Owzgw~GKB7w;vL#BM0rNCE#4jm2ye>^2DxdePGJF{TtjBCjgSCJ%?Gk@vD^1oL z|A4O&0hM+I^uB=i3Wj7g@6pTUrZ_o{2s-`*gVQcjp8wO)vwV6EocT7KKa+VekwN`# z2jGM=q$goG*W9LS4({1&tl=20)y4eoh(>=H53+{Nb;K26&R=+%j(k*$9(8)k1!=M) z6vf<$uaqLc$`{^4)~@)g^i=^CNV~U&%1M`4seDxww3l2HIXocs%OO_H-LTil8JQUs z%ZPXSdE^ep+FH=XEqYhFW#R9BodAniwYJ?pmWUu(Z_cO<+T}Bifa3(fJ%^Q0<`dk+ zYhKUP>$X~g4-`EMr^)^tR3y7xLM@~6)b|fqt#ar!C5xpkwFmhNWfZ2LI+2S}SBV2x zB(wf_{2CB}Cuw79N_2Gf+xo5mUqwHz==4c`ZL5iY5@;9SCw@xE&%M?D zL&>(=uW-p1p&+FV?(Y~rVf#KB6OQg5GIXBXnqokU2y{Ps^Gv^u8lkbtmw49SlCK}t)2nDUMIjZE;=UStdZK@qCJyQV= zp}-p8p1`M|^nXV3$-l>N_oe^b&zO4x-7b8=#0SQBpQA$}EwgFC^lsksJS7)P6D2s? zu#!K5Ha>T5u|0UjMmB}z#5Rjtz*=H6vPbuch0v{8oD#{A=>Z(#z~b+;UNfW!%}<|jQiymmkMH1jH&MDh%qWbOIS^To7mzXbQI?Yz2#7~2HW)cE!>IK+~} z&CI&zV0Tnp{BL-dyk|`x{3`Wijm}_j@ZmMrcQ^sqG(`g53B7f%pHqFNMtOTOgwY#5 z@K-pXV41RwDvCwzQn7zT@TWvw&Ta{(s5KuIU!7Ql(6Wl$KHR<@*deFVdF*g|$^~Q& zJxonM_;Z3F{yMb4mObga_P_B)Ea_Lc{rJb($t2VP-aMD_ki`mVyxB|df5Jnh)2B<=M&%%% zCvB=0^IxXAfW2Je4@*u*_ElO{lqT;#PzE@50DL*`)^kz=wlusOnIN3@5tn_gsH z2VYlz(k|9_I-J7W0|(4+b+|ZT&-H~2n_~>hN-^NXXa7}d;QaVrZ8DFYE6LF-fJY4@ zaQ{8D8=v;q{-j}=?tJ#4YGk-9Utpj<9^FKj^}^dJc(KNDvqQFhpU?CC>&h%)5((gw zOBctC+Ruph88YwE0XKwar)^uMGqPqMC4WYhkG{|s$&Svx5I5xlMR$hi%%UebPwfzsYx=++e73Tv<|vT?P5@Z|s#LS~Tj_u|_rzcCi{jY3ds7wA zJ8XR|Ox`!pm*%iyd_N!$e;14XTr~Zxa}#iRx{^Ezf#-bbkO+1PldEGOPCrjAKd^+- zHhJls6x{<;Wx(tW`+KhEc_~uzjs#!ano6s_^3@na-WE12=EalUfYF=m>Pa96$-}2} X7;-Zk*B`ohbhoWo-401f;RXK(?Rkf` diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/columns.txt b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/columns.txt deleted file mode 100644 index 0d87e8d0e8d..00000000000 --- a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/columns.txt +++ /dev/null @@ -1,183 +0,0 @@ -columns format version: 1 -181 columns: -`CounterID` UInt32 -`StartDate` Date -`Sign` Int8 -`IsNew` UInt8 -`VisitID` UInt64 -`UserID` UInt64 -`StartTime` DateTime -`Duration` UInt32 -`UTCStartTime` DateTime -`PageViews` Int32 -`Hits` Int32 -`IsBounce` UInt8 -`Referer` String -`StartURL` String -`RefererDomain` String -`StartURLDomain` String -`EndURL` String -`LinkURL` String -`IsDownload` UInt8 -`TraficSourceID` Int8 -`SearchEngineID` UInt16 -`SearchPhrase` String -`AdvEngineID` UInt8 -`PlaceID` Int32 -`RefererCategories` Array(UInt16) -`URLCategories` Array(UInt16) -`URLRegions` Array(UInt32) -`RefererRegions` Array(UInt32) -`IsYandex` UInt8 -`GoalReachesDepth` Int32 -`GoalReachesURL` Int32 -`GoalReachesAny` Int32 -`SocialSourceNetworkID` UInt8 -`SocialSourcePage` String -`MobilePhoneModel` String -`ClientEventTime` DateTime -`RegionID` UInt32 -`ClientIP` UInt32 -`ClientIP6` FixedString(16) -`RemoteIP` UInt32 -`RemoteIP6` FixedString(16) -`IPNetworkID` UInt32 -`SilverlightVersion3` UInt32 -`CodeVersion` UInt32 -`ResolutionWidth` UInt16 -`ResolutionHeight` UInt16 -`UserAgentMajor` UInt16 -`UserAgentMinor` UInt16 -`WindowClientWidth` UInt16 -`WindowClientHeight` UInt16 -`SilverlightVersion2` UInt8 -`SilverlightVersion4` UInt16 -`FlashVersion3` UInt16 -`FlashVersion4` UInt16 -`ClientTimeZone` Int16 -`OS` UInt8 -`UserAgent` UInt8 -`ResolutionDepth` UInt8 -`FlashMajor` UInt8 -`FlashMinor` UInt8 -`NetMajor` UInt8 -`NetMinor` UInt8 -`MobilePhone` UInt8 -`SilverlightVersion1` UInt8 -`Age` UInt8 -`Sex` UInt8 -`Income` UInt8 -`JavaEnable` UInt8 -`CookieEnable` UInt8 -`JavascriptEnable` UInt8 -`IsMobile` UInt8 -`BrowserLanguage` UInt16 -`BrowserCountry` UInt16 -`Interests` UInt16 -`Robotness` UInt8 -`GeneralInterests` Array(UInt16) -`Params` Array(String) -`Goals.ID` Array(UInt32) -`Goals.Serial` Array(UInt32) -`Goals.EventTime` Array(DateTime) -`Goals.Price` Array(Int64) -`Goals.OrderID` Array(String) -`Goals.CurrencyID` Array(UInt32) -`WatchIDs` Array(UInt64) -`ParamSumPrice` Int64 -`ParamCurrency` FixedString(3) -`ParamCurrencyID` UInt16 -`ClickLogID` UInt64 -`ClickEventID` Int32 -`ClickGoodEvent` Int32 -`ClickEventTime` DateTime -`ClickPriorityID` Int32 -`ClickPhraseID` Int32 -`ClickPageID` Int32 -`ClickPlaceID` Int32 -`ClickTypeID` Int32 -`ClickResourceID` Int32 -`ClickCost` UInt32 -`ClickClientIP` UInt32 -`ClickDomainID` UInt32 -`ClickURL` String -`ClickAttempt` UInt8 -`ClickOrderID` UInt32 -`ClickBannerID` UInt32 -`ClickMarketCategoryID` UInt32 -`ClickMarketPP` UInt32 -`ClickMarketCategoryName` String -`ClickMarketPPName` String -`ClickAWAPSCampaignName` String -`ClickPageName` String -`ClickTargetType` UInt16 -`ClickTargetPhraseID` UInt64 -`ClickContextType` UInt8 -`ClickSelectType` Int8 -`ClickOptions` String -`ClickGroupBannerID` Int32 -`OpenstatServiceName` String -`OpenstatCampaignID` String -`OpenstatAdID` String -`OpenstatSourceID` String -`UTMSource` String -`UTMMedium` String -`UTMCampaign` String -`UTMContent` String -`UTMTerm` String -`FromTag` String -`HasGCLID` UInt8 -`FirstVisit` DateTime -`PredLastVisit` Date -`LastVisit` Date -`TotalVisits` UInt32 -`TraficSource.ID` Array(Int8) -`TraficSource.SearchEngineID` Array(UInt16) -`TraficSource.AdvEngineID` Array(UInt8) -`TraficSource.PlaceID` Array(UInt16) -`TraficSource.SocialSourceNetworkID` Array(UInt8) -`TraficSource.Domain` Array(String) -`TraficSource.SearchPhrase` Array(String) -`TraficSource.SocialSourcePage` Array(String) -`Attendance` FixedString(16) -`CLID` UInt32 -`YCLID` UInt64 -`NormalizedRefererHash` UInt64 -`SearchPhraseHash` UInt64 -`RefererDomainHash` UInt64 -`NormalizedStartURLHash` UInt64 -`StartURLDomainHash` UInt64 -`NormalizedEndURLHash` UInt64 -`TopLevelDomain` UInt64 -`URLScheme` UInt64 -`OpenstatServiceNameHash` UInt64 -`OpenstatCampaignIDHash` UInt64 -`OpenstatAdIDHash` UInt64 -`OpenstatSourceIDHash` UInt64 -`UTMSourceHash` UInt64 -`UTMMediumHash` UInt64 -`UTMCampaignHash` UInt64 -`UTMContentHash` UInt64 -`UTMTermHash` UInt64 -`FromHash` UInt64 -`WebVisorEnabled` UInt8 -`WebVisorActivity` UInt32 -`ParsedParams.Key1` Array(String) -`ParsedParams.Key2` Array(String) -`ParsedParams.Key3` Array(String) -`ParsedParams.Key4` Array(String) -`ParsedParams.Key5` Array(String) -`ParsedParams.ValueDouble` Array(Float64) -`Market.Type` Array(UInt8) -`Market.GoalID` Array(UInt32) -`Market.OrderID` Array(String) -`Market.OrderPrice` Array(Int64) -`Market.PP` Array(UInt32) -`Market.DirectPlaceID` Array(UInt32) -`Market.DirectOrderID` Array(UInt32) -`Market.DirectBannerID` Array(UInt32) -`Market.GoodID` Array(String) -`Market.GoodName` Array(String) -`Market.GoodQuantity` Array(Int32) -`Market.GoodPrice` Array(Int64) -`IslandID` FixedString(16) diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/count.txt b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/count.txt deleted file mode 100644 index 91d679bb3ab..00000000000 --- a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/count.txt +++ /dev/null @@ -1 +0,0 @@ -485007 \ No newline at end of file diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/default_compression_codec.txt b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/minmax_StartDate.idx b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/minmax_StartDate.idx deleted file mode 100644 index 0cb239d932a..00000000000 --- a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/minmax_StartDate.idx +++ /dev/null @@ -1 +0,0 @@ -?? \ No newline at end of file diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/partition.dat b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/partition.dat deleted file mode 100644 index 9c176950d3f897f5cc9714d7011107a21e47a229..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 Lcmdlj#LNHy11td1 diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/primary.idx b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/201403_7_9_1/primary.idx deleted file mode 100644 index 6765990e38d151d57b7f18ee97cef776d10afa02..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1314 zcmWlWeLT|%9LIkfbtfq=jO5`;VIDKrgED^QVO-ecan`Y3+vPFyba~p2$@RqD@@i2{ zj>>tskZy@YSmSc#p(3L^OkGSfgc6qf_V4HQ{k%V)@AqqiAc!t{Fjmofqgz$l&^?9GGcP7X*~wPp&bP(`jJIVv)0UyaSh*M zkuB(XdTF)(i?wCF@H^}&##iOQF2_o~-?g#M!Im&&1`Dn|yKj^p)NMXm>R#*m@vsVz zZPu-jeCFLQ^*t=4wO&zy^zU)wnbq%T1!(hoy2-Dqh&DPQy4R{(Jf|lS`+48Rtq1Z% z+p6j7Aj-yjDKgiGgKD6p7;jXsEbNaG5W+dm#7&^^dvi-JGbrjCHOfQztD2DYU%(d! z_km{*?D<8$u8rD=7HVi!`$;+)YUcXREOR_e7t*5_Sy7x$jSSCB9R@B@AJIWIMzef< z3Uw7OG_zz{t^v53DhRdu7Ts9UVjjbIt1;~5oEx$&ZnkY`XEKg6Q1=~l9;uuQSab-U z#&&&*lslThj3&V`5YsPdF+2MuVBabpIL9b?mi=3wdnT~g2Ro7Np>s~>|6Bs(AUElxC6TdRbjifCdaVjLpWd!AZw5AV&PY5$@T}XA z^(4s1ON}ZHvdGCGNg9RVBH+)R8dqiZA=7xW-8B*1Fucm!@4hc{>u-malLxhWAZu@w z>3}+RKS%!N^eBM4i}qK|Q#IO!>bB`~e)>_iKo*9*A?|!p zXVNE9x)4w7!RVl$OP|C#G)NWF;ge|*8blx;7~?4tsMQkBBDQZviaU_!V*cXFjvM8x z)Y4c6VHDsXJ%ht38|aCSp+a{%E)`~wq?dU2+D(Mbt!XU3xiIML(%*IN=>qKrH$H-# z+{DD-(9xe%Vkx|(>6@RWjC?c}vW707wIzx0JeK%rQn5n{C_33M>P`}RWBddyB&^9P zAYW=9S!vp$h>sXJkV(z`9n#0rXCkeFy>DF)%Am-9xB^-A7uuAI`$;vt9$XA9rx^CC z^v>v3dOAO+mFLBZLrTEKWf^;w`U`FDdE5%?W0!)WUE{P=rKmK}F40f7XsrU5@xd(f zyy@1>lcGKTq?J3M9P+zky~BBxX~_A*+b+EUvTDWzf`uoxMQ-jhNn-v(n8^5p;em`cqBLGPl2ap)i&2)(v|-Qn7p;U%sQbWzw7yEO`m8-r)J^xyXs) zEPHDF#dT6}c_cgvT|2Jy4E0w{MMkC|i|L@K-t{5Ec11Xp*e;)ipUxMUZo|{*nv>t~ zJ*=%i!xs75-@o78c!U$0_#yQ&bs0F+Y{rsMi*sH!>>AObT!YI?zVnalJ{)GSWU3eb EKZSBM)c^nh diff --git a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/format_version.txt b/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/format_version.txt deleted file mode 100644 index 56a6051ca2b..00000000000 --- a/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f/format_version.txt +++ /dev/null @@ -1 +0,0 @@ -1 \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/checksums.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/checksums.txt deleted file mode 100644 index 1200b25bd52ed0332f4e976e5e109661007e72bb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 262 zcmV+h0r~!8Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$axV}lqE!_QlD*Y>FAiIEP1 z+5i9m&;S4c@jnI$V{dhCbS`vwbOV+!t186UB-eP+=laeWci#XAWMOn+E@EkJtA+vp zC@FDdb-WItZml__r5>08+N>Jd_3*EI_1(J_Im&g zaA9(EX>@6CZZ2eDbOiRJGt`jAL&ph4JQlLni7WsMaB^vFVRCscX=Hc|qQj~+&$gS+ M9BY!1)str&05BDACIA2c diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/columns.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/columns.txt deleted file mode 100644 index 4835061786d..00000000000 --- a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/columns.txt +++ /dev/null @@ -1,259 +0,0 @@ -columns format version: 1 -257 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`milliseconds` UInt64 -`ProfileEvent_Query` UInt64 -`ProfileEvent_SelectQuery` UInt64 -`ProfileEvent_InsertQuery` UInt64 -`ProfileEvent_FailedQuery` UInt64 -`ProfileEvent_FailedSelectQuery` UInt64 -`ProfileEvent_FailedInsertQuery` UInt64 -`ProfileEvent_QueryTimeMicroseconds` UInt64 -`ProfileEvent_SelectQueryTimeMicroseconds` UInt64 -`ProfileEvent_InsertQueryTimeMicroseconds` UInt64 -`ProfileEvent_FileOpen` UInt64 -`ProfileEvent_Seek` UInt64 -`ProfileEvent_ReadBufferFromFileDescriptorRead` UInt64 -`ProfileEvent_ReadBufferFromFileDescriptorReadFailed` UInt64 -`ProfileEvent_ReadBufferFromFileDescriptorReadBytes` UInt64 -`ProfileEvent_WriteBufferFromFileDescriptorWrite` UInt64 -`ProfileEvent_WriteBufferFromFileDescriptorWriteFailed` UInt64 -`ProfileEvent_WriteBufferFromFileDescriptorWriteBytes` UInt64 -`ProfileEvent_ReadBufferAIORead` UInt64 -`ProfileEvent_ReadBufferAIOReadBytes` UInt64 -`ProfileEvent_WriteBufferAIOWrite` UInt64 -`ProfileEvent_WriteBufferAIOWriteBytes` UInt64 -`ProfileEvent_ReadCompressedBytes` UInt64 -`ProfileEvent_CompressedReadBufferBlocks` UInt64 -`ProfileEvent_CompressedReadBufferBytes` UInt64 -`ProfileEvent_UncompressedCacheHits` UInt64 -`ProfileEvent_UncompressedCacheMisses` UInt64 -`ProfileEvent_UncompressedCacheWeightLost` UInt64 -`ProfileEvent_IOBufferAllocs` UInt64 -`ProfileEvent_IOBufferAllocBytes` UInt64 -`ProfileEvent_ArenaAllocChunks` UInt64 -`ProfileEvent_ArenaAllocBytes` UInt64 -`ProfileEvent_FunctionExecute` UInt64 -`ProfileEvent_TableFunctionExecute` UInt64 -`ProfileEvent_MarkCacheHits` UInt64 -`ProfileEvent_MarkCacheMisses` UInt64 -`ProfileEvent_CreatedReadBufferOrdinary` UInt64 -`ProfileEvent_CreatedReadBufferAIO` UInt64 -`ProfileEvent_CreatedReadBufferAIOFailed` UInt64 -`ProfileEvent_CreatedReadBufferMMap` UInt64 -`ProfileEvent_CreatedReadBufferMMapFailed` UInt64 -`ProfileEvent_CreatedWriteBufferOrdinary` UInt64 -`ProfileEvent_CreatedWriteBufferAIO` UInt64 -`ProfileEvent_CreatedWriteBufferAIOFailed` UInt64 -`ProfileEvent_DiskReadElapsedMicroseconds` UInt64 -`ProfileEvent_DiskWriteElapsedMicroseconds` UInt64 -`ProfileEvent_NetworkReceiveElapsedMicroseconds` UInt64 -`ProfileEvent_NetworkSendElapsedMicroseconds` UInt64 -`ProfileEvent_ThrottlerSleepMicroseconds` UInt64 -`ProfileEvent_QueryMaskingRulesMatch` UInt64 -`ProfileEvent_ReplicatedPartFetches` UInt64 -`ProfileEvent_ReplicatedPartFailedFetches` UInt64 -`ProfileEvent_ObsoleteReplicatedParts` UInt64 -`ProfileEvent_ReplicatedPartMerges` UInt64 -`ProfileEvent_ReplicatedPartFetchesOfMerged` UInt64 -`ProfileEvent_ReplicatedPartMutations` UInt64 -`ProfileEvent_ReplicatedPartChecks` UInt64 -`ProfileEvent_ReplicatedPartChecksFailed` UInt64 -`ProfileEvent_ReplicatedDataLoss` UInt64 -`ProfileEvent_InsertedRows` UInt64 -`ProfileEvent_InsertedBytes` UInt64 -`ProfileEvent_DelayedInserts` UInt64 -`ProfileEvent_RejectedInserts` UInt64 -`ProfileEvent_DelayedInsertsMilliseconds` UInt64 -`ProfileEvent_DuplicatedInsertedBlocks` UInt64 -`ProfileEvent_ZooKeeperInit` UInt64 -`ProfileEvent_ZooKeeperTransactions` UInt64 -`ProfileEvent_ZooKeeperList` UInt64 -`ProfileEvent_ZooKeeperCreate` UInt64 -`ProfileEvent_ZooKeeperRemove` UInt64 -`ProfileEvent_ZooKeeperExists` UInt64 -`ProfileEvent_ZooKeeperGet` UInt64 -`ProfileEvent_ZooKeeperSet` UInt64 -`ProfileEvent_ZooKeeperMulti` UInt64 -`ProfileEvent_ZooKeeperCheck` UInt64 -`ProfileEvent_ZooKeeperClose` UInt64 -`ProfileEvent_ZooKeeperWatchResponse` UInt64 -`ProfileEvent_ZooKeeperUserExceptions` UInt64 -`ProfileEvent_ZooKeeperHardwareExceptions` UInt64 -`ProfileEvent_ZooKeeperOtherExceptions` UInt64 -`ProfileEvent_ZooKeeperWaitMicroseconds` UInt64 -`ProfileEvent_ZooKeeperBytesSent` UInt64 -`ProfileEvent_ZooKeeperBytesReceived` UInt64 -`ProfileEvent_DistributedConnectionFailTry` UInt64 -`ProfileEvent_DistributedConnectionMissingTable` UInt64 -`ProfileEvent_DistributedConnectionStaleReplica` UInt64 -`ProfileEvent_DistributedConnectionFailAtAll` UInt64 -`ProfileEvent_CompileAttempt` UInt64 -`ProfileEvent_CompileSuccess` UInt64 -`ProfileEvent_CompileFunction` UInt64 -`ProfileEvent_CompiledFunctionExecute` UInt64 -`ProfileEvent_CompileExpressionsMicroseconds` UInt64 -`ProfileEvent_CompileExpressionsBytes` UInt64 -`ProfileEvent_ExternalSortWritePart` UInt64 -`ProfileEvent_ExternalSortMerge` UInt64 -`ProfileEvent_ExternalAggregationWritePart` UInt64 -`ProfileEvent_ExternalAggregationMerge` UInt64 -`ProfileEvent_ExternalAggregationCompressedBytes` UInt64 -`ProfileEvent_ExternalAggregationUncompressedBytes` UInt64 -`ProfileEvent_SlowRead` UInt64 -`ProfileEvent_ReadBackoff` UInt64 -`ProfileEvent_ReplicaPartialShutdown` UInt64 -`ProfileEvent_SelectedParts` UInt64 -`ProfileEvent_SelectedRanges` UInt64 -`ProfileEvent_SelectedMarks` UInt64 -`ProfileEvent_SelectedRows` UInt64 -`ProfileEvent_SelectedBytes` UInt64 -`ProfileEvent_Merge` UInt64 -`ProfileEvent_MergedRows` UInt64 -`ProfileEvent_MergedUncompressedBytes` UInt64 -`ProfileEvent_MergesTimeMilliseconds` UInt64 -`ProfileEvent_MergeTreeDataWriterRows` UInt64 -`ProfileEvent_MergeTreeDataWriterUncompressedBytes` UInt64 -`ProfileEvent_MergeTreeDataWriterCompressedBytes` UInt64 -`ProfileEvent_MergeTreeDataWriterBlocks` UInt64 -`ProfileEvent_MergeTreeDataWriterBlocksAlreadySorted` UInt64 -`ProfileEvent_CannotRemoveEphemeralNode` UInt64 -`ProfileEvent_RegexpCreated` UInt64 -`ProfileEvent_ContextLock` UInt64 -`ProfileEvent_StorageBufferFlush` UInt64 -`ProfileEvent_StorageBufferErrorOnFlush` UInt64 -`ProfileEvent_StorageBufferPassedAllMinThresholds` UInt64 -`ProfileEvent_StorageBufferPassedTimeMaxThreshold` UInt64 -`ProfileEvent_StorageBufferPassedRowsMaxThreshold` UInt64 -`ProfileEvent_StorageBufferPassedBytesMaxThreshold` UInt64 -`ProfileEvent_DictCacheKeysRequested` UInt64 -`ProfileEvent_DictCacheKeysRequestedMiss` UInt64 -`ProfileEvent_DictCacheKeysRequestedFound` UInt64 -`ProfileEvent_DictCacheKeysExpired` UInt64 -`ProfileEvent_DictCacheKeysNotFound` UInt64 -`ProfileEvent_DictCacheKeysHit` UInt64 -`ProfileEvent_DictCacheRequestTimeNs` UInt64 -`ProfileEvent_DictCacheRequests` UInt64 -`ProfileEvent_DictCacheLockWriteNs` UInt64 -`ProfileEvent_DictCacheLockReadNs` UInt64 -`ProfileEvent_DistributedSyncInsertionTimeoutExceeded` UInt64 -`ProfileEvent_DataAfterMergeDiffersFromReplica` UInt64 -`ProfileEvent_DataAfterMutationDiffersFromReplica` UInt64 -`ProfileEvent_PolygonsAddedToPool` UInt64 -`ProfileEvent_PolygonsInPoolAllocatedBytes` UInt64 -`ProfileEvent_RWLockAcquiredReadLocks` UInt64 -`ProfileEvent_RWLockAcquiredWriteLocks` UInt64 -`ProfileEvent_RWLockReadersWaitMilliseconds` UInt64 -`ProfileEvent_RWLockWritersWaitMilliseconds` UInt64 -`ProfileEvent_DNSError` UInt64 -`ProfileEvent_RealTimeMicroseconds` UInt64 -`ProfileEvent_UserTimeMicroseconds` UInt64 -`ProfileEvent_SystemTimeMicroseconds` UInt64 -`ProfileEvent_SoftPageFaults` UInt64 -`ProfileEvent_HardPageFaults` UInt64 -`ProfileEvent_VoluntaryContextSwitches` UInt64 -`ProfileEvent_InvoluntaryContextSwitches` UInt64 -`ProfileEvent_OSIOWaitMicroseconds` UInt64 -`ProfileEvent_OSCPUWaitMicroseconds` UInt64 -`ProfileEvent_OSCPUVirtualTimeMicroseconds` UInt64 -`ProfileEvent_OSReadBytes` UInt64 -`ProfileEvent_OSWriteBytes` UInt64 -`ProfileEvent_OSReadChars` UInt64 -`ProfileEvent_OSWriteChars` UInt64 -`ProfileEvent_PerfCpuCycles` UInt64 -`ProfileEvent_PerfInstructions` UInt64 -`ProfileEvent_PerfCacheReferences` UInt64 -`ProfileEvent_PerfCacheMisses` UInt64 -`ProfileEvent_PerfBranchInstructions` UInt64 -`ProfileEvent_PerfBranchMisses` UInt64 -`ProfileEvent_PerfBusCycles` UInt64 -`ProfileEvent_PerfStalledCyclesFrontend` UInt64 -`ProfileEvent_PerfStalledCyclesBackend` UInt64 -`ProfileEvent_PerfRefCpuCycles` UInt64 -`ProfileEvent_PerfCpuClock` UInt64 -`ProfileEvent_PerfTaskClock` UInt64 -`ProfileEvent_PerfContextSwitches` UInt64 -`ProfileEvent_PerfCpuMigrations` UInt64 -`ProfileEvent_PerfAlignmentFaults` UInt64 -`ProfileEvent_PerfEmulationFaults` UInt64 -`ProfileEvent_PerfMinEnabledTime` UInt64 -`ProfileEvent_PerfMinEnabledRunningTime` UInt64 -`ProfileEvent_PerfDataTLBReferences` UInt64 -`ProfileEvent_PerfDataTLBMisses` UInt64 -`ProfileEvent_PerfInstructionTLBReferences` UInt64 -`ProfileEvent_PerfInstructionTLBMisses` UInt64 -`ProfileEvent_PerfLocalMemoryReferences` UInt64 -`ProfileEvent_PerfLocalMemoryMisses` UInt64 -`ProfileEvent_CreatedHTTPConnections` UInt64 -`ProfileEvent_CannotWriteToWriteBufferDiscard` UInt64 -`ProfileEvent_QueryProfilerSignalOverruns` UInt64 -`ProfileEvent_CreatedLogEntryForMerge` UInt64 -`ProfileEvent_NotCreatedLogEntryForMerge` UInt64 -`ProfileEvent_CreatedLogEntryForMutation` UInt64 -`ProfileEvent_NotCreatedLogEntryForMutation` UInt64 -`ProfileEvent_S3ReadMicroseconds` UInt64 -`ProfileEvent_S3ReadBytes` UInt64 -`ProfileEvent_S3ReadRequestsCount` UInt64 -`ProfileEvent_S3ReadRequestsErrors` UInt64 -`ProfileEvent_S3ReadRequestsThrottling` UInt64 -`ProfileEvent_S3ReadRequestsRedirects` UInt64 -`ProfileEvent_S3WriteMicroseconds` UInt64 -`ProfileEvent_S3WriteBytes` UInt64 -`ProfileEvent_S3WriteRequestsCount` UInt64 -`ProfileEvent_S3WriteRequestsErrors` UInt64 -`ProfileEvent_S3WriteRequestsThrottling` UInt64 -`ProfileEvent_S3WriteRequestsRedirects` UInt64 -`ProfileEvent_QueryMemoryLimitExceeded` UInt64 -`CurrentMetric_Query` Int64 -`CurrentMetric_Merge` Int64 -`CurrentMetric_PartMutation` Int64 -`CurrentMetric_ReplicatedFetch` Int64 -`CurrentMetric_ReplicatedSend` Int64 -`CurrentMetric_ReplicatedChecks` Int64 -`CurrentMetric_BackgroundPoolTask` Int64 -`CurrentMetric_BackgroundMovePoolTask` Int64 -`CurrentMetric_BackgroundSchedulePoolTask` Int64 -`CurrentMetric_BackgroundBufferFlushSchedulePoolTask` Int64 -`CurrentMetric_BackgroundDistributedSchedulePoolTask` Int64 -`CurrentMetric_BackgroundMessageBrokerSchedulePoolTask` Int64 -`CurrentMetric_CacheDictionaryUpdateQueueBatches` Int64 -`CurrentMetric_CacheDictionaryUpdateQueueKeys` Int64 -`CurrentMetric_DiskSpaceReservedForMerge` Int64 -`CurrentMetric_DistributedSend` Int64 -`CurrentMetric_QueryPreempted` Int64 -`CurrentMetric_TCPConnection` Int64 -`CurrentMetric_MySQLConnection` Int64 -`CurrentMetric_HTTPConnection` Int64 -`CurrentMetric_InterserverConnection` Int64 -`CurrentMetric_PostgreSQLConnection` Int64 -`CurrentMetric_OpenFileForRead` Int64 -`CurrentMetric_OpenFileForWrite` Int64 -`CurrentMetric_Read` Int64 -`CurrentMetric_Write` Int64 -`CurrentMetric_SendScalars` Int64 -`CurrentMetric_SendExternalTables` Int64 -`CurrentMetric_QueryThread` Int64 -`CurrentMetric_ReadonlyReplica` Int64 -`CurrentMetric_MemoryTracking` Int64 -`CurrentMetric_EphemeralNode` Int64 -`CurrentMetric_ZooKeeperSession` Int64 -`CurrentMetric_ZooKeeperWatch` Int64 -`CurrentMetric_ZooKeeperRequest` Int64 -`CurrentMetric_DelayedInserts` Int64 -`CurrentMetric_ContextLockWait` Int64 -`CurrentMetric_StorageBufferRows` Int64 -`CurrentMetric_StorageBufferBytes` Int64 -`CurrentMetric_DictCacheRequests` Int64 -`CurrentMetric_Revision` Int64 -`CurrentMetric_VersionInteger` Int64 -`CurrentMetric_RWLockWaitingReaders` Int64 -`CurrentMetric_RWLockWaitingWriters` Int64 -`CurrentMetric_RWLockActiveReaders` Int64 -`CurrentMetric_RWLockActiveWriters` Int64 -`CurrentMetric_GlobalThread` Int64 -`CurrentMetric_GlobalThreadActive` Int64 -`CurrentMetric_LocalThread` Int64 -`CurrentMetric_LocalThreadActive` Int64 -`CurrentMetric_DistributedFilesToInsert` Int64 diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/count.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/count.txt deleted file mode 100644 index 1e3852840b5..00000000000 --- a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/count.txt +++ /dev/null @@ -1 +0,0 @@ -211 \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/default_compression_codec.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/minmax_event_date.idx b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/minmax_event_date.idx deleted file mode 100644 index 4b6c49f8d50..00000000000 --- a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/partition.dat b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/partition.dat deleted file mode 100644 index b9ea5569e3a3b34e3a1b6a1c4cbcb46f5e2d3fac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 Lcmb0VWo7^X0I>iw diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/primary.idx b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/primary.idx deleted file mode 100644 index 714ba773856..00000000000 --- a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202010_1_28_6/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H-_H._ \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202012_29_2358_466/checksums.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202012_29_2358_466/checksums.txt deleted file mode 100644 index 8031891da7f5bd9e7a744c4e46bca92f86a61006..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11371 zcmYLPWk8f!7ryt6iFe{6BOODTASt1MgoHtZf{DX0z`!trGlPJLEP`Mwirs;Yxng&& z3brDyg|)^?n1I4}2Xw!GZ{9lRp65BwxzaS5G((e}sj*5?sWT;7t0^+IMy^s0wi?K6 zW%(_Ax9Uw%Cs*1leY>)FhMxlff0qK#8BQ+=2hf}xqKRzfe1@63$QiXG9$%}iEg9`GF^A|RoKKy+X18a0Xvu(BUq@n9ZMMb6< zY+Oa67{G?*;3s+CVc^~M{y{EFG(kPZ$}tU~e>tGsSn3eDLZ*pRt$<(=Ko40WVFN(X zfP@QyU?9oA2U$ZvWUNe?TndbMD+$hS78nM)W+>*vv%y(uvP_v;qKHr>mqP@hi1(CW zLIBFcOJJD?_J=A|NfJd|np!6L1e%Gt^z_R;y?6X=vSH1r4j1u?0bHUTR!QDpda}aK zn%|2%vtaC`gQoyD7{DCRPALCY@2;|c(d*dzr76Sh_8$Nj2#za=4#1Y1@&z=JGw3m6 zU!*u09qYa77Wdc$w<4EP+}`Z>*TvjP)umhdmafS#a9x$aLL&jdp4CopE?68{BAWf; zyUW7QuV?991_NOEu(-IWp(>?Pc9}+v(1_LPnWR`f16cE% zI4A;;Xf~dn#cbe(P5%?3Y(m5fBsf6;WdoAO&}^3N`4L%E(WvbM9 zacaDu<;v7H*hF1=4?}(P=&k)KTYAnM&yKvgzhNq#LELSv&GElAZXGKew^}56ZEc6V zVE|6U4sEb#jEgfm{49UNi zC_e7300^EFyaM1KnI%)+rH5kYiy1>Tkw-bDDTiDy-wQ7-SbyI9ai8O=CeRM8pG-gR z^z}*4_4V;SlY21I0?^$?C$Vrq=dp2_Yl2(-gQ`?6vtI&tzz&F4%e7;1V{M!XLW|2m zwGTcgZfqVgd0Obo>A0~ef_6|)1%95ry{t!JN!9or1baFlh67lLO}I3(S6d6~a^B3Q zquO~z-q(O0`A3vQtqsrCN-P*Ko8eq^0PR6hlv*as%+kt^vydD+|IYt>2l6~Vuw3eyQFiqqF#^R!=X4cnrz+ms=g+AM5>8ZHyFA@6`(@pIQOY_uIJlo!Eav)F` zHs4Ip2^}SFuU*O)E|;9LU48Q1(tlty(&AnjV5qYIRrD~#YsjU#z7hzbso{4Lj}By!O=KxeN9}o2=^HxT z;^4~{wGZj79oVdBYNuP=@KsCmFHc)|YxVWCJtXu6FctiOOFunL3Gp0napdWfo#lL| z4UsI+AM8^p6?rkTEQMSuQ3Lc9FYj9VmuJ2N0(Stj!9omDLv=8lilggoS}+`-LTpz% zisbMhrmC3GGwQ@{eS$#U?brY8A92?2p9< zG7>`Xe66{kNOs4{A8?7+t+tmt>OLJF)N|3LYsZ)I`0x!4)Oceprqjcgxo=O7Ngp3N za9*z6is^o=UACW0kQd=s9#=SK+uIiFMf>y zhASgp1MJ3L>I)}ItxCbD7d!=Xti%A!v?Kuvg-UuKPy0K)<<$K3n`ck(sXfwq{^*0- zr{BX)VuZPVc0z;OI#1L2gEkWVp>-%=FhX<^;TM7jd~QC&u_(KA_{hD+bzmT%1Ba$% zD_z-5WD9+ca$g%P-FEDS`rFOMgA3);@GqCG(eGE@wSze zz5C&2RBcAA|f&T8`>QISNsnW(^?3f~3qL0K}L$Q4{=&1wfB0}su1VfuM zX&t9x17ft93bQx7UA+1KXDX)RAh1Z4nT2N@pw(XHqcwz)*eV3mIt*)~ME@snPCkHtI$SPQt28pHN|`*HWz72miLsFV9B@Yiv=T#MJR~ARvX#>7 z$c&jwl5+svy{(WjL8xN0rBa!Og2w19AgFF$pnK6Xc@KyQ&Xq~Cx3chkUqK|$F-G1O z(pr_dmavi~59AXENw+|Ss*~45md9R{WqldCp{gTntH&a6zV==X^^=f!zmm3|&Gfx6 zY1W~fy|s-F=>%vbFyR51@ce)W91a@!ta`f1!)?-!8Nyu?9#E(pKx!|yi8@>XE4L0w zpJsjZ#WRcf6K*w?O)JNd?q&Va%Tvr=M{{x)7V1w=B5@!ik4Q2`IWcfoYV2Ls2Qq&<~VpH4Zx~W%> z>N{Q--*SC4)d}VI={-r4MY*3|<>kyi8J+h7^66-Ry|@#G^Y;%cT<17u*0Le5seiwv z?SLeECl{R|@N6N`jYEPYN_WHHG3A3)?jI#Ug6$9JF|dh+6QIp`&&-;xDHD#) zo8jj>bd%-ZHalS_+{c`7PJDGZIO}oawVL{ve_jCrjVD|VTFEAdV7uU4O?yaR`u`1kO_-hxcNSnp03Dg-MH8H0 zNP7e94d%=;Hd+8k;+Kg%NW9X7Q@C7DXo6mW9(YT1d6=D?Ai->NFn}YD2&Rlt5lOMT z0SkEn4rH@Y2{1t^QFX0y_&n&}f6ux7+`)AgQ=p=%~{V#;ZV%vI1 z(%nIvA4&Aku!*Q(g(OQej9{vJG?722*k^6tDA-$0Z0#_3@os%%+(OO|IIm~PoPVc1 zTVGkyZXO-Y;|?UTldy3X*g;ITT(RiR)xB?D4)w`9mJ{X<`~kcMQ3b%8cg0~yOKC{i!1scaCkCuoSaQ(XH6tFYj0#cyw&SC3eE$T1QKWM&Vg4!_YM+s_t-1UCjsI zDt)!g)XAqopRM1QRc;GLFo^+L3?)-&Bc?$gjRt(U5(r|r0w+iva$;5j{kR`I6u}h< z93hGWY_t@w{6a1Y01#ACE|{VjnIjiyWbSqa3ReN{7=YG;-ifhVi9%M!SmGPK?q{Uf z>lFr?*gU25or5#ZVk@cjs%&j=u2d#VmakJGkLuU;KZ zNV?bv+F(S7+>|&Y`E>1oQIr`$agQoYt%02sO*_)f+Es?J& z9^bt@X-?wsF^&c0O9I8GdMAE2z&HTe+sRLy6>%TjV~YW=VQ71=X=_N#xt&qAkk!R@Pyqb!Ut^ZmUW;ynO?`@r@;wD3? zSDorT^}x9;TTFHYK0LYRr5=tmnDVs|F}4S?QH|5gEBp#W4y*7WSPpH5N1m}|8pTc2 z$+eLA2kvq5z5ZbCsEU*CH*Om{9k+Ji`dnVp+tJti1A)iQ^dmdd1YJaS2A*l}e~wjO zbUr@*pJT?GQ%?3NYzJIV0D1f-@{eho23)gn@o#zMI;eN6)x?mu1-L0^^6rj>yX^yM zd*>PDRUS{nW~V;&5pGLKw9og6uYGJA4#)b@=3}Y1*sbJ#1f+&)f6)Y-bRL zLyasS537^vv(zZI<(Omh*7Mfd>_a?mlb*+c9=FiTVxG}>k z`WD3(8{3Y~+haVId9&vT`P90&-Ps56!h zvKc1-ikWUFJ$SWICF^v+;K-8c&CTmpZ)3wYINP?`-79}#ab4@N&GDW#Sp^W!fxub} zICFKux{A98wkz&U@3kQz`7K4e0uaI{4)@XNFg`QOqHDgTZCG3zYS&WCS%_T@a*8kf zyp=oH`em5(>+{-UkKR(<{w*lCH!)NZv=66={1o+9{gpmG17v;C%7n1_KEBqIF?GtB zv)TU2S?f!lGs{LUXrHyl2*(+mD7M2*ZkZK(SLXSd{|w9v7==teTjIw>aX1MN!ocjp zF;rJ#x;QJ>_;5zQhY8Due*6#yg_#rY=Hua1hMn`-eyHWVW`~6H_nBn2H-U97HYOk) zWVGezF)Js3Fx?&{<=?7&@)~#^#YhC8Re<8GhhE-jF7X(<(3~8v@Dhuo{oO(j-Cp

tE*-EM!D=JNt|@+#bc$NKO04cs5&wmf+Gu>W_lI>|ngkKO=) zBKe?+y1fey%C5RKs^DYPnIRirzL<6X|;J~dwz#j+T zCcJ@voL_Y@sQ~1DuGzQiqT4wc$kj)i@UmbK(@~#_w9_8K+X{~d4dU_TIhn?VWy|oK zp9MelYm8emy$2Z>e`KRC2%%%cW_&I(}!ANkVFv8=^n!(`991K?XEigeByruJ zy}wz*<3L}N!DddGG?C9Hid;KwlsO@zr(EWU!)}Ku=iv6@kB48n?02b5Qa;#K8`V&s zPXJ*Az-+u7ZVaol+{)Wld!((^Xx06my)5*fM#!`|Ds@JTOe&L4c`d?oKEYJ%bLi3E z*>eArdHOC9$rp1f@SGPly?lQDX6tE}#FMDJ+v5}o$KGW6|FtJay5jkT z#Iy0`??hxsx9GYCTbFEPnLoZ%ddYn-J8?pw9{X;O{%df18oyQN{2sJ(BS`j0n>%-} zz7zc-9PKcnPA}bj&Ex1n6YjS@|M+qE0k#|FgU4+-qim=SU*xYRVh*r%JX04M4zQWw z2>Ah$^szu|%#9ei0cJWpv|NY1%(=+ep;4no%hlR!iK3BU?in_bE7|E29`1OrPJDaQ zwDE$dwJ#X+6{}T+=Fmq0LICBW; z7y%Zf+F=3lETUj-+@;lxa~7AR3l|lbL`KFAk1Pcku{Y2czD)Cm&` zD@Jcg&sed%dURyVR?j&aK-dmVGjPShgKdz#Yv$69aO(|M%s)ClN%C&Qu2!%hxyQ+U z1-ILh#!d;fx$;f}AkoE9eh`B#LI+U_ip(5Z6pn^-k=~7fyj%modIs5A% z5ff+TcsS#+j!q=f%43^`>|js2le4kxx~*$Z9=8iGI^pK5CEl~1Y-Ak>lPdgj*jZSm z#)PmdY;>}o_fC@y`;pMP>hQsph0EXCWrJ3t;zPlqpcnD)nsV&@1gEvemc7|zHXwe% z&+=KXtr_l!L!-*8{Eq1`E+A zU?KE|oNjeOm``AJc1Da0*VAN5X}A!np$I?ZcV#e!#F|i4gT#1HfY+VVFkdNH1f8?8 zhrSwsNRE-vhoHL!f8>IhY9`?brU0V5PoW7pd1zL4xI9&zX^LKv#um2K6w^g84GMKY z6&Duxk+2XJg6@YT)5!+!Ac@Iu3t=sEr-j|wnlOh92+qt_%z&tccIqA)$?Rw0ZN>gT zV>WEX&=&vE@eW4l3(z`f&2%uoskShRX4^p2k2DslNpAO233m`MC4~48j2uV{eDk}r z`;d*3>t*==G7(q?bwwbIfd*0Ga%He`tM4DlG1*Eb&QY8MsG2qsF6=53;V(#G35AP9 zXcjON_Mmv(+N5wZVI3no!6FD2I>|D`x4(N2q|~~EMj=af<$@t`ATJDOhK51~fhR)f z@(P?(@U24BTnBiN5Y-%nRe{*lL%6dW|N%fYos2bO4u`gQMUMOx?PIAbYwll}H znl;RJOYwZKL9=T92sl?yN+uS{(dvQK|FjKqirW~y7NcbH^qq**@DZ$yN{|gzz36QmOecn zpg&0w<3=#^06Pl6`9N@nV46w1idau@#bxn!5p%)?YsKuAhzTL<3ktW?a7(7A31LN(zT-#1II_ZKS~~>cWA}Krfr?uG7l|y7HwF zGn9*mPoA|Qe`SUmQh+yTBFmhS)Bi;yGY=e%fSw1avMUXeX{2g-mbNIWiM%l3H3E#H7ru-7lg~yKf z_uHM9I*P=&S}%gzF|0ZE^jr2pRB2ySJceI=E}c6;j79(fI^}J_;LC|vKHdQYa*|2p zf{}p#$+C%!!EVk1n0>?S!^|0nr3)>V9w@E%j`hRKZ`B%peQcJ~+T9+9JO8)||C+?@ zsza3kL7SJr3m|J#Oc~d*Gi)^)N8(YnYecHY_@r&Z5?p57GNn3wVA7)WMZUzn=X;l! z^-dv#U6rYJxFctIcHE*B_ZM~S7mS;AYUNW#8Y4@Uf%Z=UcSaOr$w*(J`pn?BN~Om~Xa4*${%bqpNYROePv*%9NWo%qcL zSJu;Uejhr{n)eMb-6h?2$X8RaFap>a?~_Xuv1!@bWL1u`9iY#&2*1L1C3GmmccGy- z(1BULzEhZLnY#(UmJoD@HhvL7c>*AS$T-ph-{8gQ61SLj0$gl^s1Idf=3qbwv7X(n zlwoqkCGeZfNEB^k!n|Ub#enG+*U$vW#WBwkVMHeTnBiiYPV?ZobGc(6IT%<+rchRB z9WusGupcvBetntX!UX`<(C7v!Lg|o8{u0+X6hV%L)X8`pit$+k9@h}Ex^;CXn6Zj= zTg;+G9K^=38O;24{A(294EV(YG!-N9H3~aP*YN{{lX=Qm5;+u~Ix$KzW+Dk<24MbE z_qgz`KkCG|Tt+Y@;8*{;n?Yb5_U#INj+=8pY#{g7Xtzy6gWQt{| zGGTW#(8YZIEdnkVAOq5zvFBpiqAR5Hml8PJMMk78e**EV6o!>Wf~h6YeFHk`2H zn)@KYkT(ca3JjUkEM6%fyH}E5PH@ivWIBVI=#wmxo7qCo;SxWF@Qb* z%J7D>WPDk6smHQe;i0OQTW7zpdU4bPyC_yA3;Y=FpRmU-kryz_F4DaCoC1H(Hs7h) z_17T|wQTn|^z7!gDRa zu2bSgwLm9f3^Qt?o>V~p_`L?RdPtvLdnZu0md^TD$CSTA!%^WCPwmH)9I-y`CgtWORAgr?oAH< z!MZ7s82ElpVW7o__;m-%pJFnG`4K+X(FsG>a97n?^&MRyA929M%fT}Tw@wEZQuqfG=PD+e^|vgNWOcu0LPD7E-=y0=q64$C;a zI;kBxapdAlZi-P2b|);j{qA^Xu;3el#ABX<&B9r3;bXgb?Hh;Gp6k;YZSY!8|5$}8 z=PMUnAkhT2$kY^2RF@7Y2t>REaIh*xYvb8Wd}M833XW#4o4M*FH*D60ygtUG+hO## z`Z3`et1U8NHN2lBo$oG$WZ&g;+lb=JgmU^V1;f=)V0qS45bu_BE*obC*cCQYUI z8E3*oyKplg8i%(7E=!nW>604yB+5&)ZJkTk*Xc!cNzg+PXTLhp` zFb?3K?BFdH3WYZK^4AlRw~8B`r_stXy#a4}d^7Qx^>NO%>ZTV%w~aEs$|@%3TnTQ6 zn{#tF9(#1J>hp~vCcW6I{3;(uTwS579ZoErSo5rZ#w*{qAFL~P*A;wW^y1=dC~Z6{ z9Bs**V18w60t<>6N0CL`LYX>Nfs4Wv-#+2;gf#UTQB6y5-}g(e@b#4 z6kvkeqn8D9kM}z+G0{-q0~W^1N3l>rS$qFvBQY=uZIj8 zds__+uoRD82uq9e`;p9MFvz zJsr>LR^C%Q*uFrx!yH-wupDoiD`|-@NkXI&+WjoT)~5X<^BKH1hrgcRSuOr4}2xgMOZ_}lmFVzAW=>->oJ#gyaTRd{F; zF|umc#hUGtoyJADKfkvP$A_NNK-m&?&z{?sFPgJvh$6^;7@7nO;BHL*{Jz}}o~C-A z;aZpFxa!J+pCAlapabR>CWAL}(QbgqNdQ&>0Yfrcnl6z-e@LG3Lb*o9%&hZlal8iB7qv5aevKziE9)#09 zBPOLk>k}ZlGCtGD;9R+8q@f8umPL`>Ngxx3=)J%xR3;^inUUxh?l6WN+6DZf6Wnto zYFTrbURLc$coU&?3|fYq-_C(gkWhZZm2xgFZo88d>`F&Wox|FD0X7o7nVL8oG*HbOb%@4@OW#j!c0 zGw6m#W)DLTfGT{NOsSC^_Od|sY>L~BddP@PfjK-x64oe|pm^OR0QnIV;n|M~8G_1i zZoWiFOQE0}`A{6B^x}B~FlEwYsgxAk1f{{>wK;39^p-_4N1Lmq%a)vAfi-&buF0x{ zga3?N0fe%C8Njay*pc6ikHZ%I58~OiLe1;%+x%*01RY~yDrFM@ta|7dnNW0kelkv<-s@P}yqg?lCRrP$Aw2wFT z`~&MYQ3qGJfBzo7E60sEzj;Mf%8~>CJpfDqE#$`x!PQC67W|uK8D+h}bC@qMn+kvp zuushF)1#Sudzy9UPTIgyX6}!_FZjpXEswD`>?CYzx{XgCuFFRHM^fJ}<_*f8Ub4<@8sZ0Kzn>fVeNiQTHmCb7Fv;i zUG8LXEw4Jb<500;eH4F9kBU?mgsx0Ns5yQT(=)hGr&DSS(Mp3(m6;x-$jr;pX$v)3 zL511bN?n+`P@kKSnXAkyR4Ze(T6LmApVvJ@rP)VaUpe&4oV2iCldZ+KJ{gu~;wkHy z5j%rCC0kCbHQGo#ek$$Yn}!^ldHK3L&xjFk<1UY%l)Ld^^UKpi#=0k?TEsmPqSC{( z8HEOARxJu3X-Gh75iRc(k2#N`m3qA*M;WBk<|%cXQ73ut;iv=I3V(NwMqD>-Q5|-W zcZoq}WIEh)@XS{?ad{W%3?FksVnD>v0c9=KceerOT0}+#?xT zlO>3@aSTBpwHkx6#1N&;%u80N3|+BI-aY~e$rwaiJ9wiKy6kvCKyYFk0>?_ zky1DU`%<(sT&>Mes1tK_N<}@YqfgN{t)@I4-cG!3ynjW|o}*-TiS;-Rouv4lC@v5TSM&wO+ep*VG`Gu@ zIpY1%HIruYhIAi;ns9R2+r7`9f0@)d{oI@>uP3?;=8^#jfr~Uz!-}@qwO{18GNsqJ zmj^A+6(d(%9g&zA8?4o6lxH~PBbEt!QeY)qhQxW4b*Dlc7YSC@Rnl@@1#%R+@yS3$ zlc}&jyQ#<*Z2uFJLP{zOqWB?*6e3#4a(Ety+l0Z;QehY+X|<@Z5kHPr=4*AOi8=^O zl_sYV%(%MXT^C;;wTv1W>3Ku{bYGgL6b|HGQskm}Ha`YzF7+oQ2#-PH z$CPjk3WzCCYX0T~gY(4?X6opDTtl@evCLZh?_};O&-L`^cXe>&=F`9N`kbxv_uewe zQfkqDQA1*`t$-M|+n-W$Gt?n`waz8DdD zT&zN8h%PiJY*{pnB6q*bM;od6dWO(Ab!#H ze0+J>%JodC$Dd_x8_yi(U>7t#R;N_v7Z{WWxI_iPu<3uUgoWN5+yikvd3E$F%0)qj zZ}a=U_gL75Uk`?5oOlDf*6+qOLtd=5J5cEUF>3hP1s#nJg$rTJfnPTpmT;e{cYe-y z^qWz*1pUi(h)?zuQtlj-NINQZN7y=g6TeOr8M(=QJO1_Ki)h01vJQ?QNvdH4-O6K&+Wg{XNKi$>t#AL)W#}JuoY5{f-FfC7sCD_{H8x5;N z1#}&~l0Lco?&Z_rj#c|TwysrLu5tba`$=l{SU3kw!Z@=_^s2{P7#(fda0ed)~ zB|cf>e%8cds;1&)A8rg6bwxY08g*&BvOulMROk@-4@|O*)8^gJk=WaKzOw+Zf{l=y z0~z`{M;wVrIrxSX!}USwVM>65AE+~P22rhfPq>B8^+sgA$cCImT-=(^ZLNnJ{Ti0YT$`?cMQ*Lm|UG{&-1W6q@mXbnFr4UKL3a5BMpjLxZ#g0cxj4A#QBvllX2 zi$5C9$h)oTs~wX*MJhry=t^h_cjwLOafc)dN#Tm3Exdf(e%Sv^8m&D-d z;yQX6eP_>))f4~vF~ncW56t;@oMQoGqCp|wmml7%mOfo^YL2^4T|+k-vO6w?b-%xO zrjtp(Za3F`TFsmvefdN<-+ho$_l!cpV)q=S?mn3CF!f~Ig&$ni@ke(Kz3dwP$4lE$ zV8Y$Q{mFMlKM#)uCG>!=_Q~mVYjXR6+5QFoHbazqw*E1;zQf>}P@xFF&hSBol)W<_#K?fMVbTj)(w@TlJZ6c z&Z2Lc3g-^hY6mEl1xnp0Gh{&IFOF2$i2`FoQV}Q7!owVn4{}dX>eU#`@5Z!wfjPw~ zM<&Tj%mMOb!l%YO3~8PD!jIKyvjI6ni-2#X2LkI-1gh2A%v-SSdis2aB?qc4qsp|E z6_<7?*(q;vGj(l&)AtQqUr#7{l=^Xm@19!6aKr-0G*iXm5zE!u{Qu5p)E%mwB^rc! zAWl$lZlT7QDxT9Teq9P)n{s3Rjfz)q@7vc0j)qqXv|HbM42^4da_^_f<4m?BwjYfc zx+Q0Sr&oq%=477<^*cXp^=SVNx4NRP-spU=LZi_d;sJh&lyl68Two!0i9_v-AU~dx zyY~Zhbf+B?wLC=SakRkXAYA&0{-5nw%R|s?ZGHjlIM84?BO*&FncN`^C>{X4yt5a= zL7^oDy7{!Zeho#e8R`F%IX6EfIufnQ)M@p~OsytsD%Vo@1ye&Y?+FqNLZk-JG#OJd zVTGE^i^PVVPSev6r!}mIB}Hiy3Ntg6dInpPV~Ao}xrOnMc3Im{d}xU>vv55Zt~-lk zkdZwKztE1_{AHAbGOLG(+C}#WQfr%qb(GDX3g^eH_ml=hN@(uKQ|!h|%unj{(y(nq zH&uTf@cfu-$xfFg-9}L8BL$oZ0cZMw%_7D+ueXAr<^9u|$lz^Tp2oktSU4N<-0fF41}!aGGpdR-52NPB3!fqrAT4mrP4pNilYnr^ zj8MraHPIGpeUa^D#EGy$y#NNE6_t6&Cp2Db+Oe#3PNl&x7LNxj`yawX*Jt%EFfSBI zj#LDXwhM-QqeqRj--x?;cS?S~j{nx+H|8(LEvS<_8UYLo64_@9vVyV{swpE&!Bdg= zZw~2$s3}p}oKTHHS9;ExY@pgx)6nnJI)x>Ei2Rh=3pW_|&4Ufna_4mzbG>Qzz{nU! zidbQ(ykit1fC)6Qz?7g8#e^wT>P=EB<0ZU>jc<|jNE8ztttgOT8R63ucLu`I$cVG# zG!miS2nMo~yROATUIL_ku+%mIrKPx+fMV#fRJRv|NWyXD)Hd}YeP4m&tW6W zdJSTlan+?o@r7p^#fi~Yi;sCNPW*vIoR)KK#x@$!`0|J?cDGz)#JqWnV)oqAHyM1gexSRo&Yt1+t;R)i=Fiok4xa+kxd5S6ha(}(G_`HoVwje@h4 zT6+Ta4n|_K3{21?6hit3hpa?h670b@+JklEar`?Vp>sNfrb@SfYsGG30R4?~(X53?&euM={jcR0gFWt9cMQuz0SI5fWFCUhy_8}hv${OGsqW91 zx3(*%Htf1S4;b5pujfq{@7TO~^S*TtB?)c?-R&?*giXx{)VgaIp$WF5t+k1N)n0j8 zoAStvC+WVP8I^gsikNFhhYJ~AVNkOyG#M;A!N$WjY?)w5TCZSZ5 zInc!&O5nB33T>ewv?Noh%*x)((2$|xViDFu3Q zgi2|qxmzYwI_eJwxE`Opif_(u#I7Ao?u7ik_e-q9tJ}eU`Q`h{_`wJga5MgxvO!vP z)9)OLwT<0$bhpE21TdvkD>DtH$d-IYnBTUdjizXt3%b}91_?G-RT?L;A}G9dx3iX&2}TY)aBkZz&QJ^6poGFcQ$O!)2744g0wsbB5J8(oA6kf zVD2p>(!+$I&R*V0NBAXXo*Pp} z!H_m%f)=mwSBkY$FFv9)m;IlaK+*y7*GLuL>;J@UTfF-}<6LUDNLI{13vv6%m=)IS zJ9!;FJGt6n`ID$M-ob~KT^}*6e4Nb~7|i`}gMR*QRu(sX#fFr5-j3N~(jN>w9yXdi z;nA!fogRsPoZI>N;@7N4jY!ZFVS%WQu8n@vv|*3c>upO?Q|`>(l>XOg<4`BRjSP;n zvw3(pdrT#L@Qy_zYF$?oo+G}x?fW06)(uOW%DLU%YI>zL^%swGgCaAyQZ|}ywV&9~ zn=KXb+!4CN7nO>D9!ptQE=0S)hzeHrd$?2p;xz%4U+Z`ESa#LQ)k{~0jz9e2%l7m65&iwsYfN@UYKpY#LX81Ls|s8Hgkn{LudV1BhMdm@OcI+=>~RzrtHeE~MQiB?@Hn%jM}ZlXzMu%)>6Vh|7Z5V9?;tSKP5@6Qn_1&O`l2adt5duYPC zTjrarUVXW0`$ziYgw;&+D~98VP$*ETPR1gkfwFMm+6MOSWjo}3QH^C+@CK~>^de{d zp#Gm<@JH8GEY1#i#k9Jfcy~uD3oXUz!>^;q+h~#&oqsa1{SxyBg)93X^4K*JC=TN- z*E)>cy8lbFKxrFN8X|*)0M%S0)hy+Um+bf=og5K5ZLG@%8acCYKOhmx>oOU3=UeF3 z3hU^{JulhRE!tnZt@fgiZJIQj#FoKwqS;lQo(ul8dB+)d!7ss(-iHD~iw7j^7yWH% zyXWiTf84I>mV0&9Zl43F=YD_;gdGCahkQ&*lLXWQ;I>FAH;SD``H~$dA}Vv={k8<6_&D%eGM_{xz& z<<8;LsomSdJ+c+3(9D<%aWmcG5J!6Jcnh6-=+3!)JFzG6>JvlQ0$6@($B6ikeX=H< z=_c4R{LQT%%S0pwaqx5qlrugu)5-g{ol`hI-oL){oU2hH;s=Y15`k8_-|z2~$F&Vx zi{@mVd9?Z7*1`jeV6bA%tbxqikcUo3UoIQf>yIjU12QaL2WPQ+eC+DIFPAhP(@)+r zKG7Ky% zi3&m(?1*o9-n_Bi?@5xY?)ikhlQU9RVJy!QU|7}!*S_VKE~S*!NOGp_nK0c{M2&^# zCz^#>%=++(%*r1!;-_`${Z$yjCQ-{KS1V$734V63ksW{Q;(7hq^?FBq0Ga7kJobQ9 z9bF;+cXjCX!;|NBSmbieYeo41i)k?4wfpx^1-{%B@M`=Mb?ON7@=+9$3`Epa^qsoq zGJIi#>3>-^r#f7{v7LuJEAgX&N<*<$mlvCTMKT z@Y_xk09CvKOrct;nD@z^8o50ni#jmVa?MLA9fr_P zSMVO@6LpCR>-@w1%g!HXbJpj~5j5g|ZeObn5cF+Wx_*|lz8k_XrEM7Bg!6io z?WJs|e+`PbZ!UF3mDi_x%g3g!q{4{J9tJ`XN>S2JT{CFe*ia+p7z#m_7wRS zs-h%B7J{=f?D3d{;Mk-=DxINFq5ep**DO9!rsYJK9?*?%1PD1LwUMyD-eM;wUIvL2!;h=F-b z^-s5F%0~)skDr~byP%=WYpMx_AL(9io@{Tu#4e+8M2_wPZG>M?ewuK3+)qPPv5Uv+ zQ{fK1lxyafTE)a9M8-@)0EnCD;{%u9*Sgj&Tzkgj;buSehe=?t6MLhBCkDKw1KB-2 zVm6##+;I~&vR0!MHB!Kaa2x>nEh&diEg|%Y9&_*Psq$+thol@?y7QHS7M;t6sE!{Pa? zNPV#XQo!AvaZ`sjphuW$p#e=8M}6vlwuh?s!RStQ zELN&=u;t0e`Y&b9qodGDc?4R*SfO5cjT#fI7%Fn2B3eFgBFJ-YC6L zA6%OG%iEg0C#qj)h6Xc0-u!PhjJ4qkRZ>{7dWfZ8N74b@_dX$>X0TQCaQn)E< zVf+?UbT@K=$YD`P*jfXy;~6|2yB8D&KXN4t`7((g#c9>oi3d1;KGhu+A*yvwi4{FE zxS%jvm7^PBOTX~vt}OJjc8ztpD9O)H%HQ1 z-bjk+z|j0cbp>C%c7tP>?Qw_ba7n0XBAB0E^6Hqg9$v&C0&x zvf>Lh8esKqVzP%Llgw)A%fJ(~WFiAy88QXgNK6@FtB@yILpjDP4{<4qOa9_oz~%4R zgBYE$1W+rpJO!v{BGN@cDPRDU1>rHOdyX@dAhTMGldgu?Ww# zMZ3L8yu?;`fdi)`Ffw>6i?2LRds`4b=r60v8-!WU9#7uAg_ZBzeLRFT;?8-4Q}6AM z6z!><_=9cy1MRY;T14I^96r(P-m0|A)q6I)IR57Lf%VT!a989Wt_ z3jDGVU+B2}^2-0Np;<*u2OOtCDr^3F+W!~9^PWywEqXts>G)S;{lvjDfIP`B`J~1! zZMVp4*NLqgyX0X?ht4#7FPLP3Dr`w8XguC9}jZD_$e%_V3F@i>kyc zz7Lf{jUy)e``EGYGYtgSS|rY;*)#!yEH^+7c)G0Hon!y5n45U!XSCX9-JHpjH}wM> z2;`sk9UoDibSAZ;?&0CWlWE4#!qCC|NgZe(&}l{Ns^xvDC*hXz9ycI^6S*}ott-sM zkrx_zt^9S2_vy``N&oykdLpbg!YBH~fWP_AXUq!h-fUkd3Mx`v`WYaT=^>f1wedju54{f0RTw)5I*f<`PVV6auL_fnj2*j{{Vjw6TM|&y5mv^K7?~COjAiE_wpE-WsaOCh z(4K%?MLXL-Bg&{52)zW<&NxsD9b>ZZIFhBr67GO!vqq9d&QJne1?_1exk5|H2rQu6 z#3?>Q(YBb`oeY`TQg~b_u)y3_O_~j69TL3ea5<)Y!UGr$W`Bpm1qS$sXuOz=0%X3x z7)xSf`=3Dl%M|&>1jE-Gk_<~WhN3DE+lq{!jtTJP0H6$(>|~0ah}z~iUi3n`cx9GK zr_8+Al~ln@lPM!1FzVt^DG2n?h}RknYNu2(1Lmugv}On+2#Bsx_~Ed z%UHDlf90=Vue~HNVh2cvO}I#Wq+6+zc5$PVe5YuSwRD~g?4SmcCYi-be{b~MP1D*=s@ou z4BQOpQp-@Kue;@W&wR5p<)rtrjIkBYFq9{>&?kf9v5x+dwK*;2)VWvl=jAfZ zz}zeOM~3V#>AXpL!a3vW=$+!$v5a(&cp};>UZIh&qy#+bQU@j*Lm^~r3z1b6$go~5 zCivR9+;U;G?;Ebw?weh_SNu8c(MM282Pe-}mvwcWW%GJ_?~`=JL;IOD`llz|#(Mko zJhNEbZcstGTjYwjYPXmDA_EfC+TsQQIVF)3_KE0=B)^FMKi7O6fHWw>B}G@UgLxN0V+0ncOw0 zPp9LY?k7S$1w)78Zyd)M&E*2}6ll53`3>_#CsB9{7OmS9jz!?hsa@2BF)UnT7)Di z$S8P{2rNM-J>gCVg0Mt948coI_zIVV<`k{NE{+J&*WQ4}7l+ z^URk_7fac9_xrSd@pq?&5E^Q|3|P6QZY&J zg;Nj>5q6LX^6*p27jq7Uz1jG#!r3V<2)=77c|S?NHqpQK<;?B>)Snz2{K;4n8b4zY zm-k>3v3$_HKHb?jA|tUg(wzvJwc+6mG1b?`12~2R5qGYPO~t_WewP;}7%~~cxm-xr zG8V3WtI`=hyuL^hZHw&?S_l9=!FOrWvytIlwkLU(zvP{FuGY91t65pSBoE%k=ATa?OFU5-g6}<6N18!MA`C1f!BEC zM1PePSvL#IW_3wDTT?%1N7*m;JNWoid^d4see-QgH)j{cw0keMcOX3w)}aPGa-Zq> z6z%CzlXgf`g4dC#K}c$t;aP9FaUM11+{4AzmD4v$MsFPF`gwy2Pn=6-2`G9X zVprs68QsrmFv30v4@RHqwdE53u01!)GcM$IJ$_`k9NCOQs6BE?&v!PhV}7ln-@MOS z7W^D_%b58pQ+!tGB@x0`nL~o#6~}xzzGqY4j#Spd%eKMbEbx^E&7UrNq2-JvQuX{i;_@X^&08 zH;19m>_6SNh^BpslDz!<_18*?<0NFBuhQfzO46%QovHET&F-qK66T`y(datM>Z!=; z*>+xD!f7NffG@*TLA*NB9Ufv>1{fJ;aAp9pP!tuhyA|WvEjG5# zXFt2U8;qX{DBm6i-}m1=H&5)d*4k^|%v?ogo~|%o=a8+@=F9aCMGCD>sZn=x=*rgq zqnTf@_WN~A*;aRiE=@wHDSi^iFQibbRjBn*3cXgDnciETnU|y06sog& z7iMQGwBf2kU2Z~Vt|F^YrHIvNREct3Ugr#@`Y$5ZtclyBaTr}!Id|1h7yskfo~( zg?dHSMifNTkdSUfjD0{n=01v2=ydWNMQ^PpPodq6Tye^ueTI^uo zJ_ec6N;ucH2jU8fNiDNW7@l&+qW`$RPN2N!rrnx64;GUqGXFHsV#?8H;iK;F)AkQZ zRJ!y-Qak}QF{6$==SYe^zFSq)^TKKOTXm?TKjPAD7}qk}$g?$eGcOG}bEsX})Uihz|r-v7X-U#^%v$$Jp;4nj?) zA##|!P-Uhb7DKKrNh-*a>lJZ@ib6$ixgIX*mN6FgT?ZpOT!kcy7<-QhWER*@QL+MA z+IuD=Yq}IMHjbg_u|};|jL`SjWacHym3nvR-zM^P-;>3!KOX24gcJ6X_Jif8a1)76 zKA1K5N838zv;X`GZ!@RHsSgt2G0?xDuF`Jd$p6X|)q=6RvKreX2Osn(RH2fWD6%5e zI)zp*z?Py@NECtaC?uJSZJb9TPw2Tn(Znl;7b@MZ5xDQKZNU*Jz^@+8o6M=;0aBQ6+Eq`Md2R^ea902V|LCoq!u~_Jhcd*=m3=VHm>2sDWfJ%yeLZ8}v8tzLO!^dbiU_<=TrjM8 z(KmSCUJ`S|@#31%Hi2O*)GUX9#*Fqc@3DKb~XY93Q z$WhcqK%)^uD`0qb704fA<`<@gL?-G@1fhr)B1Xh=`JRZ^faxeC6NMA0aY03{%;P9U zzD8S;s0AQUs&ndL6(7nkU%55lXVcmqAv9vP{iu2MB`_cDhq7}PS2r8Kj}c*Wzm z9FE_S4@Z99DNad`nLY*?-_b;7j6Swi=yt4LwM28G`he~v@Vxq zMCfE#sK+E0&_Y}C9Rg`5nNn4XF6`gE@UidxgKbj_``5yDGC%jZdec8Xn>wBRw@O@O z?f;k%m_e9899kiqKYdY5*7d_FkEYoMUp~Uc9nkn#twNDspjRB^Q6+4JP5(s_wzkW| zxGUm$@oSmQB$H43Yc1^n*5uRV{)mB+13SvN9oH4cyBiB_;C|`Le5?-k_A7wIqx(b+jbj+dm@|z z6Wi)W(Thv(E&G=4{Z*SDe#k$DhuqN)japR_uP9I{Gv!)D3t(=Sb6cYHGnV)m=4mDn zU5J6Ia{y|uawU<7mck;pQ#moGrw|WEAlR{t zt#u0VE>i0L<{1{MF>3DzkCEe|A6N89t(ju}S32DpJG(W0YZlnMME?V|m-a(+0t~{E5&EORcujEzV@WTwT$XKSvgIWB{6R3C!}RA?sx`ZuB^d zVr^qL9jkVT5O@qwXrB=jBJs#kXz#(oZ6)qY@^Ut`J`{4#zx}s?cb4863GuXc!p9F8 z!#ZrKII||bW8d>>>5LJY8i+L>8+fKFWkOzzV@*Wc2kX%oE`G&DTG(8hNJ;J!y4Z)V zi0J$E%-r{j!?dWLv}zrHx@K35h2r_?RL^g723rncjJr$n%~TY~ojX{g>8DT>D6}I@ zksi?+u1w@gU|9hNBuNY`F4rSLp;KWv>cVMRZ8O5HMkdKi&48gSL{AO435&}4CWzH) zvVrKria?L02ZJh;2dh+?%$qRKD{h zeC|=p^X85bhz0I$A_py-Mqd~@|I53_W75w}nI}M95w~|pZlT(+wf`ko7dYiPpX%wK zld&nCDLQg$v_aPdzBO^G4;iO_%A^L zkoahnGE=M3DKa(ctO}l`=o==3G5-k?4nTA<5MVMUG2w;k%nQ_ptz?*~h-=(F)RGWQ zLSbg6LdU_D^f)3MqZ%+=Xq~kU#fObhWETF#gSML_1{nmV=o{mx$zM(!6j@!xWH;m8 zTcv3d!DL#l9@<`>%6g71O>6MBwOMuTBIg%*R{zCHS|GQU`8pJSsM@dTIEjoxcLriE=zfR%XWGiPE&b*X~YkrX;4Kv~s0&AKV@3IbKo(~fk92v6)TiCbjk0?MnL(MS} zmZO+(xl*-BW@Tu?BiQf=J%_|GQBm>&3v59J4B<@!R5SoPL;pr%)D2N9Y-itTEf(<; z0O(=4wh4)b@GgQh&}J!bI-oWnWY*bCeY2}yxyLtK{cu``4tBP?#c4uy|5)sPQvC2M zx475h@-z1uv4q<^vrYJ?#C}JgO4Kv0a#rQZg)^EqVurs#NRJU~x)Bp#7CaeAGB}0? z3tiZVy=Vf^Hhi8A#DpJ&xPx#V7m}r4eHCMlN59#fc57N<{qxglKRjs@Zc|Y3KSe7S z`7!;FwTW*&6}&(aE(~`z`pF!)b7P6i%(yFy#=0-qQ}OR8WHpiu4~E8@dKDYyvCV?L?S+||S;mk&8vMuT)u#XPoSkmzfhi_yl>W>7N&!-cS6=fI> znutI^GzVWb;V^C_Fj1L58L}>DvBpl&dWOTm_h({Dd*{2D;Tlm~979^^_#`$=B`?tR zA?%)3wafz774bJK+u8l>srFCweD3qMZvfkf^Y;vP-_*ma{LMUDTCR9CvMU2o!iSXc zH#reHqO$no&{-!W*S02-6OAYc;q#czL!e?yONO-}w>^T?Dhb>q!&Mv-iK zupX$SkZZv{GNl8M7%xVWIH9C7=5%V$E=Hz_4I(;&Cxa94Ha?vXb3a2!gAqn+Ii8@X zTH8`jY+>vD50)-L!a<0xMD5ZO^m3JA659qECdPA&z5fd!nuHQ{W^@M+$avRsR%#0M zVIwjXimVL2m3=@ua-y3dN_4H-h8BwL$zQ0QBI~}4j)YkGTS{XQk4}fnV=Dk7jC5>Gtt@O=D={66OlTF{1f7jQh#TofJ7dTH9-SkmJrVLrQfJzkQ4efu z<(o<95O4|B!6OejG!iiQ3>OV@rJ)=P!-+)18V1bn=5vA}i_~cK#~{>p^QFdxX1@vq zzKBY&11k`MEN2X;!}2it#p8TuFyEEV!WPi}bGyF-o;iOSn?D%p80J3xw^;0-?@7^!g?H>cY5V6sWRPZ z)`dOqXN5X%XvKvn$%pk`H6?9c(01+EuQ^FOCFUW%-`db_)}UnO_g(%^gIGIiN61WX zp~EYS>h^k6kCWZE?T{9CU9jH|-+Q-PGs=OAn-l#8ti8FX^Bnwj$moO8jZasf%X(&V zqK{mg6$|D|xV%u6?p;UzAdD77(Ur0#lM`1?cRb!%boV7%h@B#1R$8;~?Q5CxRGDqY*^$z%^9Rh# z+wM5m(d0XH&+~r+MxS~+b4ibm58kDjoypFY(7v!z<6)eo^_l@0Sv}Wx8(Y?Q$!Ff@ zdL;BiSSYR|=i+Y4mshTGzwF_X_GtBxa%)n37W{f4kItFElq!TZ|waIy+rI_)D2@tMOKbWGeKPFpH{TX_4TpckK?u_e5%~9 z`ih%yszn;tP4M-9pZyRX4+$HCjYDX_b1ROB_WM^J#J77bcMfI9v47#7p z{n6uw3%)xPTun)JHJ@X(4Vh~|7W>sQd#o>*oEMx*nmOU^zTcOt`<3d3g8=d^TrpY~ zaV{j&HG8Vw!q3tI_Bf(7D3NC$HZfyTrA7Wx$KzMi;N^rr;^@K~w!^?nGwJ zn}7RF=)=`zXDh?!O+kJKfB_J82v!~TH7Q9FlKW!14D_hM;51m4Y_yoZLa0qh8~EJs zL3Ia~w>vN-_f}E*qy1NFA_oM+#5R3g;$uGOQevl?7xQ)xAKM5|0Fr_2S5Gb-+-|Mh zvu5ql>qkCZ+2MQ=6`C5L4L32hJdh7lqFR^6uN#~{(0T5P*E^-EHNO@>_uXE%UK-G- zS*^O$Nw?PJ%iQH+8iTmF5+-Hr#W9<*^P38Gr6%XgR($Q)Urc)-MEi@uAH5rG5;Eo2 z%Fll_9+}d2TG`%)VT<5p?ss|HKR#cW9j^~Ke!FhVb$9|W6R(50*j4vQzH8TezwIyD z_Bxd1+YQ;+5^grwITIY)=~`7DC|RsKwDRQE1<3=FD&b9yxy1o@Z}mKLd~tT?dE1Ya z=vn9z>5VWfBi?Z4-y5lHC9K*@zC8e|D_Mwq=Cr-5@6)^2(v)n6-8Yg}9(vkM09$Hv$IB-3+uJ?3cDn3>iDW>n@6DZo4ebn0YQM5u|tkLGhD>465fjTHI^O!wXVADn@rjB4PSg- z$(V42eszN20gk^;Jj+Fc`@c{PjhAE`yVdlmcp3UhdUXnyR+-6G9oAZAO!jh%SV9<*+AW5<+5=1}L@qLb>V_VXv4Q#<#t*Y5tt}{3oG#FRIrc z3x9nSBG;sBcg6~F(4`ea=M6{?e)#?!GLR-{4#@z*?xHvtq2*o0Gk|(7kh-=H#vDue z@cM-Fjx6WJoP{%Bez-rX=Dr^n(4%sqHB%cz@s3Z8?fcq@5^&OSN#fm3ng<|N3SPDSOb+Lm`46;SpbJ>BKD z_!`7DghV;K%>00fMZvj}dDYBK^EcZb ztVlE;4L#$B4!N`F*0B#eZC-zvv@C6&r^v_yO?cUj_&YD26nl-BK2_OqOp!>3h;v9T z^{OK`Nljgw@UG4Q%UsS65bzoc-uN})8AIXtcZ+QoANY9R|31dR-Ng8GDrUtzeN2W8%D4XN-xxflw|() zv1TtwfIVn2aHm5KA9*1hBBfuDowPd^7;|yyC<2yaS-b*@O$v2pKPlB>8JrY1>veN7 zKZ2g8_RuXPGAej>JO_`*9tDLVk2=wX0t;ym!Zn)O)DxsuKIx2#5HWsKYQ+o>DJYCm z=4gjm(-({-ThyBI!Te_RWrnWMz{LK&KN34>DPmg`SrZ>sBL)=QXW}UJ;Yy7einN#s z4$Ci8m0|js+r`*7hTX}7UBG|O1qJB961)E-=xEWbPIPJ*TLF3QIZel*9lMCSaav55H4V4ZcAG6pfr8-Q#s%LCge4$#s z*s+T;KT2vTo!MLpfg2d+66$-kjv*ut6sWlPKK#&_Zf9e4rxg^m9 zfvvPJ%QMclC<2T}jjAL^qt*pyWht@}HBd2p=3*)8FEm(-%`V}2F5{0JBGqFNo@ez~@>eDJSq=rhT|R{zd`MJab!X?2bI5h6 zOXH=Kx?VygH}F3Qq;I-Q_XOAt7X7{Q_{+5g6-ocebD#X94 zq@9*uk$#w}t*txVe=QnsOT)mI@c}w|+Ohm?FaHIZS+dXVb5v^&OzX57$hrsmToW_y zS^7(_;JEQ$=MMGz#-XW5hSvihz4Z17>4bL=sg}v&`Zr#z7-SDQjfC#+ZpXr58hE7} zktCO4Q-uhyt*<>0=*o6|^fIAv!|e39`<~BBj|FV(1v7u>;*CURTDi08?2Zg(mt#|^ zVLxH075*ZBUEZ&JyD_xyl+CjTnVopH3y_<(u?C%Yi!4m?zvGtuTqp0B`1^mS7A>6! zgg11Bm2Xy>K)9(Pta}xi7{%cmjGJcmT<&mh+=K_~Z;l%OZeeUKZ}NRMGhUIS7*TN3 z1}rH<^@*h1ZZ%=}n@LMbMnxC6YaPeJ3)k;68E13g(%%#B4mV5Fei(9}HKq$vU7Scu zLWe=ld$2w2F0iz3F{dT!Ll|4k*^^ACAWP8+k=zpOOD%={qUgexhr@{BirA8Pq zK)v`1!Sf~5jIm`q2tkh+YQ*CC1k?z%V4TINq@~CMUnlG(LI;|g8!k^oY|H2a5DO9u z%5LWDQJ!cLVvM{go=t?uz&2(a;0TbF)50q(A8xk9=4>CKXa?dNImOB>QskMSP;gYW z%o>}HTn8^-ZR7ZB)ot&5-Q>{ax2PU}vi>4oz1Fn;(Syq$BQoryv&?W0gcYC)UzW8W zca<}s?$o~3SLH>q%wir}-IX3SlZpOe_{P-!v>AI*$bZz8I<?bd89{I6*Qm(eN-2cMXV-+BUnsA2> zGj6D^4J*8UUxFq)>|NZY@sn~lJZO6L_n@&J$;F6+GQ>LaX zsNmELVRx@#U6V6CO49W%|A0?fMsG?IL`5mM znbtcP5T)B`w{v?$w#b1^awb2re?GWA?XlgFbHWM zEJY(3x)%O4LRczELGVype3?fhuoE1%oMA+ZgmEQ0C|Y$#B3*JVbB@fLKG#qGzn*CYSyb;}cM=4`V4M-13zsGKz5%o8D#Km{MV>LYtt1LL(77v*!eVfQ>x|xn5U%r#$%pnq!1pn~# zW`OK&6OHwtIl}V;RwWIJkA0bP=0tC%0rx)ieZOmUv3TU+m!kY8-qSh^{WKKnC2n49 zBDNnew|i%HAKxIM4C0~>!`kq1hJ>4F3)4zHcou9b22u4#$(vxn6NGblu$C4qwEoz? za$kN=Bwc5V?GRcBtav53(C4*s#KOTA5zZG&x;g!ndH|1emyWr-)6F9?vrWnT>oM(j z@%Zp=VvoUA;WpbUYjt(LF5EHmm>|uBtihp^(Ha}DRVcS73~*Dn`DE*?ii^EpPA;vU zQj#gZ{ui*>2+uA*PE~B~et!HDmxwc0NBH%j>BtOk1_x?;NBj7|UZ%e`cm?+)`;Xs8 z{gE9>3q|q3He8uiw=f~y0zY~TMR+Tc2*Kgza=kNfAZ8Bw(?SaX>;IQ4-&wYM^Z!bE z2MECX%pcrm$BDTB{X4AA6q#2 zCl=se15TGn^}Qr3>cenwv%a_P(qGw1Xa(TXa>9Tyob;$dm0tOdSWqv9Lt08lulU^c z*ds=tyMfT}*q(jW91_jge++TR&uq$vI$kjpR*aoA9NH%oGakrDJ?vJN+X-vnCnU8_ zzSG~1cHnb_Mn<6)d=9D*QHD6u1i}#+DV7Ej4)JQw2KAyd1TuDxl$(hqMpmJTn1g=~ zgJ0Nm1sXMnZ$)$QpW2S2F!PdKoj^ZA&v~{e**Z_TebKai9hUijt*k>*g_Bg!$;hX*2Cf^eubFp+-yZu)(?TWA#eZ{3~O_y|V zuSk1%^-lcnB5FGT$@G<8Uo0<_k)ug^+sRX^!=)ArtEVk__C1wWu62&84{1O5W$=pydeKkva!Tswv%GI9yN4WS(7%J0w}Pll zL7&+#GuJA6EwQR-bIPXQY_A@#ky*Y{oi87e4%O>2Tq{;)jo>f=i>=b@c7EEqH}$k# zq5n-JDS%&}l)d@2E}h}uM^3O+c1c-$Bk#?NfvV>iAT$cR><@3KuhWpPvmL#83`$Lc z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOB(Jyhd}l+lM_DLQ|R1S9JggWMOn+E@EkJ+)pEt zS0B@B+5na71A~^(h?oKJe_7^kUF)~R705&JjF@Ba(+MsB@NR8#YcsGwevk!s&HdgP zC{Zt|h7q9Z02OU%Zf#+BUuAY>ZggK{VRU6KX=Hc=VZjEVxwC1aJrFfB;*n}Y01a?q za&&2QX>V>WWMOm!K^bOwf>Qhj>E&4HXupM*01I$(X>DP0c`j*WcnnTemQio;UGet* Kde0cxq6Ywpes2{3 diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/columns.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/columns.txt deleted file mode 100644 index 4835061786d..00000000000 --- a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/columns.txt +++ /dev/null @@ -1,259 +0,0 @@ -columns format version: 1 -257 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`milliseconds` UInt64 -`ProfileEvent_Query` UInt64 -`ProfileEvent_SelectQuery` UInt64 -`ProfileEvent_InsertQuery` UInt64 -`ProfileEvent_FailedQuery` UInt64 -`ProfileEvent_FailedSelectQuery` UInt64 -`ProfileEvent_FailedInsertQuery` UInt64 -`ProfileEvent_QueryTimeMicroseconds` UInt64 -`ProfileEvent_SelectQueryTimeMicroseconds` UInt64 -`ProfileEvent_InsertQueryTimeMicroseconds` UInt64 -`ProfileEvent_FileOpen` UInt64 -`ProfileEvent_Seek` UInt64 -`ProfileEvent_ReadBufferFromFileDescriptorRead` UInt64 -`ProfileEvent_ReadBufferFromFileDescriptorReadFailed` UInt64 -`ProfileEvent_ReadBufferFromFileDescriptorReadBytes` UInt64 -`ProfileEvent_WriteBufferFromFileDescriptorWrite` UInt64 -`ProfileEvent_WriteBufferFromFileDescriptorWriteFailed` UInt64 -`ProfileEvent_WriteBufferFromFileDescriptorWriteBytes` UInt64 -`ProfileEvent_ReadBufferAIORead` UInt64 -`ProfileEvent_ReadBufferAIOReadBytes` UInt64 -`ProfileEvent_WriteBufferAIOWrite` UInt64 -`ProfileEvent_WriteBufferAIOWriteBytes` UInt64 -`ProfileEvent_ReadCompressedBytes` UInt64 -`ProfileEvent_CompressedReadBufferBlocks` UInt64 -`ProfileEvent_CompressedReadBufferBytes` UInt64 -`ProfileEvent_UncompressedCacheHits` UInt64 -`ProfileEvent_UncompressedCacheMisses` UInt64 -`ProfileEvent_UncompressedCacheWeightLost` UInt64 -`ProfileEvent_IOBufferAllocs` UInt64 -`ProfileEvent_IOBufferAllocBytes` UInt64 -`ProfileEvent_ArenaAllocChunks` UInt64 -`ProfileEvent_ArenaAllocBytes` UInt64 -`ProfileEvent_FunctionExecute` UInt64 -`ProfileEvent_TableFunctionExecute` UInt64 -`ProfileEvent_MarkCacheHits` UInt64 -`ProfileEvent_MarkCacheMisses` UInt64 -`ProfileEvent_CreatedReadBufferOrdinary` UInt64 -`ProfileEvent_CreatedReadBufferAIO` UInt64 -`ProfileEvent_CreatedReadBufferAIOFailed` UInt64 -`ProfileEvent_CreatedReadBufferMMap` UInt64 -`ProfileEvent_CreatedReadBufferMMapFailed` UInt64 -`ProfileEvent_CreatedWriteBufferOrdinary` UInt64 -`ProfileEvent_CreatedWriteBufferAIO` UInt64 -`ProfileEvent_CreatedWriteBufferAIOFailed` UInt64 -`ProfileEvent_DiskReadElapsedMicroseconds` UInt64 -`ProfileEvent_DiskWriteElapsedMicroseconds` UInt64 -`ProfileEvent_NetworkReceiveElapsedMicroseconds` UInt64 -`ProfileEvent_NetworkSendElapsedMicroseconds` UInt64 -`ProfileEvent_ThrottlerSleepMicroseconds` UInt64 -`ProfileEvent_QueryMaskingRulesMatch` UInt64 -`ProfileEvent_ReplicatedPartFetches` UInt64 -`ProfileEvent_ReplicatedPartFailedFetches` UInt64 -`ProfileEvent_ObsoleteReplicatedParts` UInt64 -`ProfileEvent_ReplicatedPartMerges` UInt64 -`ProfileEvent_ReplicatedPartFetchesOfMerged` UInt64 -`ProfileEvent_ReplicatedPartMutations` UInt64 -`ProfileEvent_ReplicatedPartChecks` UInt64 -`ProfileEvent_ReplicatedPartChecksFailed` UInt64 -`ProfileEvent_ReplicatedDataLoss` UInt64 -`ProfileEvent_InsertedRows` UInt64 -`ProfileEvent_InsertedBytes` UInt64 -`ProfileEvent_DelayedInserts` UInt64 -`ProfileEvent_RejectedInserts` UInt64 -`ProfileEvent_DelayedInsertsMilliseconds` UInt64 -`ProfileEvent_DuplicatedInsertedBlocks` UInt64 -`ProfileEvent_ZooKeeperInit` UInt64 -`ProfileEvent_ZooKeeperTransactions` UInt64 -`ProfileEvent_ZooKeeperList` UInt64 -`ProfileEvent_ZooKeeperCreate` UInt64 -`ProfileEvent_ZooKeeperRemove` UInt64 -`ProfileEvent_ZooKeeperExists` UInt64 -`ProfileEvent_ZooKeeperGet` UInt64 -`ProfileEvent_ZooKeeperSet` UInt64 -`ProfileEvent_ZooKeeperMulti` UInt64 -`ProfileEvent_ZooKeeperCheck` UInt64 -`ProfileEvent_ZooKeeperClose` UInt64 -`ProfileEvent_ZooKeeperWatchResponse` UInt64 -`ProfileEvent_ZooKeeperUserExceptions` UInt64 -`ProfileEvent_ZooKeeperHardwareExceptions` UInt64 -`ProfileEvent_ZooKeeperOtherExceptions` UInt64 -`ProfileEvent_ZooKeeperWaitMicroseconds` UInt64 -`ProfileEvent_ZooKeeperBytesSent` UInt64 -`ProfileEvent_ZooKeeperBytesReceived` UInt64 -`ProfileEvent_DistributedConnectionFailTry` UInt64 -`ProfileEvent_DistributedConnectionMissingTable` UInt64 -`ProfileEvent_DistributedConnectionStaleReplica` UInt64 -`ProfileEvent_DistributedConnectionFailAtAll` UInt64 -`ProfileEvent_CompileAttempt` UInt64 -`ProfileEvent_CompileSuccess` UInt64 -`ProfileEvent_CompileFunction` UInt64 -`ProfileEvent_CompiledFunctionExecute` UInt64 -`ProfileEvent_CompileExpressionsMicroseconds` UInt64 -`ProfileEvent_CompileExpressionsBytes` UInt64 -`ProfileEvent_ExternalSortWritePart` UInt64 -`ProfileEvent_ExternalSortMerge` UInt64 -`ProfileEvent_ExternalAggregationWritePart` UInt64 -`ProfileEvent_ExternalAggregationMerge` UInt64 -`ProfileEvent_ExternalAggregationCompressedBytes` UInt64 -`ProfileEvent_ExternalAggregationUncompressedBytes` UInt64 -`ProfileEvent_SlowRead` UInt64 -`ProfileEvent_ReadBackoff` UInt64 -`ProfileEvent_ReplicaPartialShutdown` UInt64 -`ProfileEvent_SelectedParts` UInt64 -`ProfileEvent_SelectedRanges` UInt64 -`ProfileEvent_SelectedMarks` UInt64 -`ProfileEvent_SelectedRows` UInt64 -`ProfileEvent_SelectedBytes` UInt64 -`ProfileEvent_Merge` UInt64 -`ProfileEvent_MergedRows` UInt64 -`ProfileEvent_MergedUncompressedBytes` UInt64 -`ProfileEvent_MergesTimeMilliseconds` UInt64 -`ProfileEvent_MergeTreeDataWriterRows` UInt64 -`ProfileEvent_MergeTreeDataWriterUncompressedBytes` UInt64 -`ProfileEvent_MergeTreeDataWriterCompressedBytes` UInt64 -`ProfileEvent_MergeTreeDataWriterBlocks` UInt64 -`ProfileEvent_MergeTreeDataWriterBlocksAlreadySorted` UInt64 -`ProfileEvent_CannotRemoveEphemeralNode` UInt64 -`ProfileEvent_RegexpCreated` UInt64 -`ProfileEvent_ContextLock` UInt64 -`ProfileEvent_StorageBufferFlush` UInt64 -`ProfileEvent_StorageBufferErrorOnFlush` UInt64 -`ProfileEvent_StorageBufferPassedAllMinThresholds` UInt64 -`ProfileEvent_StorageBufferPassedTimeMaxThreshold` UInt64 -`ProfileEvent_StorageBufferPassedRowsMaxThreshold` UInt64 -`ProfileEvent_StorageBufferPassedBytesMaxThreshold` UInt64 -`ProfileEvent_DictCacheKeysRequested` UInt64 -`ProfileEvent_DictCacheKeysRequestedMiss` UInt64 -`ProfileEvent_DictCacheKeysRequestedFound` UInt64 -`ProfileEvent_DictCacheKeysExpired` UInt64 -`ProfileEvent_DictCacheKeysNotFound` UInt64 -`ProfileEvent_DictCacheKeysHit` UInt64 -`ProfileEvent_DictCacheRequestTimeNs` UInt64 -`ProfileEvent_DictCacheRequests` UInt64 -`ProfileEvent_DictCacheLockWriteNs` UInt64 -`ProfileEvent_DictCacheLockReadNs` UInt64 -`ProfileEvent_DistributedSyncInsertionTimeoutExceeded` UInt64 -`ProfileEvent_DataAfterMergeDiffersFromReplica` UInt64 -`ProfileEvent_DataAfterMutationDiffersFromReplica` UInt64 -`ProfileEvent_PolygonsAddedToPool` UInt64 -`ProfileEvent_PolygonsInPoolAllocatedBytes` UInt64 -`ProfileEvent_RWLockAcquiredReadLocks` UInt64 -`ProfileEvent_RWLockAcquiredWriteLocks` UInt64 -`ProfileEvent_RWLockReadersWaitMilliseconds` UInt64 -`ProfileEvent_RWLockWritersWaitMilliseconds` UInt64 -`ProfileEvent_DNSError` UInt64 -`ProfileEvent_RealTimeMicroseconds` UInt64 -`ProfileEvent_UserTimeMicroseconds` UInt64 -`ProfileEvent_SystemTimeMicroseconds` UInt64 -`ProfileEvent_SoftPageFaults` UInt64 -`ProfileEvent_HardPageFaults` UInt64 -`ProfileEvent_VoluntaryContextSwitches` UInt64 -`ProfileEvent_InvoluntaryContextSwitches` UInt64 -`ProfileEvent_OSIOWaitMicroseconds` UInt64 -`ProfileEvent_OSCPUWaitMicroseconds` UInt64 -`ProfileEvent_OSCPUVirtualTimeMicroseconds` UInt64 -`ProfileEvent_OSReadBytes` UInt64 -`ProfileEvent_OSWriteBytes` UInt64 -`ProfileEvent_OSReadChars` UInt64 -`ProfileEvent_OSWriteChars` UInt64 -`ProfileEvent_PerfCpuCycles` UInt64 -`ProfileEvent_PerfInstructions` UInt64 -`ProfileEvent_PerfCacheReferences` UInt64 -`ProfileEvent_PerfCacheMisses` UInt64 -`ProfileEvent_PerfBranchInstructions` UInt64 -`ProfileEvent_PerfBranchMisses` UInt64 -`ProfileEvent_PerfBusCycles` UInt64 -`ProfileEvent_PerfStalledCyclesFrontend` UInt64 -`ProfileEvent_PerfStalledCyclesBackend` UInt64 -`ProfileEvent_PerfRefCpuCycles` UInt64 -`ProfileEvent_PerfCpuClock` UInt64 -`ProfileEvent_PerfTaskClock` UInt64 -`ProfileEvent_PerfContextSwitches` UInt64 -`ProfileEvent_PerfCpuMigrations` UInt64 -`ProfileEvent_PerfAlignmentFaults` UInt64 -`ProfileEvent_PerfEmulationFaults` UInt64 -`ProfileEvent_PerfMinEnabledTime` UInt64 -`ProfileEvent_PerfMinEnabledRunningTime` UInt64 -`ProfileEvent_PerfDataTLBReferences` UInt64 -`ProfileEvent_PerfDataTLBMisses` UInt64 -`ProfileEvent_PerfInstructionTLBReferences` UInt64 -`ProfileEvent_PerfInstructionTLBMisses` UInt64 -`ProfileEvent_PerfLocalMemoryReferences` UInt64 -`ProfileEvent_PerfLocalMemoryMisses` UInt64 -`ProfileEvent_CreatedHTTPConnections` UInt64 -`ProfileEvent_CannotWriteToWriteBufferDiscard` UInt64 -`ProfileEvent_QueryProfilerSignalOverruns` UInt64 -`ProfileEvent_CreatedLogEntryForMerge` UInt64 -`ProfileEvent_NotCreatedLogEntryForMerge` UInt64 -`ProfileEvent_CreatedLogEntryForMutation` UInt64 -`ProfileEvent_NotCreatedLogEntryForMutation` UInt64 -`ProfileEvent_S3ReadMicroseconds` UInt64 -`ProfileEvent_S3ReadBytes` UInt64 -`ProfileEvent_S3ReadRequestsCount` UInt64 -`ProfileEvent_S3ReadRequestsErrors` UInt64 -`ProfileEvent_S3ReadRequestsThrottling` UInt64 -`ProfileEvent_S3ReadRequestsRedirects` UInt64 -`ProfileEvent_S3WriteMicroseconds` UInt64 -`ProfileEvent_S3WriteBytes` UInt64 -`ProfileEvent_S3WriteRequestsCount` UInt64 -`ProfileEvent_S3WriteRequestsErrors` UInt64 -`ProfileEvent_S3WriteRequestsThrottling` UInt64 -`ProfileEvent_S3WriteRequestsRedirects` UInt64 -`ProfileEvent_QueryMemoryLimitExceeded` UInt64 -`CurrentMetric_Query` Int64 -`CurrentMetric_Merge` Int64 -`CurrentMetric_PartMutation` Int64 -`CurrentMetric_ReplicatedFetch` Int64 -`CurrentMetric_ReplicatedSend` Int64 -`CurrentMetric_ReplicatedChecks` Int64 -`CurrentMetric_BackgroundPoolTask` Int64 -`CurrentMetric_BackgroundMovePoolTask` Int64 -`CurrentMetric_BackgroundSchedulePoolTask` Int64 -`CurrentMetric_BackgroundBufferFlushSchedulePoolTask` Int64 -`CurrentMetric_BackgroundDistributedSchedulePoolTask` Int64 -`CurrentMetric_BackgroundMessageBrokerSchedulePoolTask` Int64 -`CurrentMetric_CacheDictionaryUpdateQueueBatches` Int64 -`CurrentMetric_CacheDictionaryUpdateQueueKeys` Int64 -`CurrentMetric_DiskSpaceReservedForMerge` Int64 -`CurrentMetric_DistributedSend` Int64 -`CurrentMetric_QueryPreempted` Int64 -`CurrentMetric_TCPConnection` Int64 -`CurrentMetric_MySQLConnection` Int64 -`CurrentMetric_HTTPConnection` Int64 -`CurrentMetric_InterserverConnection` Int64 -`CurrentMetric_PostgreSQLConnection` Int64 -`CurrentMetric_OpenFileForRead` Int64 -`CurrentMetric_OpenFileForWrite` Int64 -`CurrentMetric_Read` Int64 -`CurrentMetric_Write` Int64 -`CurrentMetric_SendScalars` Int64 -`CurrentMetric_SendExternalTables` Int64 -`CurrentMetric_QueryThread` Int64 -`CurrentMetric_ReadonlyReplica` Int64 -`CurrentMetric_MemoryTracking` Int64 -`CurrentMetric_EphemeralNode` Int64 -`CurrentMetric_ZooKeeperSession` Int64 -`CurrentMetric_ZooKeeperWatch` Int64 -`CurrentMetric_ZooKeeperRequest` Int64 -`CurrentMetric_DelayedInserts` Int64 -`CurrentMetric_ContextLockWait` Int64 -`CurrentMetric_StorageBufferRows` Int64 -`CurrentMetric_StorageBufferBytes` Int64 -`CurrentMetric_DictCacheRequests` Int64 -`CurrentMetric_Revision` Int64 -`CurrentMetric_VersionInteger` Int64 -`CurrentMetric_RWLockWaitingReaders` Int64 -`CurrentMetric_RWLockWaitingWriters` Int64 -`CurrentMetric_RWLockActiveReaders` Int64 -`CurrentMetric_RWLockActiveWriters` Int64 -`CurrentMetric_GlobalThread` Int64 -`CurrentMetric_GlobalThreadActive` Int64 -`CurrentMetric_LocalThread` Int64 -`CurrentMetric_LocalThreadActive` Int64 -`CurrentMetric_DistributedFilesToInsert` Int64 diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/count.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/count.txt deleted file mode 100644 index 301160a9306..00000000000 --- a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/count.txt +++ /dev/null @@ -1 +0,0 @@ -8 \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/default_compression_codec.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/minmax_event_date.idx b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/minmax_event_date.idx deleted file mode 100644 index 16cc6680505..00000000000 --- a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/partition.dat b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/partition.dat deleted file mode 100644 index da540c2d1a825be3ef80d13baa834834b2248491..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmXRDWo7^X0wDm8 diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/primary.idx b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/primary.idx deleted file mode 100644 index 3709b3e4f82..00000000000 --- a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4607_4607_0/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H"`H"` \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/checksums.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/checksums.txt deleted file mode 100644 index df2f868f3e238b0545f87bcd10382a47e43e9b21..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 260 zcmV+f0sH=AXk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$c3z@`!4L&OV%^7r2tD!?&< z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOEL%%`Kt3xZ#Bpu>(3pY7Tm^@j&dUd`w1=p@NR8#YcsGwZggK{VRU6KX=Hc=VZjEVxwC1aJrFfB;*n}Y01a?q za&&2QX>V>WWMOm!K^bOwf>Qhj>E&4HXupM*01I$(X>DP0c`j*Wcnsy^i!!b(-JKN- K=7_>$hb#bNOKuqe diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/columns.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/columns.txt deleted file mode 100644 index 4835061786d..00000000000 --- a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/columns.txt +++ /dev/null @@ -1,259 +0,0 @@ -columns format version: 1 -257 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`milliseconds` UInt64 -`ProfileEvent_Query` UInt64 -`ProfileEvent_SelectQuery` UInt64 -`ProfileEvent_InsertQuery` UInt64 -`ProfileEvent_FailedQuery` UInt64 -`ProfileEvent_FailedSelectQuery` UInt64 -`ProfileEvent_FailedInsertQuery` UInt64 -`ProfileEvent_QueryTimeMicroseconds` UInt64 -`ProfileEvent_SelectQueryTimeMicroseconds` UInt64 -`ProfileEvent_InsertQueryTimeMicroseconds` UInt64 -`ProfileEvent_FileOpen` UInt64 -`ProfileEvent_Seek` UInt64 -`ProfileEvent_ReadBufferFromFileDescriptorRead` UInt64 -`ProfileEvent_ReadBufferFromFileDescriptorReadFailed` UInt64 -`ProfileEvent_ReadBufferFromFileDescriptorReadBytes` UInt64 -`ProfileEvent_WriteBufferFromFileDescriptorWrite` UInt64 -`ProfileEvent_WriteBufferFromFileDescriptorWriteFailed` UInt64 -`ProfileEvent_WriteBufferFromFileDescriptorWriteBytes` UInt64 -`ProfileEvent_ReadBufferAIORead` UInt64 -`ProfileEvent_ReadBufferAIOReadBytes` UInt64 -`ProfileEvent_WriteBufferAIOWrite` UInt64 -`ProfileEvent_WriteBufferAIOWriteBytes` UInt64 -`ProfileEvent_ReadCompressedBytes` UInt64 -`ProfileEvent_CompressedReadBufferBlocks` UInt64 -`ProfileEvent_CompressedReadBufferBytes` UInt64 -`ProfileEvent_UncompressedCacheHits` UInt64 -`ProfileEvent_UncompressedCacheMisses` UInt64 -`ProfileEvent_UncompressedCacheWeightLost` UInt64 -`ProfileEvent_IOBufferAllocs` UInt64 -`ProfileEvent_IOBufferAllocBytes` UInt64 -`ProfileEvent_ArenaAllocChunks` UInt64 -`ProfileEvent_ArenaAllocBytes` UInt64 -`ProfileEvent_FunctionExecute` UInt64 -`ProfileEvent_TableFunctionExecute` UInt64 -`ProfileEvent_MarkCacheHits` UInt64 -`ProfileEvent_MarkCacheMisses` UInt64 -`ProfileEvent_CreatedReadBufferOrdinary` UInt64 -`ProfileEvent_CreatedReadBufferAIO` UInt64 -`ProfileEvent_CreatedReadBufferAIOFailed` UInt64 -`ProfileEvent_CreatedReadBufferMMap` UInt64 -`ProfileEvent_CreatedReadBufferMMapFailed` UInt64 -`ProfileEvent_CreatedWriteBufferOrdinary` UInt64 -`ProfileEvent_CreatedWriteBufferAIO` UInt64 -`ProfileEvent_CreatedWriteBufferAIOFailed` UInt64 -`ProfileEvent_DiskReadElapsedMicroseconds` UInt64 -`ProfileEvent_DiskWriteElapsedMicroseconds` UInt64 -`ProfileEvent_NetworkReceiveElapsedMicroseconds` UInt64 -`ProfileEvent_NetworkSendElapsedMicroseconds` UInt64 -`ProfileEvent_ThrottlerSleepMicroseconds` UInt64 -`ProfileEvent_QueryMaskingRulesMatch` UInt64 -`ProfileEvent_ReplicatedPartFetches` UInt64 -`ProfileEvent_ReplicatedPartFailedFetches` UInt64 -`ProfileEvent_ObsoleteReplicatedParts` UInt64 -`ProfileEvent_ReplicatedPartMerges` UInt64 -`ProfileEvent_ReplicatedPartFetchesOfMerged` UInt64 -`ProfileEvent_ReplicatedPartMutations` UInt64 -`ProfileEvent_ReplicatedPartChecks` UInt64 -`ProfileEvent_ReplicatedPartChecksFailed` UInt64 -`ProfileEvent_ReplicatedDataLoss` UInt64 -`ProfileEvent_InsertedRows` UInt64 -`ProfileEvent_InsertedBytes` UInt64 -`ProfileEvent_DelayedInserts` UInt64 -`ProfileEvent_RejectedInserts` UInt64 -`ProfileEvent_DelayedInsertsMilliseconds` UInt64 -`ProfileEvent_DuplicatedInsertedBlocks` UInt64 -`ProfileEvent_ZooKeeperInit` UInt64 -`ProfileEvent_ZooKeeperTransactions` UInt64 -`ProfileEvent_ZooKeeperList` UInt64 -`ProfileEvent_ZooKeeperCreate` UInt64 -`ProfileEvent_ZooKeeperRemove` UInt64 -`ProfileEvent_ZooKeeperExists` UInt64 -`ProfileEvent_ZooKeeperGet` UInt64 -`ProfileEvent_ZooKeeperSet` UInt64 -`ProfileEvent_ZooKeeperMulti` UInt64 -`ProfileEvent_ZooKeeperCheck` UInt64 -`ProfileEvent_ZooKeeperClose` UInt64 -`ProfileEvent_ZooKeeperWatchResponse` UInt64 -`ProfileEvent_ZooKeeperUserExceptions` UInt64 -`ProfileEvent_ZooKeeperHardwareExceptions` UInt64 -`ProfileEvent_ZooKeeperOtherExceptions` UInt64 -`ProfileEvent_ZooKeeperWaitMicroseconds` UInt64 -`ProfileEvent_ZooKeeperBytesSent` UInt64 -`ProfileEvent_ZooKeeperBytesReceived` UInt64 -`ProfileEvent_DistributedConnectionFailTry` UInt64 -`ProfileEvent_DistributedConnectionMissingTable` UInt64 -`ProfileEvent_DistributedConnectionStaleReplica` UInt64 -`ProfileEvent_DistributedConnectionFailAtAll` UInt64 -`ProfileEvent_CompileAttempt` UInt64 -`ProfileEvent_CompileSuccess` UInt64 -`ProfileEvent_CompileFunction` UInt64 -`ProfileEvent_CompiledFunctionExecute` UInt64 -`ProfileEvent_CompileExpressionsMicroseconds` UInt64 -`ProfileEvent_CompileExpressionsBytes` UInt64 -`ProfileEvent_ExternalSortWritePart` UInt64 -`ProfileEvent_ExternalSortMerge` UInt64 -`ProfileEvent_ExternalAggregationWritePart` UInt64 -`ProfileEvent_ExternalAggregationMerge` UInt64 -`ProfileEvent_ExternalAggregationCompressedBytes` UInt64 -`ProfileEvent_ExternalAggregationUncompressedBytes` UInt64 -`ProfileEvent_SlowRead` UInt64 -`ProfileEvent_ReadBackoff` UInt64 -`ProfileEvent_ReplicaPartialShutdown` UInt64 -`ProfileEvent_SelectedParts` UInt64 -`ProfileEvent_SelectedRanges` UInt64 -`ProfileEvent_SelectedMarks` UInt64 -`ProfileEvent_SelectedRows` UInt64 -`ProfileEvent_SelectedBytes` UInt64 -`ProfileEvent_Merge` UInt64 -`ProfileEvent_MergedRows` UInt64 -`ProfileEvent_MergedUncompressedBytes` UInt64 -`ProfileEvent_MergesTimeMilliseconds` UInt64 -`ProfileEvent_MergeTreeDataWriterRows` UInt64 -`ProfileEvent_MergeTreeDataWriterUncompressedBytes` UInt64 -`ProfileEvent_MergeTreeDataWriterCompressedBytes` UInt64 -`ProfileEvent_MergeTreeDataWriterBlocks` UInt64 -`ProfileEvent_MergeTreeDataWriterBlocksAlreadySorted` UInt64 -`ProfileEvent_CannotRemoveEphemeralNode` UInt64 -`ProfileEvent_RegexpCreated` UInt64 -`ProfileEvent_ContextLock` UInt64 -`ProfileEvent_StorageBufferFlush` UInt64 -`ProfileEvent_StorageBufferErrorOnFlush` UInt64 -`ProfileEvent_StorageBufferPassedAllMinThresholds` UInt64 -`ProfileEvent_StorageBufferPassedTimeMaxThreshold` UInt64 -`ProfileEvent_StorageBufferPassedRowsMaxThreshold` UInt64 -`ProfileEvent_StorageBufferPassedBytesMaxThreshold` UInt64 -`ProfileEvent_DictCacheKeysRequested` UInt64 -`ProfileEvent_DictCacheKeysRequestedMiss` UInt64 -`ProfileEvent_DictCacheKeysRequestedFound` UInt64 -`ProfileEvent_DictCacheKeysExpired` UInt64 -`ProfileEvent_DictCacheKeysNotFound` UInt64 -`ProfileEvent_DictCacheKeysHit` UInt64 -`ProfileEvent_DictCacheRequestTimeNs` UInt64 -`ProfileEvent_DictCacheRequests` UInt64 -`ProfileEvent_DictCacheLockWriteNs` UInt64 -`ProfileEvent_DictCacheLockReadNs` UInt64 -`ProfileEvent_DistributedSyncInsertionTimeoutExceeded` UInt64 -`ProfileEvent_DataAfterMergeDiffersFromReplica` UInt64 -`ProfileEvent_DataAfterMutationDiffersFromReplica` UInt64 -`ProfileEvent_PolygonsAddedToPool` UInt64 -`ProfileEvent_PolygonsInPoolAllocatedBytes` UInt64 -`ProfileEvent_RWLockAcquiredReadLocks` UInt64 -`ProfileEvent_RWLockAcquiredWriteLocks` UInt64 -`ProfileEvent_RWLockReadersWaitMilliseconds` UInt64 -`ProfileEvent_RWLockWritersWaitMilliseconds` UInt64 -`ProfileEvent_DNSError` UInt64 -`ProfileEvent_RealTimeMicroseconds` UInt64 -`ProfileEvent_UserTimeMicroseconds` UInt64 -`ProfileEvent_SystemTimeMicroseconds` UInt64 -`ProfileEvent_SoftPageFaults` UInt64 -`ProfileEvent_HardPageFaults` UInt64 -`ProfileEvent_VoluntaryContextSwitches` UInt64 -`ProfileEvent_InvoluntaryContextSwitches` UInt64 -`ProfileEvent_OSIOWaitMicroseconds` UInt64 -`ProfileEvent_OSCPUWaitMicroseconds` UInt64 -`ProfileEvent_OSCPUVirtualTimeMicroseconds` UInt64 -`ProfileEvent_OSReadBytes` UInt64 -`ProfileEvent_OSWriteBytes` UInt64 -`ProfileEvent_OSReadChars` UInt64 -`ProfileEvent_OSWriteChars` UInt64 -`ProfileEvent_PerfCpuCycles` UInt64 -`ProfileEvent_PerfInstructions` UInt64 -`ProfileEvent_PerfCacheReferences` UInt64 -`ProfileEvent_PerfCacheMisses` UInt64 -`ProfileEvent_PerfBranchInstructions` UInt64 -`ProfileEvent_PerfBranchMisses` UInt64 -`ProfileEvent_PerfBusCycles` UInt64 -`ProfileEvent_PerfStalledCyclesFrontend` UInt64 -`ProfileEvent_PerfStalledCyclesBackend` UInt64 -`ProfileEvent_PerfRefCpuCycles` UInt64 -`ProfileEvent_PerfCpuClock` UInt64 -`ProfileEvent_PerfTaskClock` UInt64 -`ProfileEvent_PerfContextSwitches` UInt64 -`ProfileEvent_PerfCpuMigrations` UInt64 -`ProfileEvent_PerfAlignmentFaults` UInt64 -`ProfileEvent_PerfEmulationFaults` UInt64 -`ProfileEvent_PerfMinEnabledTime` UInt64 -`ProfileEvent_PerfMinEnabledRunningTime` UInt64 -`ProfileEvent_PerfDataTLBReferences` UInt64 -`ProfileEvent_PerfDataTLBMisses` UInt64 -`ProfileEvent_PerfInstructionTLBReferences` UInt64 -`ProfileEvent_PerfInstructionTLBMisses` UInt64 -`ProfileEvent_PerfLocalMemoryReferences` UInt64 -`ProfileEvent_PerfLocalMemoryMisses` UInt64 -`ProfileEvent_CreatedHTTPConnections` UInt64 -`ProfileEvent_CannotWriteToWriteBufferDiscard` UInt64 -`ProfileEvent_QueryProfilerSignalOverruns` UInt64 -`ProfileEvent_CreatedLogEntryForMerge` UInt64 -`ProfileEvent_NotCreatedLogEntryForMerge` UInt64 -`ProfileEvent_CreatedLogEntryForMutation` UInt64 -`ProfileEvent_NotCreatedLogEntryForMutation` UInt64 -`ProfileEvent_S3ReadMicroseconds` UInt64 -`ProfileEvent_S3ReadBytes` UInt64 -`ProfileEvent_S3ReadRequestsCount` UInt64 -`ProfileEvent_S3ReadRequestsErrors` UInt64 -`ProfileEvent_S3ReadRequestsThrottling` UInt64 -`ProfileEvent_S3ReadRequestsRedirects` UInt64 -`ProfileEvent_S3WriteMicroseconds` UInt64 -`ProfileEvent_S3WriteBytes` UInt64 -`ProfileEvent_S3WriteRequestsCount` UInt64 -`ProfileEvent_S3WriteRequestsErrors` UInt64 -`ProfileEvent_S3WriteRequestsThrottling` UInt64 -`ProfileEvent_S3WriteRequestsRedirects` UInt64 -`ProfileEvent_QueryMemoryLimitExceeded` UInt64 -`CurrentMetric_Query` Int64 -`CurrentMetric_Merge` Int64 -`CurrentMetric_PartMutation` Int64 -`CurrentMetric_ReplicatedFetch` Int64 -`CurrentMetric_ReplicatedSend` Int64 -`CurrentMetric_ReplicatedChecks` Int64 -`CurrentMetric_BackgroundPoolTask` Int64 -`CurrentMetric_BackgroundMovePoolTask` Int64 -`CurrentMetric_BackgroundSchedulePoolTask` Int64 -`CurrentMetric_BackgroundBufferFlushSchedulePoolTask` Int64 -`CurrentMetric_BackgroundDistributedSchedulePoolTask` Int64 -`CurrentMetric_BackgroundMessageBrokerSchedulePoolTask` Int64 -`CurrentMetric_CacheDictionaryUpdateQueueBatches` Int64 -`CurrentMetric_CacheDictionaryUpdateQueueKeys` Int64 -`CurrentMetric_DiskSpaceReservedForMerge` Int64 -`CurrentMetric_DistributedSend` Int64 -`CurrentMetric_QueryPreempted` Int64 -`CurrentMetric_TCPConnection` Int64 -`CurrentMetric_MySQLConnection` Int64 -`CurrentMetric_HTTPConnection` Int64 -`CurrentMetric_InterserverConnection` Int64 -`CurrentMetric_PostgreSQLConnection` Int64 -`CurrentMetric_OpenFileForRead` Int64 -`CurrentMetric_OpenFileForWrite` Int64 -`CurrentMetric_Read` Int64 -`CurrentMetric_Write` Int64 -`CurrentMetric_SendScalars` Int64 -`CurrentMetric_SendExternalTables` Int64 -`CurrentMetric_QueryThread` Int64 -`CurrentMetric_ReadonlyReplica` Int64 -`CurrentMetric_MemoryTracking` Int64 -`CurrentMetric_EphemeralNode` Int64 -`CurrentMetric_ZooKeeperSession` Int64 -`CurrentMetric_ZooKeeperWatch` Int64 -`CurrentMetric_ZooKeeperRequest` Int64 -`CurrentMetric_DelayedInserts` Int64 -`CurrentMetric_ContextLockWait` Int64 -`CurrentMetric_StorageBufferRows` Int64 -`CurrentMetric_StorageBufferBytes` Int64 -`CurrentMetric_DictCacheRequests` Int64 -`CurrentMetric_Revision` Int64 -`CurrentMetric_VersionInteger` Int64 -`CurrentMetric_RWLockWaitingReaders` Int64 -`CurrentMetric_RWLockWaitingWriters` Int64 -`CurrentMetric_RWLockActiveReaders` Int64 -`CurrentMetric_RWLockActiveWriters` Int64 -`CurrentMetric_GlobalThread` Int64 -`CurrentMetric_GlobalThreadActive` Int64 -`CurrentMetric_LocalThread` Int64 -`CurrentMetric_LocalThreadActive` Int64 -`CurrentMetric_DistributedFilesToInsert` Int64 diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/count.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/count.txt deleted file mode 100644 index c7930257dfe..00000000000 --- a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/count.txt +++ /dev/null @@ -1 +0,0 @@ -7 \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/default_compression_codec.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/minmax_event_date.idx b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/minmax_event_date.idx deleted file mode 100644 index 16cc6680505..00000000000 --- a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/partition.dat b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/partition.dat deleted file mode 100644 index da540c2d1a825be3ef80d13baa834834b2248491..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmXRDWo7^X0wDm8 diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/primary.idx b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/primary.idx deleted file mode 100644 index dc6e4026178..00000000000 --- a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/202102_4608_4608_0/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H"`H"` \ No newline at end of file diff --git a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/format_version.txt b/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/format_version.txt deleted file mode 100644 index 56a6051ca2b..00000000000 --- a/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0/format_version.txt +++ /dev/null @@ -1 +0,0 @@ -1 \ No newline at end of file diff --git a/programs/server/store/2c6/2c6da17b-121a-4775-b287-b0d80cd04080/hits.sql b/programs/server/store/2c6/2c6da17b-121a-4775-b287-b0d80cd04080/hits.sql deleted file mode 100644 index 81ffc1acf48..00000000000 --- a/programs/server/store/2c6/2c6da17b-121a-4775-b287-b0d80cd04080/hits.sql +++ /dev/null @@ -1,141 +0,0 @@ -ATTACH TABLE _ UUID '72667623-6371-4258-be57-799522ee4e64' -( - `WatchID` UInt64, - `JavaEnable` UInt8, - `Title` String, - `GoodEvent` Int16, - `EventTime` DateTime, - `EventDate` Date, - `CounterID` UInt32, - `ClientIP` UInt32, - `ClientIP6` FixedString(16), - `RegionID` UInt32, - `UserID` UInt64, - `CounterClass` Int8, - `OS` UInt8, - `UserAgent` UInt8, - `URL` String, - `Referer` String, - `URLDomain` String, - `RefererDomain` String, - `Refresh` UInt8, - `IsRobot` UInt8, - `RefererCategories` Array(UInt16), - `URLCategories` Array(UInt16), - `URLRegions` Array(UInt32), - `RefererRegions` Array(UInt32), - `ResolutionWidth` UInt16, - `ResolutionHeight` UInt16, - `ResolutionDepth` UInt8, - `FlashMajor` UInt8, - `FlashMinor` UInt8, - `FlashMinor2` String, - `NetMajor` UInt8, - `NetMinor` UInt8, - `UserAgentMajor` UInt16, - `UserAgentMinor` FixedString(2), - `CookieEnable` UInt8, - `JavascriptEnable` UInt8, - `IsMobile` UInt8, - `MobilePhone` UInt8, - `MobilePhoneModel` String, - `Params` String, - `IPNetworkID` UInt32, - `TraficSourceID` Int8, - `SearchEngineID` UInt16, - `SearchPhrase` String, - `AdvEngineID` UInt8, - `IsArtifical` UInt8, - `WindowClientWidth` UInt16, - `WindowClientHeight` UInt16, - `ClientTimeZone` Int16, - `ClientEventTime` DateTime, - `SilverlightVersion1` UInt8, - `SilverlightVersion2` UInt8, - `SilverlightVersion3` UInt32, - `SilverlightVersion4` UInt16, - `PageCharset` String, - `CodeVersion` UInt32, - `IsLink` UInt8, - `IsDownload` UInt8, - `IsNotBounce` UInt8, - `FUniqID` UInt64, - `HID` UInt32, - `IsOldCounter` UInt8, - `IsEvent` UInt8, - `IsParameter` UInt8, - `DontCountHits` UInt8, - `WithHash` UInt8, - `HitColor` FixedString(1), - `UTCEventTime` DateTime, - `Age` UInt8, - `Sex` UInt8, - `Income` UInt8, - `Interests` UInt16, - `Robotness` UInt8, - `GeneralInterests` Array(UInt16), - `RemoteIP` UInt32, - `RemoteIP6` FixedString(16), - `WindowName` Int32, - `OpenerName` Int32, - `HistoryLength` Int16, - `BrowserLanguage` FixedString(2), - `BrowserCountry` FixedString(2), - `SocialNetwork` String, - `SocialAction` String, - `HTTPError` UInt16, - `SendTiming` Int32, - `DNSTiming` Int32, - `ConnectTiming` Int32, - `ResponseStartTiming` Int32, - `ResponseEndTiming` Int32, - `FetchTiming` Int32, - `RedirectTiming` Int32, - `DOMInteractiveTiming` Int32, - `DOMContentLoadedTiming` Int32, - `DOMCompleteTiming` Int32, - `LoadEventStartTiming` Int32, - `LoadEventEndTiming` Int32, - `NSToDOMContentLoadedTiming` Int32, - `FirstPaintTiming` Int32, - `RedirectCount` Int8, - `SocialSourceNetworkID` UInt8, - `SocialSourcePage` String, - `ParamPrice` Int64, - `ParamOrderID` String, - `ParamCurrency` FixedString(3), - `ParamCurrencyID` UInt16, - `GoalsReached` Array(UInt32), - `OpenstatServiceName` String, - `OpenstatCampaignID` String, - `OpenstatAdID` String, - `OpenstatSourceID` String, - `UTMSource` String, - `UTMMedium` String, - `UTMCampaign` String, - `UTMContent` String, - `UTMTerm` String, - `FromTag` String, - `HasGCLID` UInt8, - `RefererHash` UInt64, - `URLHash` UInt64, - `CLID` UInt32, - `YCLID` UInt64, - `ShareService` String, - `ShareURL` String, - `ShareTitle` String, - `ParsedParams.Key1` Array(String), - `ParsedParams.Key2` Array(String), - `ParsedParams.Key3` Array(String), - `ParsedParams.Key4` Array(String), - `ParsedParams.Key5` Array(String), - `ParsedParams.ValueDouble` Array(Float64), - `IslandID` FixedString(16), - `RequestNum` UInt32, - `RequestTry` UInt8 -) -ENGINE = MergeTree -PARTITION BY toYYYYMM(EventDate) -ORDER BY (CounterID, EventDate, intHash32(UserID), EventTime) -SAMPLE BY intHash32(UserID) -SETTINGS index_granularity = 8192 diff --git a/programs/server/store/2c6/2c6da17b-121a-4775-b287-b0d80cd04080/visits.sql b/programs/server/store/2c6/2c6da17b-121a-4775-b287-b0d80cd04080/visits.sql deleted file mode 100644 index cd2d3509c4e..00000000000 --- a/programs/server/store/2c6/2c6da17b-121a-4775-b287-b0d80cd04080/visits.sql +++ /dev/null @@ -1,189 +0,0 @@ -ATTACH TABLE _ UUID '044d7000-c91c-482d-9f0f-719f226eff0f' -( - `CounterID` UInt32, - `StartDate` Date, - `Sign` Int8, - `IsNew` UInt8, - `VisitID` UInt64, - `UserID` UInt64, - `StartTime` DateTime, - `Duration` UInt32, - `UTCStartTime` DateTime, - `PageViews` Int32, - `Hits` Int32, - `IsBounce` UInt8, - `Referer` String, - `StartURL` String, - `RefererDomain` String, - `StartURLDomain` String, - `EndURL` String, - `LinkURL` String, - `IsDownload` UInt8, - `TraficSourceID` Int8, - `SearchEngineID` UInt16, - `SearchPhrase` String, - `AdvEngineID` UInt8, - `PlaceID` Int32, - `RefererCategories` Array(UInt16), - `URLCategories` Array(UInt16), - `URLRegions` Array(UInt32), - `RefererRegions` Array(UInt32), - `IsYandex` UInt8, - `GoalReachesDepth` Int32, - `GoalReachesURL` Int32, - `GoalReachesAny` Int32, - `SocialSourceNetworkID` UInt8, - `SocialSourcePage` String, - `MobilePhoneModel` String, - `ClientEventTime` DateTime, - `RegionID` UInt32, - `ClientIP` UInt32, - `ClientIP6` FixedString(16), - `RemoteIP` UInt32, - `RemoteIP6` FixedString(16), - `IPNetworkID` UInt32, - `SilverlightVersion3` UInt32, - `CodeVersion` UInt32, - `ResolutionWidth` UInt16, - `ResolutionHeight` UInt16, - `UserAgentMajor` UInt16, - `UserAgentMinor` UInt16, - `WindowClientWidth` UInt16, - `WindowClientHeight` UInt16, - `SilverlightVersion2` UInt8, - `SilverlightVersion4` UInt16, - `FlashVersion3` UInt16, - `FlashVersion4` UInt16, - `ClientTimeZone` Int16, - `OS` UInt8, - `UserAgent` UInt8, - `ResolutionDepth` UInt8, - `FlashMajor` UInt8, - `FlashMinor` UInt8, - `NetMajor` UInt8, - `NetMinor` UInt8, - `MobilePhone` UInt8, - `SilverlightVersion1` UInt8, - `Age` UInt8, - `Sex` UInt8, - `Income` UInt8, - `JavaEnable` UInt8, - `CookieEnable` UInt8, - `JavascriptEnable` UInt8, - `IsMobile` UInt8, - `BrowserLanguage` UInt16, - `BrowserCountry` UInt16, - `Interests` UInt16, - `Robotness` UInt8, - `GeneralInterests` Array(UInt16), - `Params` Array(String), - `Goals.ID` Array(UInt32), - `Goals.Serial` Array(UInt32), - `Goals.EventTime` Array(DateTime), - `Goals.Price` Array(Int64), - `Goals.OrderID` Array(String), - `Goals.CurrencyID` Array(UInt32), - `WatchIDs` Array(UInt64), - `ParamSumPrice` Int64, - `ParamCurrency` FixedString(3), - `ParamCurrencyID` UInt16, - `ClickLogID` UInt64, - `ClickEventID` Int32, - `ClickGoodEvent` Int32, - `ClickEventTime` DateTime, - `ClickPriorityID` Int32, - `ClickPhraseID` Int32, - `ClickPageID` Int32, - `ClickPlaceID` Int32, - `ClickTypeID` Int32, - `ClickResourceID` Int32, - `ClickCost` UInt32, - `ClickClientIP` UInt32, - `ClickDomainID` UInt32, - `ClickURL` String, - `ClickAttempt` UInt8, - `ClickOrderID` UInt32, - `ClickBannerID` UInt32, - `ClickMarketCategoryID` UInt32, - `ClickMarketPP` UInt32, - `ClickMarketCategoryName` String, - `ClickMarketPPName` String, - `ClickAWAPSCampaignName` String, - `ClickPageName` String, - `ClickTargetType` UInt16, - `ClickTargetPhraseID` UInt64, - `ClickContextType` UInt8, - `ClickSelectType` Int8, - `ClickOptions` String, - `ClickGroupBannerID` Int32, - `OpenstatServiceName` String, - `OpenstatCampaignID` String, - `OpenstatAdID` String, - `OpenstatSourceID` String, - `UTMSource` String, - `UTMMedium` String, - `UTMCampaign` String, - `UTMContent` String, - `UTMTerm` String, - `FromTag` String, - `HasGCLID` UInt8, - `FirstVisit` DateTime, - `PredLastVisit` Date, - `LastVisit` Date, - `TotalVisits` UInt32, - `TraficSource.ID` Array(Int8), - `TraficSource.SearchEngineID` Array(UInt16), - `TraficSource.AdvEngineID` Array(UInt8), - `TraficSource.PlaceID` Array(UInt16), - `TraficSource.SocialSourceNetworkID` Array(UInt8), - `TraficSource.Domain` Array(String), - `TraficSource.SearchPhrase` Array(String), - `TraficSource.SocialSourcePage` Array(String), - `Attendance` FixedString(16), - `CLID` UInt32, - `YCLID` UInt64, - `NormalizedRefererHash` UInt64, - `SearchPhraseHash` UInt64, - `RefererDomainHash` UInt64, - `NormalizedStartURLHash` UInt64, - `StartURLDomainHash` UInt64, - `NormalizedEndURLHash` UInt64, - `TopLevelDomain` UInt64, - `URLScheme` UInt64, - `OpenstatServiceNameHash` UInt64, - `OpenstatCampaignIDHash` UInt64, - `OpenstatAdIDHash` UInt64, - `OpenstatSourceIDHash` UInt64, - `UTMSourceHash` UInt64, - `UTMMediumHash` UInt64, - `UTMCampaignHash` UInt64, - `UTMContentHash` UInt64, - `UTMTermHash` UInt64, - `FromHash` UInt64, - `WebVisorEnabled` UInt8, - `WebVisorActivity` UInt32, - `ParsedParams.Key1` Array(String), - `ParsedParams.Key2` Array(String), - `ParsedParams.Key3` Array(String), - `ParsedParams.Key4` Array(String), - `ParsedParams.Key5` Array(String), - `ParsedParams.ValueDouble` Array(Float64), - `Market.Type` Array(UInt8), - `Market.GoalID` Array(UInt32), - `Market.OrderID` Array(String), - `Market.OrderPrice` Array(Int64), - `Market.PP` Array(UInt32), - `Market.DirectPlaceID` Array(UInt32), - `Market.DirectOrderID` Array(UInt32), - `Market.DirectBannerID` Array(UInt32), - `Market.GoodID` Array(String), - `Market.GoodName` Array(String), - `Market.GoodQuantity` Array(Int32), - `Market.GoodPrice` Array(Int64), - `IslandID` FixedString(16) -) -ENGINE = CollapsingMergeTree(Sign) -PARTITION BY toYYYYMM(StartDate) -ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID) -SAMPLE BY intHash32(UserID) -SETTINGS index_granularity = 8192 diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/checksums.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/checksums.txt deleted file mode 100644 index 939ea6ccf187b34a8db6e4def524f9dd13691092..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 260 zcmV+f0sH=AXk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$bhh(-qXp!7uuz9v-dm@^cD z*Z=?k&Hw-a@jV6!V{dhCbS`vwbON#VDF3f}{FN>b+}S`mO_BfzWMOn+E@EkJ`W*io z<|Oi02OU%Zf#+BUuAY>ZggK{VRU6KX=Hc=Yhv{9L4)i+mY_Q2(jfMG01a?q za&&2QX>V>WWMOm!_Mvet<#01I$(X>DP0c`j*Wcns^T1OQK`i$9(N KipbM-Mdtv4FmBEO diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/columns.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/columns.txt deleted file mode 100644 index 5ea415161bf..00000000000 --- a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/columns.txt +++ /dev/null @@ -1,47 +0,0 @@ -columns format version: 1 -45 columns: -`type` Enum8('QueryStart' = 1, 'QueryFinish' = 2, 'ExceptionBeforeStart' = 3, 'ExceptionWhileProcessing' = 4) -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`query_start_time` DateTime -`query_start_time_microseconds` DateTime64(6) -`query_duration_ms` UInt64 -`read_rows` UInt64 -`read_bytes` UInt64 -`written_rows` UInt64 -`written_bytes` UInt64 -`result_rows` UInt64 -`result_bytes` UInt64 -`memory_usage` UInt64 -`current_database` String -`query` String -`exception_code` Int32 -`exception` String -`stack_trace` String -`is_initial_query` UInt8 -`user` String -`query_id` String -`address` IPv6 -`port` UInt16 -`initial_user` String -`initial_query_id` String -`initial_address` IPv6 -`initial_port` UInt16 -`interface` UInt8 -`os_user` String -`client_hostname` String -`client_name` String -`client_revision` UInt32 -`client_version_major` UInt32 -`client_version_minor` UInt32 -`client_version_patch` UInt32 -`http_method` UInt8 -`http_user_agent` String -`quota_key` String -`revision` UInt32 -`thread_ids` Array(UInt64) -`ProfileEvents.Names` Array(String) -`ProfileEvents.Values` Array(UInt64) -`Settings.Names` Array(String) -`Settings.Values` Array(String) diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/count.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/count.txt deleted file mode 100644 index 9a037142aa3..00000000000 --- a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/count.txt +++ /dev/null @@ -1 +0,0 @@ -10 \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/default_compression_codec.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/minmax_event_date.idx b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/minmax_event_date.idx deleted file mode 100644 index 4b6c49f8d50..00000000000 --- a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/partition.dat b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/partition.dat deleted file mode 100644 index b9ea5569e3a3b34e3a1b6a1c4cbcb46f5e2d3fac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 Lcmb0VWo7^X0I>iw diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/primary.idx b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/primary.idx deleted file mode 100644 index d599340cc82..00000000000 --- a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202010_1_5_1/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H!._Hm._ \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/checksums.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/checksums.txt deleted file mode 100644 index 747078ab876a6913b8e0a9a92d3d0a3e3a1deb0f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 262 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyWHkv{Wolw154dF1>;$$|v}nZZR-0 zTma&a_H3NV`K5U!dL%eM=a3btktj z6fTJR@=T@IyyV0s#)et!Y;O}d4yN1KT?$>ea}nQfR!#$k4|%yo*~S~V4GOsV4O}$! zZrdrRHA`0hWDw2G%*#!zh)*p`%`1rq8l0+^nNq=`zaq;cGh)rekNa9x-^dl4GVm58 z7L{a{09~#JRK)U=M_>H!p{7$hOFDXtB}~#8xC@Fha}$dyLArT1wS3n4_i1|9>D}5g J%+@!V7yzW3XQcoD diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/columns.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/columns.txt deleted file mode 100644 index 5ea415161bf..00000000000 --- a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/columns.txt +++ /dev/null @@ -1,47 +0,0 @@ -columns format version: 1 -45 columns: -`type` Enum8('QueryStart' = 1, 'QueryFinish' = 2, 'ExceptionBeforeStart' = 3, 'ExceptionWhileProcessing' = 4) -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`query_start_time` DateTime -`query_start_time_microseconds` DateTime64(6) -`query_duration_ms` UInt64 -`read_rows` UInt64 -`read_bytes` UInt64 -`written_rows` UInt64 -`written_bytes` UInt64 -`result_rows` UInt64 -`result_bytes` UInt64 -`memory_usage` UInt64 -`current_database` String -`query` String -`exception_code` Int32 -`exception` String -`stack_trace` String -`is_initial_query` UInt8 -`user` String -`query_id` String -`address` IPv6 -`port` UInt16 -`initial_user` String -`initial_query_id` String -`initial_address` IPv6 -`initial_port` UInt16 -`interface` UInt8 -`os_user` String -`client_hostname` String -`client_name` String -`client_revision` UInt32 -`client_version_major` UInt32 -`client_version_minor` UInt32 -`client_version_patch` UInt32 -`http_method` UInt8 -`http_user_agent` String -`quota_key` String -`revision` UInt32 -`thread_ids` Array(UInt64) -`ProfileEvents.Names` Array(String) -`ProfileEvents.Values` Array(UInt64) -`Settings.Names` Array(String) -`Settings.Values` Array(String) diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/count.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/count.txt deleted file mode 100644 index 780fea92d29..00000000000 --- a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/count.txt +++ /dev/null @@ -1 +0,0 @@ -77 \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/default_compression_codec.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/minmax_event_date.idx b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/minmax_event_date.idx deleted file mode 100644 index 4fedf1265b2..00000000000 --- a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/partition.dat b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/partition.dat deleted file mode 100644 index 7af4421fcb5ebc50aa06a5f4dee6e454020552a6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 Lcmb0UWo7^X0J#7) diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/primary.idx b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/primary.idx deleted file mode 100644 index 7f33001d913..00000000000 --- a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202012_6_47_9/primary.idx +++ /dev/null @@ -1 +0,0 @@ -Hb_H_ \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202101_48_74_6/checksums.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202101_48_74_6/checksums.txt deleted file mode 100644 index eec9f2828ad60100f834ee2ec7c2ecdd572e4d2f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 261 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyUI``tQA+<)x6f|4kELHWvPFy2-%6 za2|+1+Ocsa=a=S{=#^BIFika>Ad;2)__e!oc(_Su$x;T6l*E!my`;>%-kfLCXPdd* ze|I8LNK`6r-p_c(P3^3@vsGTkiX2IichpIIeqjR>r#{1nyxgK};|<&n9adSpqodSI zI&*}wm1=%7h~{SI(O)RPe>EV>iAT$c8rTFS^0FlRNAYdvChb+}S`mO_BfzWMOn+E@EkJqa$s) z<(i%4_5Su;$|AM?s{R3`Wi?hP8GT=Tu0P7G#$xv>0SPVu@NR8#YcsG56t3`xr|pUp z;{9}LW*LD002OU%Zf#+BUuAY>ZggK{VRU6KX=Hc=138%)U-7Z;v9?{s9d$U?01a?q za&&2QX>V>WWMOm!K^bOwf>Qhj>E&4HXupM*01I$(X>DP0c`j*Wcnn{AlpFuovTY~I Kn?YGm%iaJK*Kw-= diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/columns.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/columns.txt deleted file mode 100644 index 5ea415161bf..00000000000 --- a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/columns.txt +++ /dev/null @@ -1,47 +0,0 @@ -columns format version: 1 -45 columns: -`type` Enum8('QueryStart' = 1, 'QueryFinish' = 2, 'ExceptionBeforeStart' = 3, 'ExceptionWhileProcessing' = 4) -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`query_start_time` DateTime -`query_start_time_microseconds` DateTime64(6) -`query_duration_ms` UInt64 -`read_rows` UInt64 -`read_bytes` UInt64 -`written_rows` UInt64 -`written_bytes` UInt64 -`result_rows` UInt64 -`result_bytes` UInt64 -`memory_usage` UInt64 -`current_database` String -`query` String -`exception_code` Int32 -`exception` String -`stack_trace` String -`is_initial_query` UInt8 -`user` String -`query_id` String -`address` IPv6 -`port` UInt16 -`initial_user` String -`initial_query_id` String -`initial_address` IPv6 -`initial_port` UInt16 -`interface` UInt8 -`os_user` String -`client_hostname` String -`client_name` String -`client_revision` UInt32 -`client_version_major` UInt32 -`client_version_minor` UInt32 -`client_version_patch` UInt32 -`http_method` UInt8 -`http_user_agent` String -`quota_key` String -`revision` UInt32 -`thread_ids` Array(UInt64) -`ProfileEvents.Names` Array(String) -`ProfileEvents.Values` Array(UInt64) -`Settings.Names` Array(String) -`Settings.Values` Array(String) diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/count.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/count.txt deleted file mode 100644 index 9a037142aa3..00000000000 --- a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/count.txt +++ /dev/null @@ -1 +0,0 @@ -10 \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/default_compression_codec.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/minmax_event_date.idx b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/minmax_event_date.idx deleted file mode 100644 index 0e3b7fb75b8..00000000000 --- a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/partition.dat b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/partition.dat deleted file mode 100644 index da540c2d1a825be3ef80d13baa834834b2248491..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmXRDWo7^X0wDm8 diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/primary.idx b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/primary.idx deleted file mode 100644 index 0b5fcd2fdea..00000000000 --- a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_75_79_1/primary.idx +++ /dev/null @@ -1 +0,0 @@ -HC!`HG"` \ No newline at end of file diff --git a/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_80_80_0/checksums.txt b/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df/202102_80_80_0/checksums.txt deleted file mode 100644 index a466972c338851bb9e376b5fec0b27c54f763a01..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 260 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyUI6hQ;Hcxb2IaD6D#6V%Tn`7;(^Ad>Sd->up}O2Td;F;<|11GQ{zVyvz!@t3lfV; zGE0E&)&nYHagV>iAT$cnNbtE_VA_ox+gYO}M76zw zkN^Mxh5!Hn@jV0yV{dhCbS`vwbOCIfRlVt&lYFUDR9N&B1^)mDWMOn+E@EkJ&k!F~ zY}HzkJ8Te4L?36TX=VY?FDFqniJ!N25;D3H*NTaDs0l6r@Fs0?Ycucyl#~%aH>Tie q3B6siBLUYk01I$(X>DP0c`j*WcnlPm#22y_7RAN-7E)Vm7x4gDE=jup diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/columns.txt b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/columns.txt deleted file mode 100644 index 725a526b957..00000000000 --- a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/columns.txt +++ /dev/null @@ -1,13 +0,0 @@ -columns format version: 1 -11 columns: -`event_date` Date -`event_time` DateTime -`timestamp_ns` UInt64 -`signal` Int32 -`thread_id` UInt64 -`query_id` String -`trace` Array(UInt64) -`trace_full` Array(String) -`version` String -`revision` UInt32 -`build_id` String diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/count.txt b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/count.txt deleted file mode 100644 index 56a6051ca2b..00000000000 --- a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/count.txt +++ /dev/null @@ -1 +0,0 @@ -1 \ No newline at end of file diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/default_compression_codec.txt b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/primary.idx b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/primary.idx deleted file mode 100644 index b3b53069cfb..00000000000 --- a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_1_1_0/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H9 `H9 ` \ No newline at end of file diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/checksums.txt b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/checksums.txt deleted file mode 100644 index 41a34e68660fcf30f2ebf3886af70965d1c50d97..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 188 zcmV;t07L&{Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$cC)=R{W#rbM&U?0f^HAjYm zkN^Mxh5!Hn@jV0yV{dhCbS`vwbOCIfRlVt&lYFUDR9N&B1^)mDWMOn+E@EkJ&=4}7 zAlkfU=Me)DyD->zPV@oLFCA^8bN;KGdEy@&Pv!bL6$vf?@Fs0?Ycucy8(uAC5Vito q|AbpmqG_6t01I$(X>DP0c`j*WcnnVC4s8oxdoa!Z4je{MT>k*Ano9!! diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/columns.txt b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/columns.txt deleted file mode 100644 index 725a526b957..00000000000 --- a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/columns.txt +++ /dev/null @@ -1,13 +0,0 @@ -columns format version: 1 -11 columns: -`event_date` Date -`event_time` DateTime -`timestamp_ns` UInt64 -`signal` Int32 -`thread_id` UInt64 -`query_id` String -`trace` Array(UInt64) -`trace_full` Array(String) -`version` String -`revision` UInt32 -`build_id` String diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/count.txt b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/count.txt deleted file mode 100644 index 56a6051ca2b..00000000000 --- a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/count.txt +++ /dev/null @@ -1 +0,0 @@ -1 \ No newline at end of file diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/default_compression_codec.txt b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/primary.idx b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/primary.idx deleted file mode 100644 index 24c83f6002c..00000000000 --- a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/all_2_2_0/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H `H ` \ No newline at end of file diff --git a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/format_version.txt b/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/format_version.txt deleted file mode 100644 index 56a6051ca2b..00000000000 --- a/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665/format_version.txt +++ /dev/null @@ -1 +0,0 @@ -1 \ No newline at end of file diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/checksums.txt b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/checksums.txt deleted file mode 100644 index 94103f5a76ccb7f25b7a74030c04dbc2413b0242..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 260 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyY$U>CqQtR5+I_knLZ-OHz2#4F(2= zb3pvjmW?wxzcjBzucV@cX`=X>wR6M7)Yk+*GdQ7`_n(0yC9xz?FDWyxQ#5noG1-=x zMe7?wHfev7dG?oaNwm=1jm!7WVfXwmTE9_VNPjb@9>a&c+@fsb3oO0Y#oSLXwp>@( z{bpjsq?khtqPdxQxrr6=sb#5oCGkLGQ}r@aDp&%G|HSG1ZQ1rGFrD>==$*L?yakCx zC7C5ack2NavHaxG7yo;x>6FfrjviwPlXM2|f}+ga#G*=&Zk~`GM+A);B?QWr7Vt)1 I5q`=5051<|M*si- diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/columns.txt b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/columns.txt deleted file mode 100644 index 7775d6f8eb4..00000000000 --- a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/columns.txt +++ /dev/null @@ -1,20 +0,0 @@ -columns format version: 1 -18 columns: -`event_type` Enum8('NewPart' = 1, 'MergeParts' = 2, 'DownloadPart' = 3, 'RemovePart' = 4, 'MutatePart' = 5, 'MovePart' = 6) -`event_date` Date -`event_time` DateTime -`duration_ms` UInt64 -`database` String -`table` String -`part_name` String -`partition_id` String -`path_on_disk` String -`rows` UInt64 -`size_in_bytes` UInt64 -`merged_from` Array(String) -`bytes_uncompressed` UInt64 -`read_rows` UInt64 -`read_bytes` UInt64 -`peak_memory_usage` UInt64 -`error` UInt16 -`exception` String diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/count.txt b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/count.txt deleted file mode 100644 index eb13855b7d7..00000000000 --- a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/count.txt +++ /dev/null @@ -1 +0,0 @@ -79 \ No newline at end of file diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/default_compression_codec.txt b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/minmax_event_date.idx b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/minmax_event_date.idx deleted file mode 100644 index 07abc658bdb..00000000000 --- a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/partition.dat b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/partition.dat deleted file mode 100644 index 7af4421fcb5ebc50aa06a5f4dee6e454020552a6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 Lcmb0UWo7^X0J#7) diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/primary.idx b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/primary.idx deleted file mode 100644 index 16d6a0eea67..00000000000 --- a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202012_1_18_4/primary.idx +++ /dev/null @@ -1 +0,0 @@ -He_H(|_ \ No newline at end of file diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/checksums.txt b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/checksums.txt deleted file mode 100644 index 2b7fff5ccb9418c7ab7aa25bd1fb7d93eceec517..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 260 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyZ>tGIK5Cl6r%;##xVQ61m%&ZZI$~ zoCD&Iwrrfq`K5U!dLfa6k3NUy@yiSHP=3yLyx6N@TAx_KfGyWeaxDL=3MNbdhr I+Xd$s04~;Rw*UYD diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/columns.txt b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/columns.txt deleted file mode 100644 index 7775d6f8eb4..00000000000 --- a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/columns.txt +++ /dev/null @@ -1,20 +0,0 @@ -columns format version: 1 -18 columns: -`event_type` Enum8('NewPart' = 1, 'MergeParts' = 2, 'DownloadPart' = 3, 'RemovePart' = 4, 'MutatePart' = 5, 'MovePart' = 6) -`event_date` Date -`event_time` DateTime -`duration_ms` UInt64 -`database` String -`table` String -`part_name` String -`partition_id` String -`path_on_disk` String -`rows` UInt64 -`size_in_bytes` UInt64 -`merged_from` Array(String) -`bytes_uncompressed` UInt64 -`read_rows` UInt64 -`read_bytes` UInt64 -`peak_memory_usage` UInt64 -`error` UInt16 -`exception` String diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/count.txt b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/count.txt deleted file mode 100644 index 9a037142aa3..00000000000 --- a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/count.txt +++ /dev/null @@ -1 +0,0 @@ -10 \ No newline at end of file diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/default_compression_codec.txt b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/minmax_event_date.idx b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/minmax_event_date.idx deleted file mode 100644 index 7d66cd0aae2..00000000000 --- a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/partition.dat b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/partition.dat deleted file mode 100644 index a713aecd075daae7071deceaa93147871ef28fbf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmXRFWo7^X0v!O3 diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/primary.idx b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/primary.idx deleted file mode 100644 index 3e5042c4bf2..00000000000 --- a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/202101_19_22_1/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H_HE_ \ No newline at end of file diff --git a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/format_version.txt b/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/format_version.txt deleted file mode 100644 index 56a6051ca2b..00000000000 --- a/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d/format_version.txt +++ /dev/null @@ -1 +0,0 @@ -1 \ No newline at end of file diff --git a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/checksums.txt b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/checksums.txt deleted file mode 100644 index 322c6666b3efd30007e1786d71ebf7dbb6bc46f1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10792 zcmX|HcRbbq_kX?b`(FFno3hEsY|xNB%F0MWT<*m+?&Z2hG;B&%s8l3WDkCF18Ym-F zMwCdUVcmP}mEXI3zQ2F`aqoQ`uh)5<^E}V3N(6qi0e)R^Q=BoWMwR6U5K%rs`c zqebZ+C)gsghP0ebQHp1vhrPLI*c<=ZJ<`Lhf@PpljHCTEw>F?*11Nr)#cv=Ph4Sk{ zQLLo`hh*=scf4W)%=+F02SCT9rCf)2dF=i;=;5C>o-L#63GI$Q0ca@JhfltIz~o-- zVUp^>;#xklXaLv>(0EdQs&h*!P=04R%#`ObPt5>c0}44Tm_RWkhmfe0GfrS#Y_lu| zx|E`M@cx-EORtQYclOXES)l&BMwX0*vC@&#Tjz%O_zQrt*9PAq0OP>|%EJ6XqEKe|$sBK0qv> zd*fHs$uDQ1;-^d8BM$~`#{cxhaCw|pof<5$WkoJ)4d^_lS6&$lo*2oEywRSzc&(=y z@EQ6O2_&jfV}_%mZaSgmq`#_nod)CXsPIh1ssY%(%dE-G4-{53w~=WN`J~iO0qLO zTj>9Ihk_eeM7^mtzDyk%|8P&J&}n|p%YE2=06vL)t_WF_v&c$`FU?zELT>H33Zwz| zJ}0msRuQ!h*9loRNR=OjpIhTU2Gxu!L-~uthHtNoO-Y_qdX{85ruZvQPleM7@b5c? zoT@RrLus9trQ*%V{WXS!*a+a20e}zEI7KE2Atz%LK7ddQrDqnX52l{lGtE%9xZrMR z9xB*XMq81Zw$y2IKR4X|Nyi@g`viDEYVL9h_4M z;)hwLE=NifG>O4)0goY>L?U<&Lh3}4w>x67G^PnXG@XOz=$xF&3dtdE9P)b{uhSIro;(5EhGeoIkzh>1dH731 zbaZ73T8oE#RZblwzm_49+*bYxBs|NQn%t^Ej(yc<|WRl5jM z#-GooSiP z8d=$eAg`MK@;#jOM~4xwCm1NPN9p^qpgWnTz!J)~t$xqtfD!Fl`^RB}AwkKU`vHu6 z&e_Jh@cd2Ny4V5b;<`yAZv`M=WNm3k4hZxoPzi`fF;7>cVMl+zh2V!t2Xud&@MBNC zT<8J*pjxlKC1^{RKTLeyeQrNVieK;ufPDcSX6^UrU@OR-h+x}nm=XUu2}G4a4dOBq z)q;$}6Yylj=CAA(G}gaQ|A^G$fru9xc6G`>9LZn{H5}fKHPzkR*>Lz&-OHaNLt;h% z4O0P)c}>l%zT~(AbYs9E-HB@eXQ=n z*P3_L6vX6a5c+Gf!_F#Z`^LrVFLRPa3;+ra1axL?Z&3?1p3~L)U7q*cPz-AYJVs;^ zwHMMbBT{iFI%Y2Z38cVoi~W>_3s}V-AMUteAmM>pLyaVDU3$t=YZ*@5`Nz-xoaKL! z2Q$iy8su7D3N73@Wa#DARSvL@7!4yFbtm$;Mc4LoYVELtIHZ(oQ8|HqgI|Y_pqEe= zv%G7#&1Y-{&7Z!k=%^jve%yIza>8T2FN2A*t=`#2*Y$tHti!|2lHF+Mk1uABukB zn!Oxc2-r;=NW>t1ctLFR_6sz4dvB~&tng@kiXu91QI)~2O~x5O!qaTB3xx+pm3N!$wHG9geseT)!qb*6WqhacSj>6-94Yne;1 z$>-py+)A*5lFIA!d9(98d&TH-qu!#B*>>jB*)^AvV)y0NYW^@e^P%N*av9wI}np#%G1i#?H+fJ6J_wGv>DMX{2Qmq12TQjL3&mM+g0d-0QHuewu-)K^%JkTm|TCw+YqL^;O*`Hde34 z1@7NG1^7(~BmxEJ&xw{M1n0t=Vk2`gOdL|tV7QrBP@*}Ywz~%N6H2XaReBO7sRqZD zP*i@Yx#6N?uW!G?CM>fNmRssD9?uuGS?T?BEEb3=0h?fA1VLRM!KU&D7g3ClCIw7d zPL+Ae_O+|dVVxVOE!2Itf~au$^CC*m7yi2WJCr zQ!>s!*p`6v^daDV!K!H9oJzFeNbr6Ez1N8ey>InPx@Dn_8y#pone#{0Z(fz1^${`= zji-#a@pl!rcO5ZUVosefC?p*c8N9_01myvy7kSsx+6!KRrv_LmY5r~cyVZrT31@o{ z_()9g@07lGXD&}A>G{eBAS!eO0N4+K4?cEPde%_+)Qfqxl){o{VE}7NCgXM3z$?TU z<2U-kMbqz{`DOWVqkzSdrsb{ANWSNvW&W7Sr+G6V6AIzX*EzeL?+mo-FAtAW#)?#| z;-_*hyPR3TY!-#2z+7xUc4Pi>YqT%^T(s@=1F0clN-tp=%KnMxgsjEo9+|1IlfAFr z*KN}{2HW+#yP>?(VskE>&y9{xYnuS9nZ3P@F@y7y~kO8T5@KRYjd=9g7Z_4er)9Dlr2@-Ak7@xOa4-_9n+ zGYD{*+gK5(VPuM53v#lh%4O8Vk|!#)_$)qQ%!gC!=CE)JLTScwBMkIe><)7%svcBO zzU;9LYz*diyn6OTTl|YQE4BUm-D%T+&79;(rZLgc?_Qp}LeB3fo={v)6&mP9<>vd(<0X?c0jGJe5jl+HPsZt^>FB6xEQoX|&R;9>7Wgq3 zMjW)`a6v#Y(B$xH*4=%+zu0$pnjS4XybZI97@1`HSDKdAw_g%+tHCpbw}9O|_#G0h zKdVu092dXw?XiAJZF}VlP+sp&B0Yt4(^TQ9JG@Ogb-^r(hrav#MKTA3M-~+5=zuQ@ z;TQahm~P(c%tD+VY(XUXu_ED6)2=f-zW3YXyXVKNWi#Wwonq(g5w|zHPlbFIs_?#X zC~wTAE-ZZ8?F`n_yXE@XBRw7&a35QGszxXQ9Ol86WDjB(2OV9WyXTp?2$NrIcrpX8 z;iWy=tmS}2*2&k+lh^4<)26%5>ucM{wEh?OkiSp*o%UNxwT>g7ZpV1dTGT<&R%EIH z(o=a>vC=Uw<}#q%>c_J`Bfq|>3AHpbmhF1tjClT5gvnyP`A)~f*)Jf%JuQ|0JkPSf z^XMz>>BwhKyq9q;cMQ)1Zu4Mke>{bahhV@$N5?!t#hGj_|7Kfu%T_XS_E2hOMk!BQjMM(1Ok<{3a7k1WMb}5Zp zdGMVqg0itCdyuJZi2J`Z@uoko>8a2V`HuOO+}ilbDgxFtn#KaPY~LqaR%nVgiaMR>g*hxq32_xGt1u7OXv z7Ey*Pav?!F=Y#cx7mr>X^K{?#{t~lWf7}hV3;8{IF0ASEw!__k^C&J9ca@WluFG|J zS5l#L`)ZlP)R!;GfguVx&|+uiea0mGes((M;$j1e+eO`RWmfcP zWpaH-0qgEtwwNu{H`2V1sP}lN)&BeB`>%_42NJv#1x@MbL=`LgbwY9r8=G@Lh z|B+$_?9pz1@f)0v303inEI4@V7~rahWo&#Lc^GhRtS0StQa{%nnxUo<(}#>QfoP@` zRLL-LfA!%N;EF|&6jd^;qTVT02tWevTp7KOquZj}jymjZq{pl^wEi{~+$qCF`# z1N(Yb7c!G?a{5fIzPZ8o4(eLG%tE#-@y4XM=mq%=Tm7P6#j1x^~L8{TzMh|8{^5AP#vtW@atKR*{>hRf(%W0wfcrmk|fCIt%I3&LZCwSni2Ni@ZrWJTM zk1su4%aV#jEuz{R$4!|t%9$M4yEOdv$5a2S98e~QW=x1!)puc2VXl?(Y9~j4jzX+# z6?4aF{0c?yc-9;x-m$o2pz&@G3mqCUTbw4$K3g6fS8J$kiG1h?pzZpIi8UOrHLP8K zgcL9^Nofd-iKa!&$bYGy7DCy#7AJ(-mKM#sMvIPrhnY33qqcP;^myYK~ z6%T}>n62$B01abK;_OX-{Z=)F*s^pm+d~_0S_cYqNL%5U5p+oGW33)>$ZnxFUA18u z2;x~?t0h3oDEa*Q!`Wluwn7meRKp{pG5?GC`A?dkKU+n1ucXLa$5O9!0B%`e7fi)b z^;6NSqD^DWLG`^~(@N@!(G{&Xa*NSfP(@-9x+JAbU}ba90f|>H_#aT|@HW;gz@tWq zyIXzc4=)Nl6<2I1l=w@!VhnKsfjFW!i2$}lD}FF__0--B`1SH?U&l=EmokkT&{kq0 zI*qhI>==w(_6e%Uj_*sw{Kvi)QR!-U-`9*<@y|THDuy;6S%3-%u_I7IiJk;iXhCEE zTAOOfJG0$0Z1C@Dsiu? z2Y=t)d^Fm>Qd4>g@b3n;hyB)DaZBc~Y4rPnnRKMviRO~1bA1xAnZXnZOJDce6!KI;4iItp@opm=E ziI%r)0MB0LAj1#}g+THY5ndC0*uvb^6`TqlskV(=iL54OTz(1tnfbtx@cyc9jeO%X z64URrJ0~A?@*V&%4&fG8uN3m#G#c%QakKBA=KSCd_!Iz27D4O8B#f{y@7;bymZYt$ z)L&-3O@aPw8>mRlimbFj{UKrl>jCo@b>I2_X>xQ}!1CfzrTu)Wf^8}BmmL~U0d7UG z$C`qF%g+$0UqUH8(u?RWH)ogbn{T<%sZN2e_vWF0Y3BhK@6RHv)@e;cKjims0rWqu zg$_G(kADi}y*$otxti)I&Q=aM-GHVIg(%EQM>o9Y?w1P>%Lf@}L(PyWf z=->Lel!1_q3W^sm4+7;hjgJw|Xe6SyM4NK-4mop$TFDhy3g9n0y73M>K=*IlXKM1f za&AoTQ^yyY+fsG;n1-+(fRJ$8f+W{bhb@60Xfr_xUz`qc4FC|gCh~>I*Dd3a%$l`` zOKuu$5{37~C!#hWM#)F?*$bUJo%*avYQK5i+WXo@0|3AT@@zqmJLSU5q;x`EYC;Rv z4;=FPYe?&_q3H(kaF;t5-3aR>tbaxCNiH($MPFy=^x`7zt z*9yXHK_B?us+w@RQ=TVZiWmUP64gMC;O(qCOED^H4C;< z!>0F(ul7~kZV;$`S|~8!j~LqHkK9eeGgI!^1+feph1C|xcQ3B{BZl@OUkJD6Lk}B_ z3-4w2tIUtND8C=zEr&YC40Q>A*1Ifgp6O#-Gh8-OWd9mMe!mVE@w}NgYYS0iN!y{f z(=t6C>>Pt)PvHDR2u9?PSHfFjKmRPjZNu`CB|@JPuj#>$esHA*K%}zTnH`Vm9 zc0l{$8RO%&@G6vZs+l?SK!eO3=cuS6r)SUz3PxluKfqv<>l#&m{6ycn(M@KOlZn#Sb3HNMK$=IXfguXGf;3R88f@h+lmr zznziTVJt=gFcq>f(3O4paOhxVpap>2CMybHWQK9*O7eTTr{5?^cYK+?ivN ze4YUQr^+%|W+VoauDE?-Q(jW5<7sPT$b|FdsarVPNy>&()o0#f zfFqt3UlXHu{6tG7F0GaG+<4Y`F7KDdLjHbeIww~7>A}4PMxGYG*RG!tpYXq`x7|g~ z!{c9KW8}TpqmtK{j~G1tISRNGfCkd%-s43Yt-e-l)Gk5O&%~qIH~BYE%y$C zr-5}Tmopm0Uc$x;IGU$#v-1U10Hdgo2`yq`ER0h*FsQ}@;sMT<;El|Ja-n&#UkfD= z-XlJT2c0Z?I?j4(3OXSVHNCpbEj5|#DPcHkZ}SU#$H!4wm=6cg|K4>S?t3?wMhq>x z!dV+v6!~Jv5D4!Am}`Gq(zoMq`Hj zKw5kE|D;anPh3qj`Bn|K^8WO`Be8xWT_|Y^QYY!?^j8|MMpS(hE)PWu_w7|0WOUhI zf@#puweYsP0aY2xL%k~!)8-)^DCl3K#efMpb6?43*bhdhkLs`ZeoF!zlK@WOUlh%K zhFWB`IHo^vXk+}Xs?ia0>EypD%!Q9xX7`?1DWX;!+sG3fUE5<08uWpjer-fuqe0$_ zf0{il_037(9zdoN%x$z0;`1tb8rE`uJr;d?O5nTN_4Kae-!1>nMtEx~X|F9CC&@0C zU%ydQ%KP8UBqaSVG3sP6oo6UzQ0ai9DKayGswC)`rn|r3z>);A$+cbfoPQ3T>TUcu zyu`JD+N`iT*4q)#Ht@rJ+B#3N8kv}2sw8Mw(dt|AjpIu;4u=(k*&eT20Zv;&PzWKI zY84VtjcCk_+-T;KupE|fru1NuZzo$G$frnGBb|epqO9K8teD_3=g`75jmK47NMBR~ z7NG+(W^CpB{F*wBvZ+6Jy7ajKDSL`E8y%A|R|P3QsDDwp&_AaA8bp%|6h5{<3s9kn z;6X|1@mhs!iAR?-3{JWLfS7Ut3~tslrtJC8HO~v8P3`8W);568mJm$#51}FfW<&^_ z7oek8JN&u7*oe{h&c~VWx7QZ&4exhHW>!6tQi&zL*K*L0YLvhko*&z_dl~h^$(a;( zR8`{eC1!Ef*@0(35ga!o5WRiOkT#@wWCVKD{&SbZW&2iFw^zO?5(PeqY)dHo5%s1G z4RC9?c25#NU*DF1H-P^Muq)8slu9m9zn+PoE`>2Vmezc4fTkl6{|0ee;l*)?^~%Fa z&aOI|PKjiok9I<>H&}+c-$q{cOaDl3{eIP&P4UqBU4Z&GPoqIcrB|699_kpa$E0+7 z3yE_!V~D|lWKu9ef`^WIFc8b!H2>3~v>5Nc?`ieLhkU#z7|_gEZC z-s5*ebQ^ybV5#12>nyG8{!trWliyiOkw8cpaO^-Zzaxd2OM9=+m#rb`@?EhXUWjwQ zN6HlH4dc2kPGF^amVdq~^|98Pus%|>Nyua)k~92^i4sWIsEKmD^H4(=VDWs?Bmzy6 zj=omt6rpVK8RZcFwyvl2_;htYgh4H#f_Hm8eBacEN=wm7f93IAV_RKYVI4m1`@Q+{ z-GY1fuVvzou&x7MI|7d4=~F64M-Nv}Ulsb_`1R5HdH0WL6Ganl&TMEMWifA^J5YGS zJ0v~h4EaZ$9v?FB@$-_3Ele6@JkKWqPqaKOC_xZC(pa0t_cSH*7A%Q0hB zRWQ2(<=Q+)U7ckG^c6~f!{GA?faH24=4wJ!4pje!&SQBHYUmDDFxu$Df8Met8XaG4 zb8{y2DgaJ9!VCgB>+MUZP`##uukY8^IbCV2sA-CQGKF47x%R@(KUjqA4Ouaml|R1x z>2K`2025z&7K zx={26F=9eb?iWbk<)m(CbkneAbs-@!s!0liT+^k((!6o>xj#@fb0Q>=X~pIe>rL5p9JP2_7&x% zC(Gn&uA_Z_DP|gW3u?Tw952rt?oJX38dn^-F#L~V9zsGQIw=<#4xvgHRx#`)uh~7@ zyN1@uLv2A6T}NxjqPd8c48Qg2ugBv_h@o`E(1t3};lnS&Q?)(p&W0=XP@ z!7FM2U^ob@z}mH$l)887#g_hIByyLoFyL1Md^SL7Q#5giKd34H-5odo3zumKuu!25 zsI_1Kz3zD-R&2~s)}!4B^Cpd`wB08z!oc>GtK+Ljt}c(rMPZ^1j{xf$%n+Zu_{iEt zYw_4v-;Scs9(EpXGRWk*v-N8?>W(yL1Dz4DR(Vi!`LI=08`~a$wgU_nr|gX9Mkcvz z*m?ocJET>>Yfr%;3a>Ksx2eO7t}j5v1utYU4@+Fu?&`m9lh<>XLm=fdm)oqS*vCcl z+er!@O`UApn;Y{mTlTI_k1fw5zP5_vT)Daj0rLqoO?>rhr;9a7x;RuLbKGN)Z`V!U zI$LP=?$2)2N{`Mv8=FJ7G$kHXJ-ImcmPPaLro|eQ#hVP>?$W(RMbGU{?XWO&13XA> zG-21oewLO)wT?~`>sDzAe;&w+O9vlUgJ`YsT3xd-a?&CqE9N}+CP!3Q+_s6@8XWqG z{nFscMjPi2sWaBD>SQ3K05Cf^kg?<111Nz1QGaN+*H~XKr_0gifwUbQTc~$SKWjrh z;&0q-Y-{Wl-d+12hA%PoTYui=cb8&PxLV`Syy`^)SmaD4*d?) zI{D?L@k_%{$~(I0g|b7;r?{p?3fSk&u#a^*ww3h-rs42u_ba@ii!&=*4^GINZ~}J3 zQW2Jp`TSslXR_rQ58=tCW|o*g1o- z)|`ctT1wgzdRk@go4^@>GU9-4VV6bA>jI@pCxEsTau{;Yv1_> z`=aS%n{u4{LL{zi1JW`~{vHoP&RudA7Ae*_b}j$lU#SukiiAK;LJBnfVD$R)+QY?^ z*kA$1Z0BqPqEa3xZCSIMJI@k<-e@aLD zF+;YBwbO~L&)m!tntAvBCpN>K8krZIa*Qpk`4?HG_slQ=ryMwrOzsJ!PIkITW);`9 z0AHTTJIT)mp7xP1)^;aAC3%(TVRv_~oK6KB{`#wVu296qECBzT^fK5EeK*(AD+?N1 zb&FHJag=2Xi3od3OQe@G#nMFT<3)Md&^6dE;eI8LPjL5+5^h2B_Xg1^ccOVpRc{dF z5+;TooZ^hk0Pw%X3_2wKSY)qj(WKelueS(n8#<-J9^&M7+$ z=SKf=;YhNCZa(^fo{!Eb-`T0S%(Z#5G2n!jq|^Uwu})&q3y&@1b6Sd5M5=mZnvf8) zx3niv9)kr@S`oY6gO4hOjWM2MT}#L|fl?YYpI3^`WlN6Q=x7QsjW{wPd0=*6TUeiG zHP_HyE{yPu3YI-7yh{~hfLs$<@Y46jK}%wzSFS*_rlG+_dA?8X^=hOm_G=CP+dh*q zxSn^mYM(4w%d=3u4UON9{YBUzfZ5o@%-u&)FViZ1{|4soW|1Z`YEO`%W8QrjhNQIQ zm0qMod|8q6j^&Pv?0(6-j)FFuQ?+!@+2x_tLY>1RYPQj41q%5Xbz!V&Uq?&8SV4Wr zizWQuNjn}9*4ul%420xD72IpS9ypeMhDWfv!xLeR`ZAsEOh=bGl$4rCGwY|`c>?Gk z6{5lKe$BnxH}M37HNnkGNlY1lEg7J~5gGjZ&sK~}CL!zO+FIkyo{vwN7mh%A@I30y z>Do;TcGt4GnDj1r`~jzJ&4#lcfNl8%@ zvSjAZDsV^%U^arFCk{w*!@tEl&zC95sDCc8z; z8zQy;r}(eH!a!1-dcI9!>A?%`H~uW^^+y9ieO49pI-;o;OhNu?cCjk@htbp2 zNuAhj8nl4M>s#-u>uCx!Irhl&*4EJs!0ObjiELS=2*f$Ge zfg;>z`^kl!J};t5f6a$ZpJhWT&v8VwlMhyZS(P#G;Z$42;(2mx6tH^sY05Ku4`ls553pOZ%%@COh{0l09tZx}icvadr>CE~-;hIe)Dg_qIVf1GoZXDPDW z2e5%SWNiAKjmC+`QPCUw+YI-`B^e97(U5{P&44g)Ofl!CYbC OT{m@gI+HDZ!T$k}efzWk diff --git a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/columns.txt b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/columns.txt deleted file mode 100644 index abc4136c586..00000000000 --- a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/columns.txt +++ /dev/null @@ -1,135 +0,0 @@ -columns format version: 1 -133 columns: -`WatchID` UInt64 -`JavaEnable` UInt8 -`Title` String -`GoodEvent` Int16 -`EventTime` DateTime -`EventDate` Date -`CounterID` UInt32 -`ClientIP` UInt32 -`ClientIP6` FixedString(16) -`RegionID` UInt32 -`UserID` UInt64 -`CounterClass` Int8 -`OS` UInt8 -`UserAgent` UInt8 -`URL` String -`Referer` String -`URLDomain` String -`RefererDomain` String -`Refresh` UInt8 -`IsRobot` UInt8 -`RefererCategories` Array(UInt16) -`URLCategories` Array(UInt16) -`URLRegions` Array(UInt32) -`RefererRegions` Array(UInt32) -`ResolutionWidth` UInt16 -`ResolutionHeight` UInt16 -`ResolutionDepth` UInt8 -`FlashMajor` UInt8 -`FlashMinor` UInt8 -`FlashMinor2` String -`NetMajor` UInt8 -`NetMinor` UInt8 -`UserAgentMajor` UInt16 -`UserAgentMinor` FixedString(2) -`CookieEnable` UInt8 -`JavascriptEnable` UInt8 -`IsMobile` UInt8 -`MobilePhone` UInt8 -`MobilePhoneModel` String -`Params` String -`IPNetworkID` UInt32 -`TraficSourceID` Int8 -`SearchEngineID` UInt16 -`SearchPhrase` String -`AdvEngineID` UInt8 -`IsArtifical` UInt8 -`WindowClientWidth` UInt16 -`WindowClientHeight` UInt16 -`ClientTimeZone` Int16 -`ClientEventTime` DateTime -`SilverlightVersion1` UInt8 -`SilverlightVersion2` UInt8 -`SilverlightVersion3` UInt32 -`SilverlightVersion4` UInt16 -`PageCharset` String -`CodeVersion` UInt32 -`IsLink` UInt8 -`IsDownload` UInt8 -`IsNotBounce` UInt8 -`FUniqID` UInt64 -`HID` UInt32 -`IsOldCounter` UInt8 -`IsEvent` UInt8 -`IsParameter` UInt8 -`DontCountHits` UInt8 -`WithHash` UInt8 -`HitColor` FixedString(1) -`UTCEventTime` DateTime -`Age` UInt8 -`Sex` UInt8 -`Income` UInt8 -`Interests` UInt16 -`Robotness` UInt8 -`GeneralInterests` Array(UInt16) -`RemoteIP` UInt32 -`RemoteIP6` FixedString(16) -`WindowName` Int32 -`OpenerName` Int32 -`HistoryLength` Int16 -`BrowserLanguage` FixedString(2) -`BrowserCountry` FixedString(2) -`SocialNetwork` String -`SocialAction` String -`HTTPError` UInt16 -`SendTiming` Int32 -`DNSTiming` Int32 -`ConnectTiming` Int32 -`ResponseStartTiming` Int32 -`ResponseEndTiming` Int32 -`FetchTiming` Int32 -`RedirectTiming` Int32 -`DOMInteractiveTiming` Int32 -`DOMContentLoadedTiming` Int32 -`DOMCompleteTiming` Int32 -`LoadEventStartTiming` Int32 -`LoadEventEndTiming` Int32 -`NSToDOMContentLoadedTiming` Int32 -`FirstPaintTiming` Int32 -`RedirectCount` Int8 -`SocialSourceNetworkID` UInt8 -`SocialSourcePage` String -`ParamPrice` Int64 -`ParamOrderID` String -`ParamCurrency` FixedString(3) -`ParamCurrencyID` UInt16 -`GoalsReached` Array(UInt32) -`OpenstatServiceName` String -`OpenstatCampaignID` String -`OpenstatAdID` String -`OpenstatSourceID` String -`UTMSource` String -`UTMMedium` String -`UTMCampaign` String -`UTMContent` String -`UTMTerm` String -`FromTag` String -`HasGCLID` UInt8 -`RefererHash` UInt64 -`URLHash` UInt64 -`CLID` UInt32 -`YCLID` UInt64 -`ShareService` String -`ShareURL` String -`ShareTitle` String -`ParsedParams.Key1` Array(String) -`ParsedParams.Key2` Array(String) -`ParsedParams.Key3` Array(String) -`ParsedParams.Key4` Array(String) -`ParsedParams.Key5` Array(String) -`ParsedParams.ValueDouble` Array(Float64) -`IslandID` FixedString(16) -`RequestNum` UInt32 -`RequestTry` UInt8 diff --git a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/count.txt b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/count.txt deleted file mode 100644 index 9034d8fd0a3..00000000000 --- a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/count.txt +++ /dev/null @@ -1 +0,0 @@ -8873898 \ No newline at end of file diff --git a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/default_compression_codec.txt b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/minmax_EventDate.idx b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/minmax_EventDate.idx deleted file mode 100644 index 0cb239d932a..00000000000 --- a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/minmax_EventDate.idx +++ /dev/null @@ -1 +0,0 @@ -?? \ No newline at end of file diff --git a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/partition.dat b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/partition.dat deleted file mode 100644 index 9c176950d3f897f5cc9714d7011107a21e47a229..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 Lcmdlj#LNHy11td1 diff --git a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/primary.idx b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/201403_1_29_2/primary.idx deleted file mode 100644 index 9a604fd108fd6ba7f48a9c05368431b9ac5fc126..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 15330 zcmX||cU+C%AHW-gRMK5din{yWL|M@jA{xX|T1r%;G*qNQi%LclMN5%UiH1s%G)1Yb zgpgDcz9kKf-{*O*`}_0#`pk2lb;Oq^&B6N71LTr@Km*K93|N|JcPLgIIDY7Cz7mSoC8YfJH5ob+pVQYGl< zWj#)-Tw}++htkoHCJ_-5cGQEsE8UGr5CwBEci#}s;}Ar{G*fPL=w9L?`U^a>l~rRZ zCKuW0%xe)5dF-yuG}WUMJk<6Dmz^$RK6X^+AkA((!ucenwaF&*f8Kbl1l1zXbe z`TUN{Y=WrRvK5o(pXqW5qG6m3bH+Er4Bj2XXKB^^`)`%2*vLs-6t0oXLFFsAzc5jw ztSH3uVAU+7s4~z~HBr#iUNF%pww;Zt7mC6)y6xlk@@y*uy;+89*3rzc0b>R#U4_fe zl7IMQDKk-}9!}LWeYeRMGYFz$*Ya7S$!ED}_8MGvbkSLCnKKjlnv25Sv*en?&=xZe zdbt~qHz(C(^W3aWDSn&$3UxU?7e!?!`}L z%wyG24*DI3XQrLv(UP-?fr{dBFOI(^yXAhdkY5TOVO>%a_sSVI3QQM;t0EOT8W70m zBa;lAAvt&iSBtKLMLarMUyMia zKAKQ`xs8Pw^>}6pmvZ`T2NkV~yab*jMx0~x7{!H6ur96Bu0%|W|3xTeL()rAHxnP^B) z492%I-hR)c95ymB!0Fm_XS*NT9Q4^1x67Z^{AMqci>e&OV3r(BmR#-VCjM4aN??R>o#2#AF3FgxM=uA4!~$QbWyw%L$5iy#U% z-DA0Z)^s*ORP2B9^ermy%y+=pMzi~2`DD3Eqr6a}&t|@ZN%InfE zF7go(huJ-R`{HjqTNZK^69<`A()jG}jXab$8<&|e#*G{!S*V024qoi)^FDvcr=#%& zIFYxm?f9+1KsigqA?w)>0%qRc!bVG0;dUZyCtunH7D4iu>i5FYBVK%@yg?jBG^Xl2 zQf~!MoDoj=HJWylfjklJQ{+Zj(fm?6YOus}Nk_)}+Yj-Ot~)MUZ*y*3n$AGo{y2%| znsgOwGthzr+>7E}O5ORBOr%_i%kWwHx}1(y)r!Mi5c2_BP&3R!9xrj9W}8GG9IfJ^ zn_qFeibuNcJN7V;+z3vbA$5!5cm~o~ISn5}N`rJ%01FxVPXk%pdNC=lBnDd1G7Y@E z_YNO&sN|tdo2NrOm-sr(eMji%bMbW09Mjmf=HpHV%I}#D^O7`>{_;>68_n^Qfcukv zS$zHAN_f9SNkC@ztp|cb;JusmLjvr&Bczx4ZDgXfD>ER%^XtvIap!o*Ky)T}Ik{OS zKMLM%{&p1@9!HLE@L8_g|i?RPqEq5iH(eZ#NP6F)#t533bE_3 z(^@K|N0C~giQV*nn%)nw>ywc}>iQtwXs`NzQtunN!xs<4s}z z=JDtxEJk*pr@Nt$zEvg0w#^XIroS#FvGCp%XyW(M4_84hWGu~ES9+|4Maallgw^0u zmxn@sArcx)(V|VhY*+WKVw=CwplLiDuLH_|9x0>7U^JSRErESWk$IDoXbX zyucSWVavw$Iw4KS)q8&T7gD0ug~VXQB_dFSM#HDrRxv4&v2q(J2JG+DZrfJ zM8k&WX^kRHc-P&)HLqFbs!48O5=6ls{q3L+>%&_SWHce52sCNfmmt5#aSsJDd2ISmUeP-ZO>~q$8#22jYao~DO#eqlN~ag) zrU~gnFJ|iXMM!0A50@R5LaMNs2&pR(Qi!eU<&fDziuVXEinuMLA5q57m#-Gm!IBm` zA^F>O=bF?BNmb!%{cIJX-JRxwE?r{*k+J^7qoQiCM!{!-j5YuCQ;CN4r+|`|o9foM z3uX07EG$`eLVEm4&8PK>kZv8fiJ9dsB>X8Ry%2f{BI{N)6$)kaa{CW{93hcA4~`W# z2s9~}U5>vpJDE*Hr(hZ%Li8IJ3F(b@qeG3JgiLU}|4cyb0`E-0H4XwtBE^K^}e zoP;v@-E-V$ED%z>Txe(QF!G`e6Uv^ZvPa_o$>cHdC%fj+=kSSm z^4Q83tqRX}m>09r^Y{4uLSfmAg#BQmWeQU892vVA$ObX#|0uNN^^WE66&1*;I%Ffw zV7(+D>m_%p11$tZ!rXQB#haH3ypS-zo^?j+<_JkmV`<Dt1GYM!EoZ2(Aef_t{DHuaLr* zUcYG3C!~q`dr!O9a|ulvrV_xKlg{P+BR77gEp}NbvtD@q@Z}pq!o8eV7HYO{j(=UM zC?I*Pt7*I1*l9lDMIQ6D@Y{X7NJyfum>DyZgjA|$*IhHtN3zTDD;8sCZ}0GlP7nz@ z9L7_90&f&tM#3aMwtr1N!z74|$b>06U(`RK%I{HnYQ{owU2l!cnV<8$_G$1LxYUR)GAA_Z%y zFy>|V>#CaQ^;&7rv|ikV>Nm1bZ>BV4J!T?qx!;#gkUSRJpH`!Pg^%uaO2gGWHrsqF ze<24oV=|!G7`k|Q*9aY*J|hDW-e@Xm9%XS+!$TSH`ONq1K37;LGiznQ4u2DkC$iDy zFq#D9r~^SD+rmM;0J{p zcxbhlB4}Qe`=UsfXZ)kJGnXv?IYs|R=PX;$$rvFm^q1PZsai-0tx1Qhl7%!~MKe-Y zQt0JoEG2cyOehlxZOfI0%7P%C;_pRe*XIbS#8a-20_johoB>_2`T9%{#dpT0Jq1E)1P*%O}VAp7^fXJBZShk2`5{s}Ce6<5ggygp4 z>9W{mLfVJY&Ar8hbYVnqU-54tneq_CE!gk+BFx~H&ND8n;r*d(No zqZtWiH9`uIzLh1NCM5hm+jU;(C4P-av&}Xk)i~yMROurh`aW`io`{l zXzfh=-o0Csekbq)1LZ~G&);8Tm#;z7V9%U_%XZ$J@^n2#1c!!;jLOYMtyagg1Z63l!^?dFjHsOe;FR)#mj`Q2K{ z^Om!ai;FTuxOFJ1W-Nw55DnYh)cm|KpNDQgQU<$|6SmL2B-toETm|g>)i2QdbdYDGt6?0Vd2V&JHILb-=`|lR+Y<5NXyQ@$22d7(7n`SQ!TE-C zG;cr)B9uoOWJQNF(SIsSz~}v1&z9P`T=e0+Hsqqwc#F$a<)JeMOTo_EU$&=iA=K+4 zRsii-sc_Q#I34X)TnUuFNJ*!`3`R?k#o^NjrJ))lppA8p3Nb7{+T$>;EmhsdvgRb3Z^L4LeYku8?H&gzLu`o<0-@WN3nNr6)u_(%Jv(7m~j6nAS#xl z@!a6D2ba*KVj&K<&yOw>lDF51Ps^kPM8mc`h}Ag{t8F|24WrcrU%BvJK=PQ3-XaML zSRvsud2BHjZREaE^B?_IxswbZ8ml@< z7iXZamijQ-KQ8?aTWgr;%sZSUu1{V$qr^ggbk=}fi&6N(x_KPr+_DyC@aX6-+L;ap zL1Zj3uhnDkFDCk=Y5~z=CP5jy#ZYJZ?W%USBG$r#lU)q7kD@6UN_XeSG5>I zgo4EEih)5UdbV&Q*uCA7bb4|l8x6K^0+QK!>TwUBjeIT}gXSkIg|`S>AU-d&8u0y^n_!K{YZEuBm!T-6z{SsEUH%$604@K^A z2Tl9@pfeRK;2CT30G}b^Gb9SNIB49`3rJ&Q?o%UWHp<&|5WL{0LZ{xc(3F8U*e$Yk zOX+|$N`{IrM6Wcd?&0(@(6_EbV7F|^UlmdU8@)c~2bv$r4GI!ab;pT<{aKq7^TU;m zZY}T!yYw-}cj+J|`dJYGG!$TC)#%AUep1K4E<5izm8{P|llyTB9cfQImq$m`@gSIa z?#243*G z&4ULlbm$XKYG{0mDpdWfl}k9ujZsp2!l&x?QG(V*M<@k~7 zex{Ji8?|V-3mzLwyYXl|bnZ(uL|7l-y5v0Uh;X7}jtLsik}q+P``j2Ha^s#NZD$T@ zToVhLf4 zG96?U2}zLkvm*{0BUUg7B4MEl_uIb&G6|v(&jWrsfNRQQHqpzHqGdJFt!&)xWq?A$ zE;x6>iHt@6kyc{D+Xg52{;hj%S)~s5H4qi+Ej<{$396Gg(Xgx)wBCiV?paxe%QS5g z-nq%anMV0#u&d`(Z{Wb!LNzZLG;0HnlIFvDsdIA*Xx`LJRMDRWdy>Qyi0&)5x+4!( zGj-LeAUonZrRoH$yxnbSkmr+;8PQv%VDGy<9c0tv-9pl=IVjjS9qblIto82>p%X;H zbi9pAx-cd|6zqM=1_k~on;k204ad2B+OO)b&jBSY2|$g_%5__VH?jkXnK zz|2=m-Wl;e%tVpfGl8h%JPmnxW8y@^uD88fPzU>WmE$PhJNv?JLTDjMVT@ z>+p5(;?;iA^QI#g{ms7tnvSc#-I9fqpSHrAK>OSVt1b<(P^(lnM9BVT7nZ?>)5j&* zkY_sYdT;t}CP9!(jNi)mFfJ;)oDH*NtbR&X3+@S=s07u+NpI4RZ1CBfm7s2r!a{Fj za)8XDE#@V@W+4xkT$mU9?6WGEjUFw%4PLgE++$3@sstwz_IvFgZ3S4n;Y7w9BDc|w zcC!egV4df$s5q7g?5LP&@oI7(R8??I8n#k7WO0}-k097<*BmRlUj?V!mvNtsqw61S z|II+jDYqfdkLGSx-=QYAxHb=>_eaJ$h%baYgmnkTeyO-(Q1U9+?Ys-odwA<-#lbAC zm&*q)2{&aL`%e^8sf4T( z4nHLwy2C^!_bb8Yr4?<<0-+*-6BVn7wmrKz9@f{Hk3o~Cka*^6B@3N3tpZIEIdA_@ zKj{QPe+W=E)Ykm`t010cy|c~UXeKIsSPfoYY|oILZNovoK2(F3OC`Y;GoG>tqG5*T zzSYkv;3A19H89tjbHCd#y16Jzt`73V&rsMSEOg~}JzQytJbOLWJAAb3QUgS&2x_=| z0^Y5M^`Aj57shNq4Xe?S-LGfh#b@TRI&~HgF}Y3P#m9e*6&<=hzLhnBtYjnauN|9* zuAO-fv`6Xb#mx`>u_gL*1rZ#Kc%{vbx?ua^zAi7 zNa-;fJkI7KbJsQ)QKL^-w-cQF{5|v*ymXcfmCS}}jAqPR7~ixJ$uT_+bdmtU8q7Cp zk>?8z%DD3ue10A}G}a7V7*9X8LuUH#80#bo=*WKGdyp+;B~R{$-Srx^50DGKLfQbU zAX{z+Xx{(b)ZRXmK@ijc+cI8Vgsu;7^A3pTP;s@^9M-fsb3OviJH__ea~Sq`2R?$A z!!bwJ_dkHUp!Xwq!S`btin9nA1@qlOj<9+v$p1%i*S43KkMfa~_GgIa()L&1h(kw_>`oY; z-Y|X0H4@Hme{_N@s~|1bPlHYn1#69qc~RUakjZ1EYl5vT*YnZdBVRzXb+scmz!;v} z8C@{fT8$~{y?{tB8(lRMPZVV7ks)eRJQT<@zHoY9_~(*v3- z=C+KcmcZ^>q!;ETcXn*1(g)}u>ch!r>}Lgigo~P6zrlz)MV5_Ene!3pMIYELDVbG3 z+Q~wFv%bTeos|D{#VeFS5ZLWzd6XXKAaHd)@+XCXJ0U!XZRD(@ zYH>g}9gWT!1TXXdE)Mzy`{@j~LAWYb0eKWFR}PAr7z8`KE5ap{jS8v1LDPcyb-f|f z*>Qr?|2N;&9z(xIrpj;7OsJk?FI~e%Z{mML1h0PYG3o<4TJYdE_`J}WH=R3=i_V4( zftM!rLyz-CSg3Q|AJCMXIu?9;E(1xt`~zNUO~+`)Ob$}~FbtYACZrQjRWs3^CnG@3 zmdl!2nNW4890j}866!ld&(H}XW1EL~-J7A-h09>|v?2T6>y_{}9U23gsmBU6hIa^K z`Z##Os{p-+SxAFB4ziszr@;AYJm`#?fIM4Tj3s=bt7UokBwVq=+Hd(Tj(qgYa0+PO z+1ThbrY3qnJ_TOdm{{QO6cZImh>F4)JKABBew%^nZ-|Lfup^;%zYj>j&Zbx#=$Fq> zeh7t)ux4>lD)v)V{ZAp(m(Ijb1NyMx-fA9)i^MCZi^^m7yhT2LQR5>cof#m*&w74u z=c2TM84zKa<(0H@sAQ}=GZV;+@zFYTSU_-A{zQE{S(!t~sF)~oARgN)AR6}Be(x_y zSnu8`nkh=gbWS{T?OF+IjpkXRa#-fw`nn74bb`njb1X2)vLD``AcM-foJ5*4^g-NK zmIO`lpkK#s!kz>tc`VsrPqwEvAN97+hFthxkFE^;1?%1aq`=Pkj^Qkm%^Va^E+b0A zMxO*r9)K=3s^=W=Qup%xGM|sImjR*>=eIKAY^0Dj2QvFo^660OX%;f}kOiN|A8H?5 zxPpn+M$ZMGH|ldz+AX+fX}uh1zFz-VqalEWPK1#mf|Acw>%=@R>iS552p4@NlIGUa z38E19wGOOl#YCwfJK$O{Qge)sC@X0|J@z(le?OulF|IuL>~@!kVCX^JMpgl21FPpo z;Po?=9t9ZfHRkxVM-lLq=~M(6b5n0+4fN@7Ta-YS`BLxk#VI(c->3pvH{@q)ErUw% z{ZtjmdiwNro-U~z^!La-813KTxMvg7Ip}oXe6YK2x994Ua6W32Qw5SroY&Q`h>z?o z)F2m?(vb0IuV9z_YXQif2koc4O6H>+8+C|&|KqRRpOtjvbyOYV*$3*X9lyvx&t7Uk zJV(YF%ct!e)agZsc=f!v3J+-xN_A%dE&Ao0{KyD;HT#+1bJehE@K5L;)KFuC%vw$F zV|*&C{9D)%!GHX!RW%RRgZ3QA<%d@HC(dIyQF+XPxlUDjU$Ny5tZ6ws(4@FpH-3<$ zquFJA$i>0oai%d;2pp)3K+`j9Zs>Ry7rk7h1rh3Ah@LKa3jJ;qTHrI2GIC41n~7v< z7XxL#?x^#-$wGf^w86`b;ZLO*P^I!W(}4(;12roRTId9ku{x!u?QNim%cxkG5puJK z&H$X`v1rA&AEm$WQGOq8cOd+&!e!Vq<3z%^_K7w>;9kNB<|VyxMmb^=1S_kKL38nN zE$7&7roNOV6<*} zJ=5`8-ikV$q*iggr7D>O3A)1`!j33Wd<}Rhs20n5Ig5|Bty%+KW}a7g`V{U7@e6BV zcDtgnh`Ev+bZ?IVzf;NF>oBg%=?ou9-%rSzQ-@e@3 zAhVQ#)V7(x>}qLv(08OzDMP8`jJp= z+P7dEcu9@?NtV}RqC+MKsBQUO{`TceWcn3BF6&>AckVjFM~~KZX$s=Ym6?}X8|9Z()yTh2nK0_{NaeMg^j?mGtZ zklh92>-G2hay}UD7fA;oFPD%-hB|bDNSH19LsAN6{v(oaT}y@w^r?a- z^n;Cx``g1#11Aa=DWbP&Q;5I|6&tMco?8Q7AY4Yn@XEeDsRq z4Dm7(mL~j3p`$+gy)bsXFMTQ4Ex5W5znY!W=P$tPUU~qCffUhTCnpad@enfM}TGz==N-$Doh8@c=~U$t>No&X14Q z*SLe`#o-nAHT>cGSKiA^iOy>AA-yh#LiFJJcp`-#$nLR z(R$Y$v5t%OF7SgfT;T4pN`aHm>mQGR-D9nx=GrCDhimB%@gmC=zHfNXLz^oCK$f$^ zafj?E1C{R%1TSe_PCdby@N9k!gk1Dyj;-q%;iIA%$6*XX{T?x<|G_>$F$id6f9p?4 z2dGP_1%qb!{55sa%9`k@XE69|UsNXRVZ}tQDIq}k4rR$qCYnwUgXs8^lb+2*Q6*sz z4?n|P@`Hnvd6}9xpTpJ--x$(Yloemugph?04 z7oS+V^fGiX5R_*|zo5BLNHS8|!+wqeO)~Z|%g12p4CrnEO)@r=`zxWhS4gxP z^6i}xhmgTZ(^TY>X;6E@?cgas*>G=dGLIm64DYKHhrZwB>$t2@G{|r@^vp8qqaZrt z zY`?~XEbE21mlBzS%KZ~y=KZ*Rh8+m{=Y21LZ0nT=N9&+ZM`H@7_|0xf(Hl6ZwDclq zYRZNd?D@z+o?=Om=lu|R!@je8RCYQEB3N8vlxzrP5+si$+L(0yfqOph`z4U6))j6D z3x^X_AbIRytnS*z*?d&-@iNGEB=0+G^q7G*olXH#7d6g^>4!6-fmDdckz3Of3bmQT z!)cIVH$ zs;VYK*BmZ`9sBcbDjUaG1W~a)oWV_bhq(mNu(XEh6_Hn=%D6TYqIXlvM?Y=npn8w1 zK-#K9f2To({M6_*@PePx?16Ji+k06+?5W0!S=*pKF`5O@t<1N-dXr8^H`}j6W_Zt# z2y{j0th)hOfA;UR{ys|+g{a*GyC=>g*k$OV3Aut3{r3r!cHX7XC8O z$p+F`k$7bPO*WdbD+jz3Y4CJzTJTZF$y|sq9Ef4tp#o|7F&Fa0_fGA{V5NHVHprY! zSC$XM&PH}y9?0T+$J9u_IVk+Y9gq#Z+ntNA-PR@C1=+@nerg@sP!~?f2fACJJ0_;X zMQwWvfG*nA%-jY2y8?>(Yj=KXl0y*eLay!CMbWS;2TdB$@12vwLq{$aK%P|Td7Am~ z*GB*A0vN+^TAW<#H&`dD6~Zh{ZNB3j_KS`}*o9zcwRz%XPc;{v7%KvLQl56FZ3Pd# z2)GA?zXwxod1!`pG30VJT&Jlaii6_klz^9Om_xtkB|1Sg?BhZoXKn-!4cL}Kykya~ zixKCbYiIL)7;RiT=hvwO2J)hmf!(VYuk}piV66`X&wKWx#gAvR2!g*8CYKtmX=4)v zt5~HHBi(ckK{QNyQhiI02#+AB1bfnUwZa#@AhHZH^D;ZGoe~6fKEHDCdBNeklP~PK zR%$+g=zB*eZFAP%MpclRduT~loiP`gJg9=JvfBQhqFx3c9XeeNJ}*g!y_Q@Ot2<5l zV0Q*Y!w%Y}s(jweL-FP{;1geo$7#R`&f;3o?EJJOjv_|JN|~rv_Tzy7l?!{`NPZ={#$oVj6T1Sqc)4ivsOZ!M77B9z0J61Z1D_tO z4Im#yn#p^<7_p-d=R@It{&rgNa{Y=0zjE%iEshAZAz>*rjP_%gtT{_u!7N zFotfq9r-c-OjL5S2jbm}MwX7oJhXGN2V}Df-?=#XFbRSR0mb=9-FG%YG@{ecAD$dp z^ zUmR4vfS@l3H2YHp1gBS^xo#<(&GWy(OgxOKjEX(ZMjo|&5S=d>wBQx=N9I-ZgRE;4@HoD*c1Fi$!r?W4Lk1{^|1kLUzQZ7gDvygt_FNkNg`0I($TO368`42SxzOUp& zJ>jF!=>tFq=5jPwUWTus=>TLs+wSb!U2p;Z_rIj+wc?zvIc=}De7-EuV*9b!$Ht=;x?w+!XFl1D~7Wr3!=kL1I{h(!q>2&0Jn=W zNG=Fp%to;XM`omLoXcXf8cCvYLasvxp>l%fz<8`UwwQ!2~XAG$2Wnsa3 zX+Fx9`3vKlrtY6O*#duX>HY;f{11y;1x&Q6_b-e=Zjnv8E9}m4NB%-)7p+6OKAzy9 zXW`>O>yxsU|IXl`3z_4P3*NcX0Au(_nE)@wy~|p|*Rqi3-3f@WiL_5w@+7>bDU5*8PU4iqHu$jQ5C{Da>=Cba diff --git a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/format_version.txt b/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/format_version.txt deleted file mode 100644 index 56a6051ca2b..00000000000 --- a/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64/format_version.txt +++ /dev/null @@ -1 +0,0 @@ -1 \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/checksums.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/checksums.txt deleted file mode 100644 index a72e2bdb3706bcc08ec2f7d6ff976c10e56f75bd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 260 zcmV+f0sH=AXk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$cMSjn%S#tikUGF#tP8}kE# z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOBB}>nT0?1>@KmfG%gdU6%j|WMOn+E@EkJxd)

ZggK{VRU6KX=Hc=Yhv{9L4)i+mY_Q2(jfMG01a?q za&&2QX>V>WWMOm!_Mvet<#01I$(X>DP0c`j*Wcnq1^RmKF(8EcDh KC_ym>w$1=Amu)2g diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/columns.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/columns.txt deleted file mode 100644 index 193d3123896..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/columns.txt +++ /dev/null @@ -1,12 +0,0 @@ -columns format version: 1 -10 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`timestamp_ns` UInt64 -`revision` UInt32 -`trace_type` Enum8('Real' = 0, 'CPU' = 1, 'Memory' = 2, 'MemorySample' = 3) -`thread_id` UInt64 -`query_id` String -`trace` Array(UInt64) -`size` Int64 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/count.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/count.txt deleted file mode 100644 index 7813681f5b4..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/count.txt +++ /dev/null @@ -1 +0,0 @@ -5 \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/default_compression_codec.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/minmax_event_date.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/minmax_event_date.idx deleted file mode 100644 index 4b6c49f8d50..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/partition.dat b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/partition.dat deleted file mode 100644 index b9ea5569e3a3b34e3a1b6a1c4cbcb46f5e2d3fac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 Lcmb0VWo7^X0I>iw diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/primary.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/primary.idx deleted file mode 100644 index 99ab3ce7835..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202010_1_4_1/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H-_H!._ \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/checksums.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/checksums.txt deleted file mode 100644 index 49d83aebd914bce7c8ebb35018eb910ea5337b95..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 260 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyUJ56aG_hdC|uFP3IrAaDDV{y1~G} za1Mw++Olyb=a=S{=#^BIFy_n&-TQLZLq37-DI9oG#gHUryHuyxGq-0z*~@5 zRFYW&bhjQ*5z9{=eeu7Cnoj8~>F6<*FiB_NE-1>(O)RPe>E>AzQ2A(+8nc=5L>~VI Iw&B?f0H_RU;s5{u diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/columns.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/columns.txt deleted file mode 100644 index 193d3123896..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/columns.txt +++ /dev/null @@ -1,12 +0,0 @@ -columns format version: 1 -10 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`timestamp_ns` UInt64 -`revision` UInt32 -`trace_type` Enum8('Real' = 0, 'CPU' = 1, 'Memory' = 2, 'MemorySample' = 3) -`thread_id` UInt64 -`query_id` String -`trace` Array(UInt64) -`size` Int64 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/count.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/count.txt deleted file mode 100644 index 56a6051ca2b..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/count.txt +++ /dev/null @@ -1 +0,0 @@ -1 \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/default_compression_codec.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/minmax_event_date.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/minmax_event_date.idx deleted file mode 100644 index d3715e50d46..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/partition.dat b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/partition.dat deleted file mode 100644 index 7af4421fcb5ebc50aa06a5f4dee6e454020552a6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 Lcmb0UWo7^X0J#7) diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/primary.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/primary.idx deleted file mode 100644 index 0156d831c6a..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_35_35_0/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H_H_ \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/checksums.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/checksums.txt deleted file mode 100644 index f45270c50bd34e0f4400456a9a0ac4304ebae307..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 261 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyXp3#UiEg{lBXdkG?GYv@f%}=_UgM z!+9Y7XvfBxoL`z(qE}K;!W{Rh;=-DOtwxG>m6dsRwjN>NNJ%V7)Jw|DJCdSa{OCi# zm(z|B#fI`%l-4CPUb)Y{Vnx12R!7vt_w^?Z&Do&ziBq59LtbuCw($j~;%~BCO{aYv z_O55y$Dp`Sl0h^#GcPx>B0jY&HLoNdXl|-rW=aK%{)#M*%!oA;KkjQ)eIr+F%D`KY zSX7c(0(7_@P!Y>d9)0n@hni05Ea~VmmM}?Y;4Uc2%uOt+1nK5c-7Q?0BT~9Ltm4Vj Jxg9xO3;;OHZSw#C diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/columns.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/columns.txt deleted file mode 100644 index 193d3123896..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/columns.txt +++ /dev/null @@ -1,12 +0,0 @@ -columns format version: 1 -10 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`timestamp_ns` UInt64 -`revision` UInt32 -`trace_type` Enum8('Real' = 0, 'CPU' = 1, 'Memory' = 2, 'MemorySample' = 3) -`thread_id` UInt64 -`query_id` String -`trace` Array(UInt64) -`size` Int64 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/count.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/count.txt deleted file mode 100644 index a21cae3648e..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/count.txt +++ /dev/null @@ -1 +0,0 @@ -454 \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/default_compression_codec.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/minmax_event_date.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/minmax_event_date.idx deleted file mode 100644 index 4fedf1265b2..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/partition.dat b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/partition.dat deleted file mode 100644 index 7af4421fcb5ebc50aa06a5f4dee6e454020552a6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 Lcmb0UWo7^X0J#7) diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/primary.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/primary.idx deleted file mode 100644 index 698b75ec43a..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202012_5_34_6/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H_Hƥ_ \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/checksums.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/checksums.txt deleted file mode 100644 index 5d9d47463d2dc3c588c2a68beb24126184e5955d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 262 zcmV+h0r~!8Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$cvWrlG#xSW(BO;NigHQ+OX z+5i9m&;S4c@jnI$V{dhCbS`vwbOb;rV4Zj{6Wu0P+wm(Y%$fiQWMOn+E@EkJsoeoZ z83XkA_OM)y9%i<(>8`Z_;I@6CZZ2eDbOfQp0V$_ea0t>|RE;3Dk?sHsaB^vFVRCscX=Hc|1aaTY#PfTb M%D$$}R*TMQ0I^wb#{d8T diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/columns.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/columns.txt deleted file mode 100644 index 193d3123896..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/columns.txt +++ /dev/null @@ -1,12 +0,0 @@ -columns format version: 1 -10 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`timestamp_ns` UInt64 -`revision` UInt32 -`trace_type` Enum8('Real' = 0, 'CPU' = 1, 'Memory' = 2, 'MemorySample' = 3) -`thread_id` UInt64 -`query_id` String -`trace` Array(UInt64) -`size` Int64 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/count.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/count.txt deleted file mode 100644 index 0ac440f96a1..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/count.txt +++ /dev/null @@ -1 +0,0 @@ -1378 \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/default_compression_codec.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/minmax_event_date.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/minmax_event_date.idx deleted file mode 100644 index 4c89b29a856..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/partition.dat b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/partition.dat deleted file mode 100644 index a713aecd075daae7071deceaa93147871ef28fbf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmXRFWo7^X0v!O3 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/primary.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/primary.idx deleted file mode 100644 index 6180f063fac..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202101_36_115_17/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H_Hj ` \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/checksums.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/checksums.txt deleted file mode 100644 index b91b7faa4f06a1664ebb74109586cac5fd23a1bf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 261 zcmV+g0s8)9Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$bE+#qTQ)Lu{YCP+gXf%IE~ z*#H0l&j0`b@jeC#V{dhCbS`vwbOU(kK>xQg&QB-oQa%ok*L?s8WMOn+E@EkJi8YXH zVhIrlaZ>%T4I&riEH(kO-vz`k)u>G8-l<@lxx5?vo?Zzr0Pt>Ya%(ft0?3Luk)Hm4 z)H$e;_u%sC>HrmOX>M&{cwc39Wo~p|WMOn=E@@2I=Kk=xD!%mjDZJa%pX0a(OOkWOxk4<8|vG32}9| Lb!HVccCGgS;Baoy diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/columns.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/columns.txt deleted file mode 100644 index 193d3123896..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/columns.txt +++ /dev/null @@ -1,12 +0,0 @@ -columns format version: 1 -10 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`timestamp_ns` UInt64 -`revision` UInt32 -`trace_type` Enum8('Real' = 0, 'CPU' = 1, 'Memory' = 2, 'MemorySample' = 3) -`thread_id` UInt64 -`query_id` String -`trace` Array(UInt64) -`size` Int64 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/count.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/count.txt deleted file mode 100644 index 0e92c3c0fc9..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/count.txt +++ /dev/null @@ -1 +0,0 @@ -306 \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/default_compression_codec.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/minmax_event_date.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/minmax_event_date.idx deleted file mode 100644 index 0e3b7fb75b8..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/partition.dat b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/partition.dat deleted file mode 100644 index da540c2d1a825be3ef80d13baa834834b2248491..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmXRDWo7^X0wDm8 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/primary.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/primary.idx deleted file mode 100644 index b8d049dbb19..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_116_127_3/primary.idx +++ /dev/null @@ -1 +0,0 @@ -HC!`H"` \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/checksums.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/checksums.txt deleted file mode 100644 index f448a75b6063fba410c431980059309d4b9a6aa1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 260 zcmV+f0sH=AXk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$a$Hb&?2dEW#ESpcY+$ktzi z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOCIfRlVt&lYFUDR9N&B1^)mDWMOn+E@EkJ_5<}3 z3}5ITzipTtZggK{VRU6KX=Hc=VZjEVxwC1aJrFfB;*n}Y01a?q za&&2QX>V>WWMOm!K^bOwf>Qhj>E&4HXupM*01I$(X>DP0c`j*WcnsN(E!5h!GNyDe K$;I8+;Fkb~3T$=& diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/columns.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/columns.txt deleted file mode 100644 index 193d3123896..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/columns.txt +++ /dev/null @@ -1,12 +0,0 @@ -columns format version: 1 -10 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`timestamp_ns` UInt64 -`revision` UInt32 -`trace_type` Enum8('Real' = 0, 'CPU' = 1, 'Memory' = 2, 'MemorySample' = 3) -`thread_id` UInt64 -`query_id` String -`trace` Array(UInt64) -`size` Int64 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/count.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/count.txt deleted file mode 100644 index 56a6051ca2b..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/count.txt +++ /dev/null @@ -1 +0,0 @@ -1 \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/default_compression_codec.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/minmax_event_date.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/minmax_event_date.idx deleted file mode 100644 index 16cc6680505..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/partition.dat b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/partition.dat deleted file mode 100644 index da540c2d1a825be3ef80d13baa834834b2248491..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmXRDWo7^X0wDm8 diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/primary.idx b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/primary.idx deleted file mode 100644 index 9f363c02b40..00000000000 --- a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_128_128_0/primary.idx +++ /dev/null @@ -1 +0,0 @@ -HG"`HG"` \ No newline at end of file diff --git a/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_129_129_0/checksums.txt b/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a/202102_129_129_0/checksums.txt deleted file mode 100644 index d331cb1fcc1616a320c584208bbbe060c20c1947..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 261 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyV5)|94mWJa#3U_`UDuPj{Tybd!OB z;XDw3v}5B;&M(a?(JQGaVLEni-X{0LThekT`8{LRP8u?Bq$HLk>Lq37bqTze&kSbS z^7D+CCQnYxcHy^-OLj7TynlAv5BGeAS4nH{ichgE;?!sOke6GOZG3^LKys6eD>t{J z*eAbZOH(qk8ANk4^Kug_;#12~^Gf1@=BDaprc|&b9%Ngvb93e*TLDw!M-#K08F&j4 zi%K#}fDYFKDq?YzOsi}P`os3}X~c_+{jJj(xC@Fha}$dyLArVRV>iAT$d33qg;0jf$7<_1PzbQF>&8 z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOPvZe4NH(<2?+h%j6?;@(ut9WMOn+E@EkJgdK+V z5ZhW2+=>to#{T20Lq-9{RA`+*_6V{cl3!Qme$Qq=nF%fc@NR8#Ycuc(P`_DAmgVQ_ z!Oxz{GfIzl02OU%Zf#+BUuAY>ZggK{VRU6KX=Hc=Yhv{9L4)i+mY_Q2(jfMG01a?q za&&2QX>V>WWMOm!_Mvet<#01I$(X>DP0c`j*Wcns^T1OQK`i$9(N KipbM-MdtvM<#0p* diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/columns.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/columns.txt deleted file mode 100644 index 532baa6cba6..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/columns.txt +++ /dev/null @@ -1,41 +0,0 @@ -columns format version: 1 -39 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`query_start_time` DateTime -`query_start_time_microseconds` DateTime64(6) -`query_duration_ms` UInt64 -`read_rows` UInt64 -`read_bytes` UInt64 -`written_rows` UInt64 -`written_bytes` UInt64 -`memory_usage` Int64 -`peak_memory_usage` Int64 -`thread_name` String -`thread_id` UInt64 -`master_thread_id` UInt64 -`query` String -`is_initial_query` UInt8 -`user` String -`query_id` String -`address` IPv6 -`port` UInt16 -`initial_user` String -`initial_query_id` String -`initial_address` IPv6 -`initial_port` UInt16 -`interface` UInt8 -`os_user` String -`client_hostname` String -`client_name` String -`client_revision` UInt32 -`client_version_major` UInt32 -`client_version_minor` UInt32 -`client_version_patch` UInt32 -`http_method` UInt8 -`http_user_agent` String -`quota_key` String -`revision` UInt32 -`ProfileEvents.Names` Array(String) -`ProfileEvents.Values` Array(UInt64) diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/count.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/count.txt deleted file mode 100644 index 3cacc0b93c9..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/count.txt +++ /dev/null @@ -1 +0,0 @@ -12 \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/default_compression_codec.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/minmax_event_date.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/minmax_event_date.idx deleted file mode 100644 index 4b6c49f8d50..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/partition.dat b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/partition.dat deleted file mode 100644 index b9ea5569e3a3b34e3a1b6a1c4cbcb46f5e2d3fac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 Lcmb0VWo7^X0I>iw diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/primary.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/primary.idx deleted file mode 100644 index d599340cc82..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202010_1_5_1/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H!._Hm._ \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/checksums.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/checksums.txt deleted file mode 100644 index 4398dc6a7abbe10202dcc9ff464b250d9453cfcf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 262 zcmV+h0r~!8Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$aI-o22|e$OHz$TackS;rWH z+5i9m&;S4c@jnI$V{dhCbS`vwbOTsTaBK6JI5}pr1~GpWy)OU=WMOn+E@EkJsmK8g z4bfp!g8FEMWeVZj)V}Wlv(^fr`M!nXj-ryTzn4HBVbGchFaYpwZE|Zf@CkwPt9t01 zsfP|9Ri`pFb>9FLZE0?8VR&C@6CZZ2eDbOiYfFBkp7g32tUh>SBBG-m({aB^vFVRCscX=Hc|vV`+0{_>ZK M%eyNa13TFQ07qJGTL1t6 diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/columns.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/columns.txt deleted file mode 100644 index 532baa6cba6..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/columns.txt +++ /dev/null @@ -1,41 +0,0 @@ -columns format version: 1 -39 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`query_start_time` DateTime -`query_start_time_microseconds` DateTime64(6) -`query_duration_ms` UInt64 -`read_rows` UInt64 -`read_bytes` UInt64 -`written_rows` UInt64 -`written_bytes` UInt64 -`memory_usage` Int64 -`peak_memory_usage` Int64 -`thread_name` String -`thread_id` UInt64 -`master_thread_id` UInt64 -`query` String -`is_initial_query` UInt8 -`user` String -`query_id` String -`address` IPv6 -`port` UInt16 -`initial_user` String -`initial_query_id` String -`initial_address` IPv6 -`initial_port` UInt16 -`interface` UInt8 -`os_user` String -`client_hostname` String -`client_name` String -`client_revision` UInt32 -`client_version_major` UInt32 -`client_version_minor` UInt32 -`client_version_patch` UInt32 -`http_method` UInt8 -`http_user_agent` String -`quota_key` String -`revision` UInt32 -`ProfileEvents.Names` Array(String) -`ProfileEvents.Values` Array(UInt64) diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/count.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/count.txt deleted file mode 100644 index cbd6012bc6f..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/count.txt +++ /dev/null @@ -1 +0,0 @@ -204 \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/default_compression_codec.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/minmax_event_date.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/minmax_event_date.idx deleted file mode 100644 index 4fedf1265b2..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/partition.dat b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/partition.dat deleted file mode 100644 index 7af4421fcb5ebc50aa06a5f4dee6e454020552a6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 Lcmb0UWo7^X0J#7) diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/primary.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/primary.idx deleted file mode 100644 index 7f33001d913..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202012_6_45_8/primary.idx +++ /dev/null @@ -1 +0,0 @@ -Hb_H_ \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/checksums.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/checksums.txt deleted file mode 100644 index cc8c216e1b7494e413c0da9a5b31486c5d90c403..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 262 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyWf3AW-@%fo-?_t?&Cpwzzz3y2Zf2 zZ~=%v+Ou&c=a=S{=#^BIFgFFyXHxB3fAoMC=byk-#h(luDTyVCdP$ji9rGB|rzA|U zJX$kL_GQ{`uZTS%j8ms`sB}#X6EH4R>Y^4rRJ5y0}W2q%S@?YIU{wz_k+a5P6o-x>)Eq|k1_BT zBo>uqmH=I@2UNte@Gzt1@~{GqOEDq63R@?>W8f|*%FInHss!ogDczYWTxYG5;(xo& KrZ}f&2Lk}qd}`$Y diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/columns.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/columns.txt deleted file mode 100644 index 532baa6cba6..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/columns.txt +++ /dev/null @@ -1,41 +0,0 @@ -columns format version: 1 -39 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`query_start_time` DateTime -`query_start_time_microseconds` DateTime64(6) -`query_duration_ms` UInt64 -`read_rows` UInt64 -`read_bytes` UInt64 -`written_rows` UInt64 -`written_bytes` UInt64 -`memory_usage` Int64 -`peak_memory_usage` Int64 -`thread_name` String -`thread_id` UInt64 -`master_thread_id` UInt64 -`query` String -`is_initial_query` UInt8 -`user` String -`query_id` String -`address` IPv6 -`port` UInt16 -`initial_user` String -`initial_query_id` String -`initial_address` IPv6 -`initial_port` UInt16 -`interface` UInt8 -`os_user` String -`client_hostname` String -`client_name` String -`client_revision` UInt32 -`client_version_major` UInt32 -`client_version_minor` UInt32 -`client_version_patch` UInt32 -`http_method` UInt8 -`http_user_agent` String -`quota_key` String -`revision` UInt32 -`ProfileEvents.Names` Array(String) -`ProfileEvents.Values` Array(UInt64) diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/count.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/count.txt deleted file mode 100644 index aa59885c836..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/count.txt +++ /dev/null @@ -1 +0,0 @@ -143 \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/default_compression_codec.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/minmax_event_date.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/minmax_event_date.idx deleted file mode 100644 index 4c89b29a856..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/partition.dat b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/partition.dat deleted file mode 100644 index a713aecd075daae7071deceaa93147871ef28fbf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmXRFWo7^X0v!O3 diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/primary.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/primary.idx deleted file mode 100644 index f166214418a..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202101_46_72_6/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H_H ` \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/checksums.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/checksums.txt deleted file mode 100644 index 0921ff99748c1a04a7fd95cb8943ecfec846c1a0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 261 zcmV+g0s8)9Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$aAz*8X&k@J-6kO<{X=LkxI z*#H0l&j0`b@jeC#V{dhCbS`vwbOJ{BwXp-jZa9E;ck*GPUV8utWMOn+E@EkJ+%Aqj zHZie?gISC4ZtT~N;D7=A*a4?auCebz5f>&NuopMmmh}lQ0Pt>Ya%(g23Fi<6sEDe~ zu$pGM;@RW~KL8bNX>M&{cwc39Wo~p|WMOn=E@@2I=Kk=xD!%mjDZJa%pX0a(OOkWOxh%C+Pf(wa}}N LsJMgcdn4umo7ixJ diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/columns.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/columns.txt deleted file mode 100644 index 532baa6cba6..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/columns.txt +++ /dev/null @@ -1,41 +0,0 @@ -columns format version: 1 -39 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`query_start_time` DateTime -`query_start_time_microseconds` DateTime64(6) -`query_duration_ms` UInt64 -`read_rows` UInt64 -`read_bytes` UInt64 -`written_rows` UInt64 -`written_bytes` UInt64 -`memory_usage` Int64 -`peak_memory_usage` Int64 -`thread_name` String -`thread_id` UInt64 -`master_thread_id` UInt64 -`query` String -`is_initial_query` UInt8 -`user` String -`query_id` String -`address` IPv6 -`port` UInt16 -`initial_user` String -`initial_query_id` String -`initial_address` IPv6 -`initial_port` UInt16 -`interface` UInt8 -`os_user` String -`client_hostname` String -`client_name` String -`client_revision` UInt32 -`client_version_major` UInt32 -`client_version_minor` UInt32 -`client_version_patch` UInt32 -`http_method` UInt8 -`http_user_agent` String -`quota_key` String -`revision` UInt32 -`ProfileEvents.Names` Array(String) -`ProfileEvents.Values` Array(UInt64) diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/count.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/count.txt deleted file mode 100644 index 978b4e8e518..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/count.txt +++ /dev/null @@ -1 +0,0 @@ -26 \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/default_compression_codec.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/minmax_event_date.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/minmax_event_date.idx deleted file mode 100644 index 0e3b7fb75b8..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/partition.dat b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/partition.dat deleted file mode 100644 index da540c2d1a825be3ef80d13baa834834b2248491..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmXRDWo7^X0wDm8 diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/primary.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/primary.idx deleted file mode 100644 index 67f45d29597..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_73_77_1/primary.idx +++ /dev/null @@ -1 +0,0 @@ -HC!`HG"` \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/checksums.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/checksums.txt deleted file mode 100644 index 2c30b88770877a5ddaa75c2f3daf2dbc321be19f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 260 zcmV+f0sH=AXk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$cbNRM_*!6JF+5+jxa^3W%O z*Z=?k&Hw-a@jV6!V{dhCbS`vwbON#VDF3f}{FN>b+}S`mO_BfzWMOn+E@EkJz#hw3 zOT%v^;N`ZYj_KCHG`s=1SwsOGv?YnRsHJA*@;B-H;|VSR@NR8#Ycuc(Kw~dMnFgB~ zmXR7wn|S>302OU%Zf#+BUuAY>ZggK{VRU6KX=Hc=VZjEVxwC1aJrFfB;*n}Y01a?q za&&2QX>V>WWMOm!K^bOwf>Qhj>E&4HXupM*01I$(X>DP0c`j*Wcnl8Xz`TJd7NP!7 K53t!rdd~o;%5H-I diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/columns.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/columns.txt deleted file mode 100644 index 532baa6cba6..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/columns.txt +++ /dev/null @@ -1,41 +0,0 @@ -columns format version: 1 -39 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`query_start_time` DateTime -`query_start_time_microseconds` DateTime64(6) -`query_duration_ms` UInt64 -`read_rows` UInt64 -`read_bytes` UInt64 -`written_rows` UInt64 -`written_bytes` UInt64 -`memory_usage` Int64 -`peak_memory_usage` Int64 -`thread_name` String -`thread_id` UInt64 -`master_thread_id` UInt64 -`query` String -`is_initial_query` UInt8 -`user` String -`query_id` String -`address` IPv6 -`port` UInt16 -`initial_user` String -`initial_query_id` String -`initial_address` IPv6 -`initial_port` UInt16 -`interface` UInt8 -`os_user` String -`client_hostname` String -`client_name` String -`client_revision` UInt32 -`client_version_major` UInt32 -`client_version_minor` UInt32 -`client_version_patch` UInt32 -`http_method` UInt8 -`http_user_agent` String -`quota_key` String -`revision` UInt32 -`ProfileEvents.Names` Array(String) -`ProfileEvents.Values` Array(UInt64) diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/count.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/count.txt deleted file mode 100644 index 9a037142aa3..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/count.txt +++ /dev/null @@ -1 +0,0 @@ -10 \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/default_compression_codec.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/minmax_event_date.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/minmax_event_date.idx deleted file mode 100644 index 16cc6680505..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/partition.dat b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/partition.dat deleted file mode 100644 index da540c2d1a825be3ef80d13baa834834b2248491..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmXRDWo7^X0wDm8 diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/primary.idx b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/primary.idx deleted file mode 100644 index 8f3c7c8809e..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/202102_78_78_0/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H"`H"` \ No newline at end of file diff --git a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/format_version.txt b/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/format_version.txt deleted file mode 100644 index 56a6051ca2b..00000000000 --- a/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc/format_version.txt +++ /dev/null @@ -1 +0,0 @@ -1 \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/checksums.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/checksums.txt deleted file mode 100644 index e9f3f5dbfb1363c5743ef536fa11535cbb312446..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 262 zcmV+h0r~!8Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$c>P9c|kkD-grFaYpwZE|Zfumc7>wrI~> z!G@D=hNwi5yodl5ZE0?8VR&C_3*EI_1(J_Im&g zaA9(EX>@6CZZ2eDbOiRJGt`jAL&ph4JQlLni7WsMaB^vFVRCscX=Hc|qQj~+&$gS+ M9BY!1)str&0ESs^mH+?% diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/columns.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/columns.txt deleted file mode 100644 index 37cb59062b4..00000000000 --- a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/columns.txt +++ /dev/null @@ -1,15 +0,0 @@ -columns format version: 1 -13 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`microseconds` UInt32 -`thread_name` LowCardinality(String) -`thread_id` UInt64 -`level` Enum8('Fatal' = 1, 'Critical' = 2, 'Error' = 3, 'Warning' = 4, 'Notice' = 5, 'Information' = 6, 'Debug' = 7, 'Trace' = 8) -`query_id` String -`logger_name` LowCardinality(String) -`message` String -`revision` UInt32 -`source_file` LowCardinality(String) -`source_line` UInt64 diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/count.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/count.txt deleted file mode 100644 index 19afd4ebd92..00000000000 --- a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/count.txt +++ /dev/null @@ -1 +0,0 @@ -527 \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/default_compression_codec.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/minmax_event_date.idx b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/minmax_event_date.idx deleted file mode 100644 index 4b6c49f8d50..00000000000 --- a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/partition.dat b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/partition.dat deleted file mode 100644 index b9ea5569e3a3b34e3a1b6a1c4cbcb46f5e2d3fac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 Lcmb0VWo7^X0I>iw diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/primary.idx b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/primary.idx deleted file mode 100644 index 714ba773856..00000000000 --- a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202010_1_29_6/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H-_H._ \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_2358_2361_1/checksums.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202012_2358_2361_1/checksums.txt deleted file mode 100644 index e0dd83d152b7ad470cb7e6dc01063da79250d71e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 260 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyYGj&^g1ob@95*pY+^0H=Su~y1~G} za1Mw++Olyb=a=S{=#^BIFijMHvvzKnnEIOFX9gz}^Zqk%q$HLk>Lq37{ZYO&OJI4M zjCAP3lOg(izXaSEH$*n5CK&i9%nMq!Y`X9HpGOaI>M?xC%Pq<_-oU(R)d$Ag!s`2H zEpFbZu;!i*0UP(OA*i^mDlnNGyw=9*XUwBkGJZ(#GEfAi{z*~@5 zRFYW&bhjQ*5z9{=eeu7Cnoj8~>F6<*FiB_NE-1>(O)RPe>E?-ZOk)-2I#_?b=Uju7Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$b_qY0(p<%vUw$Fz9=x*t@6 z+W-In(EtDd@jwO%V{dhCbS`vwbOp+-iJ0^6C!0o57CJpSfd~KyWMOn+E@EkJ)Phz1 z%k1aRq;w4zWD2{8chZf$aFGnfz4lqGhN z^F@@yAM#diPMbsk6>Vv5ZDDv{Wp-t5bYEm)bY(7SWOxKobNpT`{e-ssQD+6%72KTw z4RB#{bZK;HZ*DGRVRQuf3@;b`!h*^yq=<|&7&K=93vhC2ZDDeGE@@u7Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$aUySV8>?*YX%1ymm|vajoc z+W-In(EtDd@jwO%V{dhCbS`vwbOk!)l;t<$46HHJYX_ISYDxeIWMOn+E@EkJvza1} zQyBilEl5*9x)JNQ3ivDm(!!zvxK+z2v32q!r+DVSDoo(32{8chZf$aFGvEe@okB*~ zbTR%3Y-@*T4j5|y6>Vv5ZDDv{Wp-t5bYEm)bY(7SWOxM38o*8P7?Ftp8RM@9Yg5Jm z4RB#{bZK;HZ*DGRVRQtc!vQI$R&WT?TvUx9wUO=s3vhC2ZDDeGE@@ diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/checksums.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/checksums.txt deleted file mode 100644 index 7c01d242d7ed0b91099869d3ec8d9393edff1e66..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 260 zcmV+f0sH=AXk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$b;Du41;5Zw_V+h#FX$e?_J z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOOj1`)FV*dCjdq%k=Y^lq&!TWMOn+E@EkJ%pskK z4+T*^fU(%s4!?yZDEI--N?tRe=#`t*XPo$DrKeG@MhPwe@NR8#YcsF|50od+F4&OI z%2q0?M&%lG02OU%Zf#+BUuAY>ZggK{VRU6KX=Hc=%oGj0*JiNNvOS&DmLLRW01a?q za&&2QX>V>WWMOm!p~C?wr&e$X(p*%HAhnV101I$(X>DP0c`j*WcntT(8?D{|1xXDg K$kqa`ii-e@x@&s? diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/columns.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/columns.txt deleted file mode 100644 index 37cb59062b4..00000000000 --- a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/columns.txt +++ /dev/null @@ -1,15 +0,0 @@ -columns format version: 1 -13 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`microseconds` UInt32 -`thread_name` LowCardinality(String) -`thread_id` UInt64 -`level` Enum8('Fatal' = 1, 'Critical' = 2, 'Error' = 3, 'Warning' = 4, 'Notice' = 5, 'Information' = 6, 'Debug' = 7, 'Trace' = 8) -`query_id` String -`logger_name` LowCardinality(String) -`message` String -`revision` UInt32 -`source_file` LowCardinality(String) -`source_line` UInt64 diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/count.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/count.txt deleted file mode 100644 index b2412e34dff..00000000000 --- a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/count.txt +++ /dev/null @@ -1 +0,0 @@ -62 \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/default_compression_codec.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/minmax_event_date.idx b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/minmax_event_date.idx deleted file mode 100644 index 76b5229cbee..00000000000 --- a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/partition.dat b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/partition.dat deleted file mode 100644 index a713aecd075daae7071deceaa93147871ef28fbf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmXRFWo7^X0v!O3 diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/primary.idx b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/primary.idx deleted file mode 100644 index e19de0711bb..00000000000 --- a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202101_3537_3539_1/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H `H ` \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_3540_4637_415/checksums.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_3540_4637_415/checksums.txt deleted file mode 100644 index 6ff207deeb3ac1e0093ae21d37c1b81c3a9d2d5b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 263 zcmV+i0r>u7Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$cnp1>605;0weWScx&VIRYS z+W-In(EtDd@jwO%V{dhCbS`vwbOlE0NoHh|IjQ9=nBu&yL8kx+WMOn+E@EkJ-sK!V z{>Uu-LH|6^^@SRDj@mx~`{{rIlayO075KqB0u)m?sefCr2{8chZf$aFGvEeBYTdjB z7Ei**<+~!{Vky7?6>Vv5ZDDv{Wp-t5bYEm)bY(7SWOxJvIhh$>@v-l*wq3;?bvV`l z4RB#{bZK;HZ*DGRVRQsR8D@EcQv3$#iBAM}yn N|2B1HzutRv6aZJdZV>iAT$b7Fo=lt*g5R6KtA;xU-PVj z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOH&t)oZggK{VRU6KX=Hc=VZjEVxwC1aJrFfB;*n}Y01a?q za&&2QX>V>WWMOm!K^bOwf>Qhj>E&4HXupM*01I$(X>DP0c`j*WcnlzQ8dORAdLOmq K{L?d@g`fZn8gaG& diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/columns.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/columns.txt deleted file mode 100644 index 37cb59062b4..00000000000 --- a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/columns.txt +++ /dev/null @@ -1,15 +0,0 @@ -columns format version: 1 -13 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`microseconds` UInt32 -`thread_name` LowCardinality(String) -`thread_id` UInt64 -`level` Enum8('Fatal' = 1, 'Critical' = 2, 'Error' = 3, 'Warning' = 4, 'Notice' = 5, 'Information' = 6, 'Debug' = 7, 'Trace' = 8) -`query_id` String -`logger_name` LowCardinality(String) -`message` String -`revision` UInt32 -`source_file` LowCardinality(String) -`source_line` UInt64 diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/count.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/count.txt deleted file mode 100644 index dce6588ca14..00000000000 --- a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/count.txt +++ /dev/null @@ -1 +0,0 @@ -36 \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/default_compression_codec.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/minmax_event_date.idx b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/minmax_event_date.idx deleted file mode 100644 index 16cc6680505..00000000000 --- a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/partition.dat b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/partition.dat deleted file mode 100644 index da540c2d1a825be3ef80d13baa834834b2248491..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmXRDWo7^X0wDm8 diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/primary.idx b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/primary.idx deleted file mode 100644 index 3b0e472e2cc..00000000000 --- a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/202102_4638_4638_0/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H"`H"` \ No newline at end of file diff --git a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/format_version.txt b/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/format_version.txt deleted file mode 100644 index 56a6051ca2b..00000000000 --- a/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500/format_version.txt +++ /dev/null @@ -1 +0,0 @@ -1 \ No newline at end of file diff --git a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/asynchronous_metric_log.sql b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/asynchronous_metric_log.sql deleted file mode 100644 index 43ff67fb63d..00000000000 --- a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/asynchronous_metric_log.sql +++ /dev/null @@ -1,12 +0,0 @@ -ATTACH TABLE _ UUID 'e6e8fcae-2f46-4616-a056-74fe935c46b1' -( - `event_date` Date, - `event_time` DateTime, - `event_time_microseconds` DateTime64(6), - `name` String, - `value` Float64 -) -ENGINE = MergeTree -PARTITION BY toYYYYMM(event_date) -ORDER BY (event_date, event_time) -SETTINGS index_granularity = 8192 diff --git a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/crash_log.sql b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/crash_log.sql deleted file mode 100644 index 6cb4dcdc376..00000000000 --- a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/crash_log.sql +++ /dev/null @@ -1,17 +0,0 @@ -ATTACH TABLE _ UUID '5ec3f88a-cd1b-467a-9769-7ff65a09c665' -( - `event_date` Date, - `event_time` DateTime, - `timestamp_ns` UInt64, - `signal` Int32, - `thread_id` UInt64, - `query_id` String, - `trace` Array(UInt64), - `trace_full` Array(String), - `version` String, - `revision` UInt32, - `build_id` String -) -ENGINE = MergeTree -ORDER BY (event_date, event_time) -SETTINGS index_granularity = 8192 diff --git a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/metric_log.sql b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/metric_log.sql deleted file mode 100644 index fbdbe3c0592..00000000000 --- a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/metric_log.sql +++ /dev/null @@ -1,264 +0,0 @@ -ATTACH TABLE _ UUID '057dfcea-531d-4525-a19a-6720aa9ea4b0' -( - `event_date` Date, - `event_time` DateTime, - `event_time_microseconds` DateTime64(6), - `milliseconds` UInt64, - `ProfileEvent_Query` UInt64, - `ProfileEvent_SelectQuery` UInt64, - `ProfileEvent_InsertQuery` UInt64, - `ProfileEvent_FailedQuery` UInt64, - `ProfileEvent_FailedSelectQuery` UInt64, - `ProfileEvent_FailedInsertQuery` UInt64, - `ProfileEvent_QueryTimeMicroseconds` UInt64, - `ProfileEvent_SelectQueryTimeMicroseconds` UInt64, - `ProfileEvent_InsertQueryTimeMicroseconds` UInt64, - `ProfileEvent_FileOpen` UInt64, - `ProfileEvent_Seek` UInt64, - `ProfileEvent_ReadBufferFromFileDescriptorRead` UInt64, - `ProfileEvent_ReadBufferFromFileDescriptorReadFailed` UInt64, - `ProfileEvent_ReadBufferFromFileDescriptorReadBytes` UInt64, - `ProfileEvent_WriteBufferFromFileDescriptorWrite` UInt64, - `ProfileEvent_WriteBufferFromFileDescriptorWriteFailed` UInt64, - `ProfileEvent_WriteBufferFromFileDescriptorWriteBytes` UInt64, - `ProfileEvent_ReadBufferAIORead` UInt64, - `ProfileEvent_ReadBufferAIOReadBytes` UInt64, - `ProfileEvent_WriteBufferAIOWrite` UInt64, - `ProfileEvent_WriteBufferAIOWriteBytes` UInt64, - `ProfileEvent_ReadCompressedBytes` UInt64, - `ProfileEvent_CompressedReadBufferBlocks` UInt64, - `ProfileEvent_CompressedReadBufferBytes` UInt64, - `ProfileEvent_UncompressedCacheHits` UInt64, - `ProfileEvent_UncompressedCacheMisses` UInt64, - `ProfileEvent_UncompressedCacheWeightLost` UInt64, - `ProfileEvent_IOBufferAllocs` UInt64, - `ProfileEvent_IOBufferAllocBytes` UInt64, - `ProfileEvent_ArenaAllocChunks` UInt64, - `ProfileEvent_ArenaAllocBytes` UInt64, - `ProfileEvent_FunctionExecute` UInt64, - `ProfileEvent_TableFunctionExecute` UInt64, - `ProfileEvent_MarkCacheHits` UInt64, - `ProfileEvent_MarkCacheMisses` UInt64, - `ProfileEvent_CreatedReadBufferOrdinary` UInt64, - `ProfileEvent_CreatedReadBufferAIO` UInt64, - `ProfileEvent_CreatedReadBufferAIOFailed` UInt64, - `ProfileEvent_CreatedReadBufferMMap` UInt64, - `ProfileEvent_CreatedReadBufferMMapFailed` UInt64, - `ProfileEvent_CreatedWriteBufferOrdinary` UInt64, - `ProfileEvent_CreatedWriteBufferAIO` UInt64, - `ProfileEvent_CreatedWriteBufferAIOFailed` UInt64, - `ProfileEvent_DiskReadElapsedMicroseconds` UInt64, - `ProfileEvent_DiskWriteElapsedMicroseconds` UInt64, - `ProfileEvent_NetworkReceiveElapsedMicroseconds` UInt64, - `ProfileEvent_NetworkSendElapsedMicroseconds` UInt64, - `ProfileEvent_ThrottlerSleepMicroseconds` UInt64, - `ProfileEvent_QueryMaskingRulesMatch` UInt64, - `ProfileEvent_ReplicatedPartFetches` UInt64, - `ProfileEvent_ReplicatedPartFailedFetches` UInt64, - `ProfileEvent_ObsoleteReplicatedParts` UInt64, - `ProfileEvent_ReplicatedPartMerges` UInt64, - `ProfileEvent_ReplicatedPartFetchesOfMerged` UInt64, - `ProfileEvent_ReplicatedPartMutations` UInt64, - `ProfileEvent_ReplicatedPartChecks` UInt64, - `ProfileEvent_ReplicatedPartChecksFailed` UInt64, - `ProfileEvent_ReplicatedDataLoss` UInt64, - `ProfileEvent_InsertedRows` UInt64, - `ProfileEvent_InsertedBytes` UInt64, - `ProfileEvent_DelayedInserts` UInt64, - `ProfileEvent_RejectedInserts` UInt64, - `ProfileEvent_DelayedInsertsMilliseconds` UInt64, - `ProfileEvent_DuplicatedInsertedBlocks` UInt64, - `ProfileEvent_ZooKeeperInit` UInt64, - `ProfileEvent_ZooKeeperTransactions` UInt64, - `ProfileEvent_ZooKeeperList` UInt64, - `ProfileEvent_ZooKeeperCreate` UInt64, - `ProfileEvent_ZooKeeperRemove` UInt64, - `ProfileEvent_ZooKeeperExists` UInt64, - `ProfileEvent_ZooKeeperGet` UInt64, - `ProfileEvent_ZooKeeperSet` UInt64, - `ProfileEvent_ZooKeeperMulti` UInt64, - `ProfileEvent_ZooKeeperCheck` UInt64, - `ProfileEvent_ZooKeeperClose` UInt64, - `ProfileEvent_ZooKeeperWatchResponse` UInt64, - `ProfileEvent_ZooKeeperUserExceptions` UInt64, - `ProfileEvent_ZooKeeperHardwareExceptions` UInt64, - `ProfileEvent_ZooKeeperOtherExceptions` UInt64, - `ProfileEvent_ZooKeeperWaitMicroseconds` UInt64, - `ProfileEvent_ZooKeeperBytesSent` UInt64, - `ProfileEvent_ZooKeeperBytesReceived` UInt64, - `ProfileEvent_DistributedConnectionFailTry` UInt64, - `ProfileEvent_DistributedConnectionMissingTable` UInt64, - `ProfileEvent_DistributedConnectionStaleReplica` UInt64, - `ProfileEvent_DistributedConnectionFailAtAll` UInt64, - `ProfileEvent_CompileAttempt` UInt64, - `ProfileEvent_CompileSuccess` UInt64, - `ProfileEvent_CompileFunction` UInt64, - `ProfileEvent_CompiledFunctionExecute` UInt64, - `ProfileEvent_CompileExpressionsMicroseconds` UInt64, - `ProfileEvent_CompileExpressionsBytes` UInt64, - `ProfileEvent_ExternalSortWritePart` UInt64, - `ProfileEvent_ExternalSortMerge` UInt64, - `ProfileEvent_ExternalAggregationWritePart` UInt64, - `ProfileEvent_ExternalAggregationMerge` UInt64, - `ProfileEvent_ExternalAggregationCompressedBytes` UInt64, - `ProfileEvent_ExternalAggregationUncompressedBytes` UInt64, - `ProfileEvent_SlowRead` UInt64, - `ProfileEvent_ReadBackoff` UInt64, - `ProfileEvent_ReplicaPartialShutdown` UInt64, - `ProfileEvent_SelectedParts` UInt64, - `ProfileEvent_SelectedRanges` UInt64, - `ProfileEvent_SelectedMarks` UInt64, - `ProfileEvent_SelectedRows` UInt64, - `ProfileEvent_SelectedBytes` UInt64, - `ProfileEvent_Merge` UInt64, - `ProfileEvent_MergedRows` UInt64, - `ProfileEvent_MergedUncompressedBytes` UInt64, - `ProfileEvent_MergesTimeMilliseconds` UInt64, - `ProfileEvent_MergeTreeDataWriterRows` UInt64, - `ProfileEvent_MergeTreeDataWriterUncompressedBytes` UInt64, - `ProfileEvent_MergeTreeDataWriterCompressedBytes` UInt64, - `ProfileEvent_MergeTreeDataWriterBlocks` UInt64, - `ProfileEvent_MergeTreeDataWriterBlocksAlreadySorted` UInt64, - `ProfileEvent_CannotRemoveEphemeralNode` UInt64, - `ProfileEvent_RegexpCreated` UInt64, - `ProfileEvent_ContextLock` UInt64, - `ProfileEvent_StorageBufferFlush` UInt64, - `ProfileEvent_StorageBufferErrorOnFlush` UInt64, - `ProfileEvent_StorageBufferPassedAllMinThresholds` UInt64, - `ProfileEvent_StorageBufferPassedTimeMaxThreshold` UInt64, - `ProfileEvent_StorageBufferPassedRowsMaxThreshold` UInt64, - `ProfileEvent_StorageBufferPassedBytesMaxThreshold` UInt64, - `ProfileEvent_DictCacheKeysRequested` UInt64, - `ProfileEvent_DictCacheKeysRequestedMiss` UInt64, - `ProfileEvent_DictCacheKeysRequestedFound` UInt64, - `ProfileEvent_DictCacheKeysExpired` UInt64, - `ProfileEvent_DictCacheKeysNotFound` UInt64, - `ProfileEvent_DictCacheKeysHit` UInt64, - `ProfileEvent_DictCacheRequestTimeNs` UInt64, - `ProfileEvent_DictCacheRequests` UInt64, - `ProfileEvent_DictCacheLockWriteNs` UInt64, - `ProfileEvent_DictCacheLockReadNs` UInt64, - `ProfileEvent_DistributedSyncInsertionTimeoutExceeded` UInt64, - `ProfileEvent_DataAfterMergeDiffersFromReplica` UInt64, - `ProfileEvent_DataAfterMutationDiffersFromReplica` UInt64, - `ProfileEvent_PolygonsAddedToPool` UInt64, - `ProfileEvent_PolygonsInPoolAllocatedBytes` UInt64, - `ProfileEvent_RWLockAcquiredReadLocks` UInt64, - `ProfileEvent_RWLockAcquiredWriteLocks` UInt64, - `ProfileEvent_RWLockReadersWaitMilliseconds` UInt64, - `ProfileEvent_RWLockWritersWaitMilliseconds` UInt64, - `ProfileEvent_DNSError` UInt64, - `ProfileEvent_RealTimeMicroseconds` UInt64, - `ProfileEvent_UserTimeMicroseconds` UInt64, - `ProfileEvent_SystemTimeMicroseconds` UInt64, - `ProfileEvent_SoftPageFaults` UInt64, - `ProfileEvent_HardPageFaults` UInt64, - `ProfileEvent_VoluntaryContextSwitches` UInt64, - `ProfileEvent_InvoluntaryContextSwitches` UInt64, - `ProfileEvent_OSIOWaitMicroseconds` UInt64, - `ProfileEvent_OSCPUWaitMicroseconds` UInt64, - `ProfileEvent_OSCPUVirtualTimeMicroseconds` UInt64, - `ProfileEvent_OSReadBytes` UInt64, - `ProfileEvent_OSWriteBytes` UInt64, - `ProfileEvent_OSReadChars` UInt64, - `ProfileEvent_OSWriteChars` UInt64, - `ProfileEvent_PerfCpuCycles` UInt64, - `ProfileEvent_PerfInstructions` UInt64, - `ProfileEvent_PerfCacheReferences` UInt64, - `ProfileEvent_PerfCacheMisses` UInt64, - `ProfileEvent_PerfBranchInstructions` UInt64, - `ProfileEvent_PerfBranchMisses` UInt64, - `ProfileEvent_PerfBusCycles` UInt64, - `ProfileEvent_PerfStalledCyclesFrontend` UInt64, - `ProfileEvent_PerfStalledCyclesBackend` UInt64, - `ProfileEvent_PerfRefCpuCycles` UInt64, - `ProfileEvent_PerfCpuClock` UInt64, - `ProfileEvent_PerfTaskClock` UInt64, - `ProfileEvent_PerfContextSwitches` UInt64, - `ProfileEvent_PerfCpuMigrations` UInt64, - `ProfileEvent_PerfAlignmentFaults` UInt64, - `ProfileEvent_PerfEmulationFaults` UInt64, - `ProfileEvent_PerfMinEnabledTime` UInt64, - `ProfileEvent_PerfMinEnabledRunningTime` UInt64, - `ProfileEvent_PerfDataTLBReferences` UInt64, - `ProfileEvent_PerfDataTLBMisses` UInt64, - `ProfileEvent_PerfInstructionTLBReferences` UInt64, - `ProfileEvent_PerfInstructionTLBMisses` UInt64, - `ProfileEvent_PerfLocalMemoryReferences` UInt64, - `ProfileEvent_PerfLocalMemoryMisses` UInt64, - `ProfileEvent_CreatedHTTPConnections` UInt64, - `ProfileEvent_CannotWriteToWriteBufferDiscard` UInt64, - `ProfileEvent_QueryProfilerSignalOverruns` UInt64, - `ProfileEvent_CreatedLogEntryForMerge` UInt64, - `ProfileEvent_NotCreatedLogEntryForMerge` UInt64, - `ProfileEvent_CreatedLogEntryForMutation` UInt64, - `ProfileEvent_NotCreatedLogEntryForMutation` UInt64, - `ProfileEvent_S3ReadMicroseconds` UInt64, - `ProfileEvent_S3ReadBytes` UInt64, - `ProfileEvent_S3ReadRequestsCount` UInt64, - `ProfileEvent_S3ReadRequestsErrors` UInt64, - `ProfileEvent_S3ReadRequestsThrottling` UInt64, - `ProfileEvent_S3ReadRequestsRedirects` UInt64, - `ProfileEvent_S3WriteMicroseconds` UInt64, - `ProfileEvent_S3WriteBytes` UInt64, - `ProfileEvent_S3WriteRequestsCount` UInt64, - `ProfileEvent_S3WriteRequestsErrors` UInt64, - `ProfileEvent_S3WriteRequestsThrottling` UInt64, - `ProfileEvent_S3WriteRequestsRedirects` UInt64, - `ProfileEvent_QueryMemoryLimitExceeded` UInt64, - `CurrentMetric_Query` Int64, - `CurrentMetric_Merge` Int64, - `CurrentMetric_PartMutation` Int64, - `CurrentMetric_ReplicatedFetch` Int64, - `CurrentMetric_ReplicatedSend` Int64, - `CurrentMetric_ReplicatedChecks` Int64, - `CurrentMetric_BackgroundPoolTask` Int64, - `CurrentMetric_BackgroundMovePoolTask` Int64, - `CurrentMetric_BackgroundSchedulePoolTask` Int64, - `CurrentMetric_BackgroundBufferFlushSchedulePoolTask` Int64, - `CurrentMetric_BackgroundDistributedSchedulePoolTask` Int64, - `CurrentMetric_BackgroundMessageBrokerSchedulePoolTask` Int64, - `CurrentMetric_CacheDictionaryUpdateQueueBatches` Int64, - `CurrentMetric_CacheDictionaryUpdateQueueKeys` Int64, - `CurrentMetric_DiskSpaceReservedForMerge` Int64, - `CurrentMetric_DistributedSend` Int64, - `CurrentMetric_QueryPreempted` Int64, - `CurrentMetric_TCPConnection` Int64, - `CurrentMetric_MySQLConnection` Int64, - `CurrentMetric_HTTPConnection` Int64, - `CurrentMetric_InterserverConnection` Int64, - `CurrentMetric_PostgreSQLConnection` Int64, - `CurrentMetric_OpenFileForRead` Int64, - `CurrentMetric_OpenFileForWrite` Int64, - `CurrentMetric_Read` Int64, - `CurrentMetric_Write` Int64, - `CurrentMetric_SendScalars` Int64, - `CurrentMetric_SendExternalTables` Int64, - `CurrentMetric_QueryThread` Int64, - `CurrentMetric_ReadonlyReplica` Int64, - `CurrentMetric_MemoryTracking` Int64, - `CurrentMetric_EphemeralNode` Int64, - `CurrentMetric_ZooKeeperSession` Int64, - `CurrentMetric_ZooKeeperWatch` Int64, - `CurrentMetric_ZooKeeperRequest` Int64, - `CurrentMetric_DelayedInserts` Int64, - `CurrentMetric_ContextLockWait` Int64, - `CurrentMetric_StorageBufferRows` Int64, - `CurrentMetric_StorageBufferBytes` Int64, - `CurrentMetric_DictCacheRequests` Int64, - `CurrentMetric_Revision` Int64, - `CurrentMetric_VersionInteger` Int64, - `CurrentMetric_RWLockWaitingReaders` Int64, - `CurrentMetric_RWLockWaitingWriters` Int64, - `CurrentMetric_RWLockActiveReaders` Int64, - `CurrentMetric_RWLockActiveWriters` Int64, - `CurrentMetric_GlobalThread` Int64, - `CurrentMetric_GlobalThreadActive` Int64, - `CurrentMetric_LocalThread` Int64, - `CurrentMetric_LocalThreadActive` Int64, - `CurrentMetric_DistributedFilesToInsert` Int64 -) -ENGINE = MergeTree -PARTITION BY toYYYYMM(event_date) -ORDER BY (event_date, event_time) -SETTINGS index_granularity = 8192 diff --git a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/part_log.sql b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/part_log.sql deleted file mode 100644 index ea3aabd51e4..00000000000 --- a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/part_log.sql +++ /dev/null @@ -1,25 +0,0 @@ -ATTACH TABLE _ UUID '721d6adf-4b1c-49bc-8fbe-1599ac4e048d' -( - `event_type` Enum8('NewPart' = 1, 'MergeParts' = 2, 'DownloadPart' = 3, 'RemovePart' = 4, 'MutatePart' = 5, 'MovePart' = 6), - `event_date` Date, - `event_time` DateTime, - `duration_ms` UInt64, - `database` String, - `table` String, - `part_name` String, - `partition_id` String, - `path_on_disk` String, - `rows` UInt64, - `size_in_bytes` UInt64, - `merged_from` Array(String), - `bytes_uncompressed` UInt64, - `read_rows` UInt64, - `read_bytes` UInt64, - `peak_memory_usage` UInt64, - `error` UInt16, - `exception` String -) -ENGINE = MergeTree -PARTITION BY toYYYYMM(event_date) -ORDER BY (event_date, event_time) -SETTINGS index_granularity = 8192 diff --git a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/query_log.sql b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/query_log.sql deleted file mode 100644 index bcafc4b0530..00000000000 --- a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/query_log.sql +++ /dev/null @@ -1,52 +0,0 @@ -ATTACH TABLE _ UUID '3f558312-ca2a-49dc-a6bf-a5daea1869df' -( - `type` Enum8('QueryStart' = 1, 'QueryFinish' = 2, 'ExceptionBeforeStart' = 3, 'ExceptionWhileProcessing' = 4), - `event_date` Date, - `event_time` DateTime, - `event_time_microseconds` DateTime64(6), - `query_start_time` DateTime, - `query_start_time_microseconds` DateTime64(6), - `query_duration_ms` UInt64, - `read_rows` UInt64, - `read_bytes` UInt64, - `written_rows` UInt64, - `written_bytes` UInt64, - `result_rows` UInt64, - `result_bytes` UInt64, - `memory_usage` UInt64, - `current_database` String, - `query` String, - `exception_code` Int32, - `exception` String, - `stack_trace` String, - `is_initial_query` UInt8, - `user` String, - `query_id` String, - `address` IPv6, - `port` UInt16, - `initial_user` String, - `initial_query_id` String, - `initial_address` IPv6, - `initial_port` UInt16, - `interface` UInt8, - `os_user` String, - `client_hostname` String, - `client_name` String, - `client_revision` UInt32, - `client_version_major` UInt32, - `client_version_minor` UInt32, - `client_version_patch` UInt32, - `http_method` UInt8, - `http_user_agent` String, - `quota_key` String, - `revision` UInt32, - `thread_ids` Array(UInt64), - `ProfileEvents.Names` Array(String), - `ProfileEvents.Values` Array(UInt64), - `Settings.Names` Array(String), - `Settings.Values` Array(String) -) -ENGINE = MergeTree -PARTITION BY toYYYYMM(event_date) -ORDER BY (event_date, event_time) -SETTINGS index_granularity = 8192 diff --git a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/query_thread_log.sql b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/query_thread_log.sql deleted file mode 100644 index 56ff3366702..00000000000 --- a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/query_thread_log.sql +++ /dev/null @@ -1,46 +0,0 @@ -ATTACH TABLE _ UUID '8b2738c3-4bea-49f7-b547-0a92673830cc' -( - `event_date` Date, - `event_time` DateTime, - `event_time_microseconds` DateTime64(6), - `query_start_time` DateTime, - `query_start_time_microseconds` DateTime64(6), - `query_duration_ms` UInt64, - `read_rows` UInt64, - `read_bytes` UInt64, - `written_rows` UInt64, - `written_bytes` UInt64, - `memory_usage` Int64, - `peak_memory_usage` Int64, - `thread_name` String, - `thread_id` UInt64, - `master_thread_id` UInt64, - `query` String, - `is_initial_query` UInt8, - `user` String, - `query_id` String, - `address` IPv6, - `port` UInt16, - `initial_user` String, - `initial_query_id` String, - `initial_address` IPv6, - `initial_port` UInt16, - `interface` UInt8, - `os_user` String, - `client_hostname` String, - `client_name` String, - `client_revision` UInt32, - `client_version_major` UInt32, - `client_version_minor` UInt32, - `client_version_patch` UInt32, - `http_method` UInt8, - `http_user_agent` String, - `quota_key` String, - `revision` UInt32, - `ProfileEvents.Names` Array(String), - `ProfileEvents.Values` Array(UInt64) -) -ENGINE = MergeTree -PARTITION BY toYYYYMM(event_date) -ORDER BY (event_date, event_time) -SETTINGS index_granularity = 8192 diff --git a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/text_log.sql b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/text_log.sql deleted file mode 100644 index 874466c610f..00000000000 --- a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/text_log.sql +++ /dev/null @@ -1,20 +0,0 @@ -ATTACH TABLE _ UUID 'b619f0a8-9836-4824-9594-a722b0a7d500' -( - `event_date` Date, - `event_time` DateTime, - `event_time_microseconds` DateTime64(6), - `microseconds` UInt32, - `thread_name` LowCardinality(String), - `thread_id` UInt64, - `level` Enum8('Fatal' = 1, 'Critical' = 2, 'Error' = 3, 'Warning' = 4, 'Notice' = 5, 'Information' = 6, 'Debug' = 7, 'Trace' = 8), - `query_id` String, - `logger_name` LowCardinality(String), - `message` String, - `revision` UInt32, - `source_file` LowCardinality(String), - `source_line` UInt64 -) -ENGINE = MergeTree -PARTITION BY toYYYYMM(event_date) -ORDER BY (event_date, event_time) -SETTINGS index_granularity = 8192 diff --git a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/trace_log.sql b/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/trace_log.sql deleted file mode 100644 index 04517208a2e..00000000000 --- a/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681/trace_log.sql +++ /dev/null @@ -1,17 +0,0 @@ -ATTACH TABLE _ UUID '72dc35e3-6e64-44bf-af3f-4d66fb70834a' -( - `event_date` Date, - `event_time` DateTime, - `event_time_microseconds` DateTime64(6), - `timestamp_ns` UInt64, - `revision` UInt32, - `trace_type` Enum8('Real' = 0, 'CPU' = 1, 'Memory' = 2, 'MemorySample' = 3), - `thread_id` UInt64, - `query_id` String, - `trace` Array(UInt64), - `size` Int64 -) -ENGINE = MergeTree -PARTITION BY toYYYYMM(event_date) -ORDER BY (event_date, event_time) -SETTINGS index_granularity = 8192 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/checksums.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/checksums.txt deleted file mode 100644 index 15f590ec5a9f148c51c40e5f8643b93ed6312e48..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 260 zcmV+f0sH=AXk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$bUKSS==f{FYCNyEFtWbCto z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOT5}}ui?1Ks zvg>S#d$7rg02OU%Zf#+BUuAY>ZggK{VRU6KX=Hc=Yhv{9L4)i+mY_Q2(jfMG01a?q za&&2QX>V>WWMOm!_Mvet<#01I$(X>DP0c`j*Wcnm83Pgz_C>>6$e Kar3Qw+64edhHlyb diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/columns.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/columns.txt deleted file mode 100644 index 5bfe4600a4e..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/columns.txt +++ /dev/null @@ -1,7 +0,0 @@ -columns format version: 1 -5 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`name` String -`value` Float64 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/count.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/count.txt deleted file mode 100644 index afbe847262c..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/count.txt +++ /dev/null @@ -1 +0,0 @@ -126 \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/default_compression_codec.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/minmax_event_date.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/minmax_event_date.idx deleted file mode 100644 index 4b6c49f8d50..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/partition.dat b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/partition.dat deleted file mode 100644 index b9ea5569e3a3b34e3a1b6a1c4cbcb46f5e2d3fac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 Lcmb0VWo7^X0I>iw diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/primary.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/primary.idx deleted file mode 100644 index 38e2c642aeb..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202010_1_3_1/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H-_HN._ \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/checksums.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/checksums.txt deleted file mode 100644 index 288379f59b4d0506587d2b915da29b5429f3c32c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 262 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyVUf?3K)m;9pkOXWjEYr54|4y2Zf2 zZ~=%v+Ou&c=a=S{=#^BIupV9eg5{F*3qhl-g^$@cNz7y5NJ%V7)Jw|DJHCbKd3ghe zFgEolo@6;#*(z-(X1n4kgRx0RBc}nwhrHaPY~v25bs?^=WxQQ@ znPqo7Se%>sj6pOvGcPx>B0jY&HLoNdXmF}tW=aK1VDXg_AS<=yCEMbz)z+F(3nVVQt3DPa}a>cj&7s0hFg}*8) KC3BoV%>V$TlWN`o diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/columns.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/columns.txt deleted file mode 100644 index 5bfe4600a4e..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/columns.txt +++ /dev/null @@ -1,7 +0,0 @@ -columns format version: 1 -5 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`name` String -`value` Float64 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/count.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/count.txt deleted file mode 100644 index 23644ad9685..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/count.txt +++ /dev/null @@ -1 +0,0 @@ -12390 \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/default_compression_codec.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/minmax_event_date.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/minmax_event_date.idx deleted file mode 100644 index 07abc658bdb..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/partition.dat b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/partition.dat deleted file mode 100644 index 7af4421fcb5ebc50aa06a5f4dee6e454020552a6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 Lcmb0UWo7^X0J#7) diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/primary.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/primary.idx deleted file mode 100644 index d57044faf98..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202012_4_295_59/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H>_H_H}_ \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/checksums.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/checksums.txt deleted file mode 100644 index d702cbf79d2a5a91c07163c9680526ae0c503ceb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 262 zcmV+h0r~!8Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$bUW>V4jhooyz_~f{Za(pN8Q&6FaYpwZE|ZfumSZKokr7~ z|403vu$n{)-iQDdZE0?8VR&C@6CZZ2eDbOfQp0V$_ea0t>|RE;3Dk?sHsaB^vFVRCscX=Hc|m=Z;zemg0s MudjjE3#Lvc0M|-wEC2ui diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/columns.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/columns.txt deleted file mode 100644 index 5bfe4600a4e..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/columns.txt +++ /dev/null @@ -1,7 +0,0 @@ -columns format version: 1 -5 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`name` String -`value` Float64 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/count.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/count.txt deleted file mode 100644 index 721ca6f552a..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/count.txt +++ /dev/null @@ -1 +0,0 @@ -6048 \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/default_compression_codec.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/minmax_event_date.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/minmax_event_date.idx deleted file mode 100644 index 4c89b29a856..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/partition.dat b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/partition.dat deleted file mode 100644 index a713aecd075daae7071deceaa93147871ef28fbf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmXRFWo7^X0v!O3 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/primary.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/primary.idx deleted file mode 100644 index 8e5b4736172..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202101_296_439_30/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H_H ` \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_440_574_27/checksums.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_440_574_27/checksums.txt deleted file mode 100644 index 318b8a518b960630dc1e834f27ce63e27b6593ab..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 262 zcmV+h0r~!8Xk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$c7oqGLyZE0?8VR&C@6CZZ2eDbOb>eW_f~A{08aeSmV>iAT$cC2%NmSkKqG{?=Jq@?bnHd z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOJne3tc?*x$p?c>F_{T(_H`vWMOn+E@EkJfCsQb zD@h%ngTaZ|pyvL@;Y0zh4$4=AdlcKPVg5TpMxB@3V+k$*@NR8#YcsF`21}@_f-!EI z40m1$`G2OI02OU%Zf#+BUuAY>ZggK{VRU6KX=Hc=VZjEVxwC1aJrFfB;*n}Y01a?q za&&2QX>V>WWMOm!K^bOwf>Qhj>E&4HXupM*01I$(X>DP0c`j*Wcnrlp=b;Ym@z8Dp Kp(2L^K0p9kBW<++ diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/columns.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/columns.txt deleted file mode 100644 index 5bfe4600a4e..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/columns.txt +++ /dev/null @@ -1,7 +0,0 @@ -columns format version: 1 -5 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`name` String -`value` Float64 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/count.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/count.txt deleted file mode 100644 index f70d7bba4ae..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/count.txt +++ /dev/null @@ -1 +0,0 @@ -42 \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/default_compression_codec.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/minmax_event_date.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/minmax_event_date.idx deleted file mode 100644 index 16cc6680505..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/partition.dat b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/partition.dat deleted file mode 100644 index da540c2d1a825be3ef80d13baa834834b2248491..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmXRDWo7^X0wDm8 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/primary.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/primary.idx deleted file mode 100644 index 47460b95be9..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_575_575_0/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H"`H"` \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/checksums.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/checksums.txt deleted file mode 100644 index 705c91fa699e37af9a5bc17585f859123272e27b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 260 zcmYe!NKMWzF3l}gNXswEO)OC;OD!tS%+Iq@FyZ=8e&PS-lZH>%6zMI~Ocl~>y1~G} za1Mw++Olyb=a=S{=#^BIFxixH$J%_|`GMoa%MT9Wmtz??QW8rN^^!94dfBVyP4w8E z^d;5x=sNDHm&BhluH*BS+J9T=m`+Tsp_!zbhWTwyJ%$f?xkcH=8yJhzv;sK_cCKB% z@k(c3X6Zc!(cH|u+{B9b)UwpPl6au8sd||y6)cGd*%s{FoVmzWz|{EB#4Kk9-h#xU zlFSmIyY+yISR5tODw~4-u)Ta5@gie?>vRV0f}+ga#G*=&Zk}DNanDuX@+;a)C2mzy I?byly0Nr6}0ssI2 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/columns.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/columns.txt deleted file mode 100644 index 5bfe4600a4e..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/columns.txt +++ /dev/null @@ -1,7 +0,0 @@ -columns format version: 1 -5 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`name` String -`value` Float64 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/count.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/count.txt deleted file mode 100644 index f70d7bba4ae..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/count.txt +++ /dev/null @@ -1 +0,0 @@ -42 \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/default_compression_codec.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/minmax_event_date.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/minmax_event_date.idx deleted file mode 100644 index 16cc6680505..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/partition.dat b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/partition.dat deleted file mode 100644 index da540c2d1a825be3ef80d13baa834834b2248491..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmXRDWo7^X0wDm8 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/primary.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/primary.idx deleted file mode 100644 index dc2b3df8c14..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_576_576_0/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H"`H"` \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/checksums.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/checksums.txt deleted file mode 100644 index a79136dea2ae4694404058e307f30c58f2c95216..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 260 zcmV+f0sH=AXk}w-b9HTVAZBlJZDDjEc4cyNX>V>iAT$b+KNoPO=+Yv2+Llc&Eg)Ee z*Z=?k&Hw-a@jV6!V{dhCbS`vwbOJne3tc?*x$p?c>F_{T(_H`vWMOn+E@EkJk_VW( zJ28oFC&TW$V{^28W&8oI4&bJ;l7ZNHLvKrW15Q;okqIsU@NR8#YcsF`wn)byrXbCF zZggK{VRU6KX=Hc=VZjEVxwC1aJrFfB;*n}Y01a?q za&&2QX>V>WWMOm!K^bOwf>Qhj>E&4HXupM*01I$(X>DP0c`j*WcnoZhRO7$)udRQM Kfs2=!gM9!_8*V`W diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/columns.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/columns.txt deleted file mode 100644 index 5bfe4600a4e..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/columns.txt +++ /dev/null @@ -1,7 +0,0 @@ -columns format version: 1 -5 columns: -`event_date` Date -`event_time` DateTime -`event_time_microseconds` DateTime64(6) -`name` String -`value` Float64 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/count.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/count.txt deleted file mode 100644 index f70d7bba4ae..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/count.txt +++ /dev/null @@ -1 +0,0 @@ -42 \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/default_compression_codec.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/default_compression_codec.txt deleted file mode 100644 index 061d1280b89..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/default_compression_codec.txt +++ /dev/null @@ -1 +0,0 @@ -CODEC(LZ4) \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/minmax_event_date.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/minmax_event_date.idx deleted file mode 100644 index 16cc6680505..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/minmax_event_date.idx +++ /dev/null @@ -1 +0,0 @@ -HH \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/partition.dat b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/partition.dat deleted file mode 100644 index da540c2d1a825be3ef80d13baa834834b2248491..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmXRDWo7^X0wDm8 diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/primary.idx b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/primary.idx deleted file mode 100644 index bd60e38174b..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/202102_577_577_0/primary.idx +++ /dev/null @@ -1 +0,0 @@ -H""`H""` \ No newline at end of file diff --git a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/format_version.txt b/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/format_version.txt deleted file mode 100644 index 56a6051ca2b..00000000000 --- a/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1/format_version.txt +++ /dev/null @@ -1 +0,0 @@ -1 \ No newline at end of file From 2c3c41d72c9ed1435ffba4fc013409c83ccb6078 Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Tue, 9 Feb 2021 21:51:41 +0300 Subject: [PATCH 003/171] remove another unnessesary files --- programs/server/data/system/asynchronous_metric_log | 1 - programs/server/data/system/crash_log | 1 - programs/server/data/system/metric_log | 1 - programs/server/data/system/part_log | 1 - programs/server/data/system/query_log | 1 - programs/server/data/system/query_thread_log | 1 - programs/server/data/system/text_log | 1 - programs/server/data/system/trace_log | 1 - programs/server/data/test/hits | 1 - programs/server/data/test/visits | 1 - programs/server/metadata/system | 1 - programs/server/metadata/test | 1 - 12 files changed, 12 deletions(-) delete mode 120000 programs/server/data/system/asynchronous_metric_log delete mode 120000 programs/server/data/system/crash_log delete mode 120000 programs/server/data/system/metric_log delete mode 120000 programs/server/data/system/part_log delete mode 120000 programs/server/data/system/query_log delete mode 120000 programs/server/data/system/query_thread_log delete mode 120000 programs/server/data/system/text_log delete mode 120000 programs/server/data/system/trace_log delete mode 120000 programs/server/data/test/hits delete mode 120000 programs/server/data/test/visits delete mode 120000 programs/server/metadata/system delete mode 120000 programs/server/metadata/test diff --git a/programs/server/data/system/asynchronous_metric_log b/programs/server/data/system/asynchronous_metric_log deleted file mode 120000 index e065d7bf424..00000000000 --- a/programs/server/data/system/asynchronous_metric_log +++ /dev/null @@ -1 +0,0 @@ -/home/max/github/ClickHouse/programs/server/store/e6e/e6e8fcae-2f46-4616-a056-74fe935c46b1 \ No newline at end of file diff --git a/programs/server/data/system/crash_log b/programs/server/data/system/crash_log deleted file mode 120000 index 7c05d325fb8..00000000000 --- a/programs/server/data/system/crash_log +++ /dev/null @@ -1 +0,0 @@ -/home/max/github/ClickHouse/programs/server/store/5ec/5ec3f88a-cd1b-467a-9769-7ff65a09c665 \ No newline at end of file diff --git a/programs/server/data/system/metric_log b/programs/server/data/system/metric_log deleted file mode 120000 index cf9f177d49f..00000000000 --- a/programs/server/data/system/metric_log +++ /dev/null @@ -1 +0,0 @@ -/home/max/github/ClickHouse/programs/server/store/057/057dfcea-531d-4525-a19a-6720aa9ea4b0 \ No newline at end of file diff --git a/programs/server/data/system/part_log b/programs/server/data/system/part_log deleted file mode 120000 index b19efce9efd..00000000000 --- a/programs/server/data/system/part_log +++ /dev/null @@ -1 +0,0 @@ -/home/max/github/ClickHouse/programs/server/store/721/721d6adf-4b1c-49bc-8fbe-1599ac4e048d \ No newline at end of file diff --git a/programs/server/data/system/query_log b/programs/server/data/system/query_log deleted file mode 120000 index 4222415ce96..00000000000 --- a/programs/server/data/system/query_log +++ /dev/null @@ -1 +0,0 @@ -/home/max/github/ClickHouse/programs/server/store/3f5/3f558312-ca2a-49dc-a6bf-a5daea1869df \ No newline at end of file diff --git a/programs/server/data/system/query_thread_log b/programs/server/data/system/query_thread_log deleted file mode 120000 index 6108d6eab87..00000000000 --- a/programs/server/data/system/query_thread_log +++ /dev/null @@ -1 +0,0 @@ -/home/max/github/ClickHouse/programs/server/store/8b2/8b2738c3-4bea-49f7-b547-0a92673830cc \ No newline at end of file diff --git a/programs/server/data/system/text_log b/programs/server/data/system/text_log deleted file mode 120000 index 4ae519959fe..00000000000 --- a/programs/server/data/system/text_log +++ /dev/null @@ -1 +0,0 @@ -/home/max/github/ClickHouse/programs/server/store/b61/b619f0a8-9836-4824-9594-a722b0a7d500 \ No newline at end of file diff --git a/programs/server/data/system/trace_log b/programs/server/data/system/trace_log deleted file mode 120000 index 6dfdc4802b3..00000000000 --- a/programs/server/data/system/trace_log +++ /dev/null @@ -1 +0,0 @@ -/home/max/github/ClickHouse/programs/server/store/72d/72dc35e3-6e64-44bf-af3f-4d66fb70834a \ No newline at end of file diff --git a/programs/server/data/test/hits b/programs/server/data/test/hits deleted file mode 120000 index 28b16b0be67..00000000000 --- a/programs/server/data/test/hits +++ /dev/null @@ -1 +0,0 @@ -/home/max/github/ClickHouse/programs/server/store/726/72667623-6371-4258-be57-799522ee4e64 \ No newline at end of file diff --git a/programs/server/data/test/visits b/programs/server/data/test/visits deleted file mode 120000 index 49e67e1c13b..00000000000 --- a/programs/server/data/test/visits +++ /dev/null @@ -1 +0,0 @@ -/home/max/github/ClickHouse/programs/server/store/044/044d7000-c91c-482d-9f0f-719f226eff0f \ No newline at end of file diff --git a/programs/server/metadata/system b/programs/server/metadata/system deleted file mode 120000 index 4ff5d3274d6..00000000000 --- a/programs/server/metadata/system +++ /dev/null @@ -1 +0,0 @@ -/home/max/github/ClickHouse/programs/server/store/dfe/dfe46a8e-9eec-4e9d-955b-685b1278c681 \ No newline at end of file diff --git a/programs/server/metadata/test b/programs/server/metadata/test deleted file mode 120000 index b4ed8633adf..00000000000 --- a/programs/server/metadata/test +++ /dev/null @@ -1 +0,0 @@ -/home/max/github/ClickHouse/programs/server/store/2c6/2c6da17b-121a-4775-b287-b0d80cd04080 \ No newline at end of file From aa46ddc3e29cf71faaafbf8f49b3fc1217371154 Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Tue, 9 Feb 2021 23:41:42 +0300 Subject: [PATCH 004/171] revert changes made to cube transform --- src/Processors/Transforms/CubeTransform.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Processors/Transforms/CubeTransform.cpp b/src/Processors/Transforms/CubeTransform.cpp index 13e4963573c..c64f39af5de 100644 --- a/src/Processors/Transforms/CubeTransform.cpp +++ b/src/Processors/Transforms/CubeTransform.cpp @@ -45,7 +45,7 @@ Chunk CubeTransform::generate() consumed_chunks.clear(); auto num_rows = cube_chunk.getNumRows(); - mask = (UInt64(1) << keys.size()); + mask = (UInt64(1) << keys.size()) - 1; current_columns = cube_chunk.getColumns(); current_zero_columns.clear(); @@ -55,11 +55,11 @@ Chunk CubeTransform::generate() current_zero_columns.emplace_back(current_columns[key]->cloneEmpty()->cloneResized(num_rows)); } - // auto gen_chunk = std::move(cube_chunk); + auto gen_chunk = std::move(cube_chunk); - if (mask > 1) + if (mask) { - mask = mask >> 1; + --mask; auto columns = current_columns; auto size = keys.size(); @@ -72,7 +72,6 @@ Chunk CubeTransform::generate() chunks.emplace_back(std::move(columns), current_columns.front()->size()); cube_chunk = merge(std::move(chunks), false); } - auto gen_chunk = std::move(cube_chunk); finalizeChunk(gen_chunk); return gen_chunk; From 544bb575bc49fe981512e6a015f61a4cc0520866 Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Tue, 9 Feb 2021 23:43:03 +0300 Subject: [PATCH 005/171] fix typo --- src/Parsers/ParserSelectQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/ParserSelectQuery.cpp b/src/Parsers/ParserSelectQuery.cpp index f77fff658d6..6c2889d2768 100644 --- a/src/Parsers/ParserSelectQuery.cpp +++ b/src/Parsers/ParserSelectQuery.cpp @@ -200,7 +200,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) return false; } - /// WITH ROLLUP, CUBE, GROU PING SETS or TOTALS + /// WITH ROLLUP, CUBE, GROUPING SETS or TOTALS if (s_with.ignore(pos, expected)) { if (s_rollup.ignore(pos, expected)) From 209ce510783e9dff6e566e764402578adbce4ef0 Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Tue, 9 Feb 2021 23:44:56 +0300 Subject: [PATCH 006/171] erase blank line to restore initial state --- src/Processors/Transforms/AggregatingTransform.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index c9d516f8a11..2a515fdf3be 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -15,7 +15,6 @@ public: AggregatedArenasChunkInfo(Arenas arenas_) : arenas(std::move(arenas_)) {} - }; class AggregatedChunkInfo : public ChunkInfo From 34d71ee816b0d9c41cdaa503788d385e7a63341b Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Tue, 9 Feb 2021 23:50:35 +0300 Subject: [PATCH 007/171] fixes --- src/Parsers/ParserSelectQuery.cpp | 2 ++ src/Processors/Transforms/AggregatingTransform.h | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Parsers/ParserSelectQuery.cpp b/src/Parsers/ParserSelectQuery.cpp index 6c2889d2768..6a17993a704 100644 --- a/src/Parsers/ParserSelectQuery.cpp +++ b/src/Parsers/ParserSelectQuery.cpp @@ -207,6 +207,8 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) select_query->group_by_with_rollup = true; else if (s_cube.ignore(pos, expected)) select_query->group_by_with_cube = true; + else if (s_grouping_sets.ignore(pos, expected)) + select_query->group_by_with_grouping_sets = true; else if (s_totals.ignore(pos, expected)) select_query->group_by_with_totals = true; else diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index 2a515fdf3be..4a55e8b9b5f 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -14,7 +14,7 @@ public: Arenas arenas; AggregatedArenasChunkInfo(Arenas arenas_) : arenas(std::move(arenas_)) - {} + {} }; class AggregatedChunkInfo : public ChunkInfo From e0548d682dea31c0f7956e3be1124ad38730ea03 Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Tue, 9 Feb 2021 23:52:12 +0300 Subject: [PATCH 008/171] fix typos --- src/Processors/Transforms/AggregatingTransform.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index 4a55e8b9b5f..2a515fdf3be 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -14,7 +14,7 @@ public: Arenas arenas; AggregatedArenasChunkInfo(Arenas arenas_) : arenas(std::move(arenas_)) - {} + {} }; class AggregatedChunkInfo : public ChunkInfo From 3195d600c59de9caf48267cc12b59a5f30831202 Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Thu, 13 May 2021 17:51:07 +0300 Subject: [PATCH 009/171] feat grouping-sets: initial changes --- src/Core/ColumnNumbers.h | 1 + src/Core/NamesAndTypes.h | 2 + src/Interpreters/Aggregator.h | 22 ++++ src/Interpreters/ExpressionAnalyzer.cpp | 11 ++ src/Interpreters/ExpressionAnalyzer.h | 4 + src/Interpreters/InterpreterSelectQuery.cpp | 106 +++++++++++++++++- src/Interpreters/InterpreterSelectQuery.h | 3 +- .../Transforms/GroupingSetsTransform.cpp | 9 +- .../Transforms/GroupingSetsTransform.h | 2 + .../Transforms/TotalsHavingTransform.cpp | 10 ++ .../Transforms/TotalsHavingTransform.h | 4 + 11 files changed, 167 insertions(+), 7 deletions(-) diff --git a/src/Core/ColumnNumbers.h b/src/Core/ColumnNumbers.h index 9441f6485a7..82868ef3cd3 100644 --- a/src/Core/ColumnNumbers.h +++ b/src/Core/ColumnNumbers.h @@ -8,5 +8,6 @@ namespace DB { using ColumnNumbers = std::vector; +using ColumnNumbersTwoDimension = std::vector; } diff --git a/src/Core/NamesAndTypes.h b/src/Core/NamesAndTypes.h index 3ac9ad2fa02..dacd5a459dc 100644 --- a/src/Core/NamesAndTypes.h +++ b/src/Core/NamesAndTypes.h @@ -109,6 +109,8 @@ public: std::optional tryGetByName(const std::string & name) const; }; +using TwoDimensionNamesAndTypesList = std::list; + } namespace std diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index c79c2c5ef64..7933d62dd84 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -879,6 +879,7 @@ public: /// What to count. const ColumnNumbers keys; + const ColumnNumbersTwoDimension keys_vector; const AggregateDescriptions aggregates; const size_t keys_size; const size_t aggregates_size; @@ -938,6 +939,27 @@ public: { } + /// two dimensional vector of aggregating keys in params + Params( + const Block & src_header_, + const ColumnNumbersTwoDimension & keys_vector_, const AggregateDescriptions & aggregates_, + bool overflow_row_, size_t max_rows_to_group_by_, OverflowMode group_by_overflow_mode_, + size_t group_by_two_level_threshold_, size_t group_by_two_level_threshold_bytes_, + size_t max_bytes_before_external_group_by_, + bool empty_result_for_aggregation_by_empty_set_, + VolumePtr tmp_volume_, size_t max_threads_, + size_t min_free_disk_space_) + : src_header(src_header_), + keys_vector(keys_vector_), aggregates(aggregates_), keys_size(keys.size()), aggregates_size(aggregates.size()), + overflow_row(overflow_row_), max_rows_to_group_by(max_rows_to_group_by_), group_by_overflow_mode(group_by_overflow_mode_), + group_by_two_level_threshold(group_by_two_level_threshold_), group_by_two_level_threshold_bytes(group_by_two_level_threshold_bytes_), + max_bytes_before_external_group_by(max_bytes_before_external_group_by_), + empty_result_for_aggregation_by_empty_set(empty_result_for_aggregation_by_empty_set_), + tmp_volume(tmp_volume_), max_threads(max_threads_), + min_free_disk_space(min_free_disk_space_) + { + } + /// Only parameters that matter during merge. Params(const Block & intermediate_header_, const ColumnNumbers & keys_, const AggregateDescriptions & aggregates_, bool overflow_row_, size_t max_threads_) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index c195cb93c5e..7232c9eec80 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -339,6 +339,12 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) if (getContext()->getSettingsRef().enable_positional_arguments) replaceForPositionalArguments(group_asts[i], select_query, ASTSelectQuery::Expression::GROUP_BY); + if (select_query->group_by_with_grouping_sets) + { + LOG_DEBUG(poco_log, "analyzeAggregation: detect group by with grouping sets"); + /// TODO + } + getRootActionsNoMakeSet(group_asts[i], true, temp_actions, false); const auto & column_name = group_asts[i]->getColumnName(); @@ -374,6 +380,11 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) /// Aggregation keys are uniqued. if (!unique_keys.count(key.name)) { + if (select_query->group_by_with_grouping_sets) + { + aggregation_keys_list.push_back({key}); + } + unique_keys.insert(key.name); aggregation_keys.push_back(key); diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index b6bb3c5fad5..7598efa40c6 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -65,6 +65,7 @@ struct ExpressionAnalyzerData bool has_aggregation = false; NamesAndTypesList aggregation_keys; bool has_const_aggregation_keys = false; + TwoDimensionNamesAndTypesList aggregation_keys_list; AggregateDescriptions aggregate_descriptions; WindowDescriptions window_descriptions; @@ -94,6 +95,8 @@ private: explicit ExtractedSettings(const Settings & settings_); }; + Poco::Logger * poco_log = &Poco::Logger::get("ExpressionAnalyzer"); + public: /// Ctor for non-select queries. Generally its usage is: /// auto actions = ExpressionAnalyzer(query, syntax, context).getActions(); @@ -321,6 +324,7 @@ public: const NamesAndTypesList & aggregationKeys() const { return aggregation_keys; } bool hasConstAggregationKeys() const { return has_const_aggregation_keys; } + const TwoDimensionNamesAndTypesList & aggregationKeysList() const { return aggregation_keys_list; } const AggregateDescriptions & aggregates() const { return aggregate_descriptions; } const PreparedSets & getPreparedSets() const { return prepared_sets; } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index f015ee73ec5..e69a4b767ec 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1186,8 +1186,18 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

aggregationKeys()) + { keys.push_back(header_before_aggregation.getPositionByName(key.name)); + LOG_DEBUG(log, "executeAggregation pushed back key with name {} and number {}", key.name, header_before_aggregation.getPositionByName(key.name)); + } AggregateDescriptions aggregates = query_analyzer->aggregates(); for (auto & descr : aggregates) @@ -2207,14 +2218,18 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPlan & query_plan, Modific query_plan.addStep(std::move(step)); } -void InterpreterSelectQuery::executeGroupingSets(QueryPlan & query_plan) +void InterpreterSelectQuery::executeGroupingSets(QueryPlan & query_plan, const ActionsDAGPtr & expression, bool overflow_row, bool final, InputOrderInfoPtr group_by_info) { + /* const auto & header_before_transform = query_plan.getCurrentDataStream().header; ColumnNumbers keys; for (const auto & key : query_analyzer->aggregationKeys()) + { keys.push_back(header_before_transform.getPositionByName(key.name)); + LOG_DEBUG(log, "executeGroupingSets pushed back key with name {} and number {}", key.name, header_before_transform.getPositionByName(key.name)); + } const Settings & settings = context->getSettingsRef(); @@ -2230,6 +2245,87 @@ void InterpreterSelectQuery::executeGroupingSets(QueryPlan & query_plan) step = std::make_unique(query_plan.getCurrentDataStream(), std::move(transform_params)); query_plan.addStep(std::move(step)); + */ + + auto expression_before_aggregation = std::make_unique(query_plan.getCurrentDataStream(), expression); + expression_before_aggregation->setStepDescription("Before GROUP BY"); + query_plan.addStep(std::move(expression_before_aggregation)); + + const auto & header_before_aggregation = query_plan.getCurrentDataStream().header; + ColumnNumbers keys; + ColumnNumbersTwoDimension keys_vector; + for (const auto & aggregation_keys : query_analyzer->aggregationKeysList()) + { + keys.clear(); + for (const auto & key : aggregation_keys) + { + keys.push_back(header_before_aggregation.getPositionByName(key.name)); + LOG_DEBUG( + log, + "executeGroupingSets add key with name {} and number {}", + key.name, + header_before_aggregation.getPositionByName(key.name)); + } + keys_vector.push_back(keys); + } + + AggregateDescriptions aggregates = query_analyzer->aggregates(); + for (auto & descr : aggregates) + if (descr.arguments.empty()) + for (const auto & name : descr.argument_names) + { + descr.arguments.push_back(header_before_aggregation.getPositionByName(name)); + LOG_DEBUG( + log, + "executeGroupingSets add descr.atruments with name {} and number {}", + name, + header_before_aggregation.getPositionByName(name)); + } + + const Settings & settings = context->getSettingsRef(); + + Aggregator::Params params( + header_before_aggregation, + keys_vector, + aggregates, + overflow_row, + settings.max_rows_to_group_by, + settings.group_by_overflow_mode, + settings.group_by_two_level_threshold, + settings.group_by_two_level_threshold_bytes, + settings.max_bytes_before_external_group_by, + settings.empty_result_for_aggregation_by_empty_set, + context->getTemporaryVolume(), + settings.max_threads, + settings.min_free_disk_space_for_temporary_data); + + SortDescription group_by_sort_description; + + if (group_by_info && settings.optimize_aggregation_in_order) + group_by_sort_description = getSortDescriptionFromGroupBy(getSelectQuery()); + else + group_by_info = nullptr; + + auto merge_threads = max_streams; + auto temporary_data_merge_threads = settings.aggregation_memory_efficient_merge_threads + ? static_cast(settings.aggregation_memory_efficient_merge_threads) + : static_cast(settings.max_threads); + + bool storage_has_evenly_distributed_read = storage && storage->hasEvenlyDistributedRead(); + + auto aggregating_step = std::make_unique( + query_plan.getCurrentDataStream(), + params, + final, + settings.max_block_size, + settings.aggregation_in_order_max_block_bytes, + merge_threads, + temporary_data_merge_threads, + storage_has_evenly_distributed_read, + std::move(group_by_info), + std::move(group_by_sort_description)); + + query_plan.addStep(std::move(aggregating_step)); } void InterpreterSelectQuery::executeExpression(QueryPlan & query_plan, const ActionsDAGPtr & expression, const std::string & description) diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 8ea04cf7b27..db77a8bd8cb 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -14,6 +14,7 @@ #include #include +#include "Interpreters/ActionsDAG.h" namespace Poco { @@ -160,7 +161,7 @@ private: }; void executeRollupOrCube(QueryPlan & query_plan, Modificator modificator); - void executeGroupingSets(QueryPlan & query_plan); + void executeGroupingSets(QueryPlan & query_plan, const ActionsDAGPtr & expression, bool overflow_row, bool final, InputOrderInfoPtr group_by_info); /** If there is a SETTINGS section in the SELECT query, then apply settings from it. * diff --git a/src/Processors/Transforms/GroupingSetsTransform.cpp b/src/Processors/Transforms/GroupingSetsTransform.cpp index 637f244ff96..ccbafac8e3e 100644 --- a/src/Processors/Transforms/GroupingSetsTransform.cpp +++ b/src/Processors/Transforms/GroupingSetsTransform.cpp @@ -19,6 +19,7 @@ GroupingSetsTransform::GroupingSetsTransform(Block header, AggregatingTransformP Chunk GroupingSetsTransform::merge(Chunks && chunks, bool final) { + LOG_DEBUG(log, "merge {} blocks", chunks.size()); BlocksList rollup_blocks; for (auto & chunk : chunks) rollup_blocks.emplace_back(getInputPort().getHeader().cloneWithColumns(chunk.detachColumns())); @@ -31,12 +32,15 @@ Chunk GroupingSetsTransform::merge(Chunks && chunks, bool final) void GroupingSetsTransform::consume(Chunk chunk) { consumed_chunks.emplace_back(std::move(chunk)); + LOG_DEBUG(log, "consumed block, now consumed_chunks size is {}", consumed_chunks.size()); } Chunk GroupingSetsTransform::generate() { + LOG_DEBUG(log, "generate start, mask = {}", mask); if (!consumed_chunks.empty()) { + LOG_DEBUG(log, "consumed_chunks not empty, size is {}", consumed_chunks.size()); if (consumed_chunks.size() > 1) grouping_sets_chunk = merge(std::move(consumed_chunks), false); else @@ -46,6 +50,7 @@ Chunk GroupingSetsTransform::generate() auto num_rows = grouping_sets_chunk.getNumRows(); mask = (UInt64(1) << keys.size()); + LOG_DEBUG(log, "changed mask, mask = {}", mask); current_columns = grouping_sets_chunk.getColumns(); current_zero_columns.clear(); @@ -56,9 +61,10 @@ Chunk GroupingSetsTransform::generate() } // auto gen_chunk = std::move(cube_chunk); - + LOG_DEBUG(log, "before if mask"); if (mask > 1) { + LOG_DEBUG(log, "in if mask > 1"); mask = mask >> 1; auto columns = current_columns; @@ -72,6 +78,7 @@ Chunk GroupingSetsTransform::generate() chunks.emplace_back(std::move(columns), current_columns.front()->size()); grouping_sets_chunk = merge(std::move(chunks), false); } + LOG_DEBUG(log, "before gen_chunk"); auto gen_chunk = std::move(grouping_sets_chunk); finalizeChunk(gen_chunk); diff --git a/src/Processors/Transforms/GroupingSetsTransform.h b/src/Processors/Transforms/GroupingSetsTransform.h index 0e300453215..10108d3c1a7 100644 --- a/src/Processors/Transforms/GroupingSetsTransform.h +++ b/src/Processors/Transforms/GroupingSetsTransform.h @@ -29,6 +29,8 @@ private: UInt64 mask = 0; + Poco::Logger * log = &Poco::Logger::get("GroupingSetsTransform"); + Chunk merge(Chunks && chunks, bool final); }; diff --git a/src/Processors/Transforms/TotalsHavingTransform.cpp b/src/Processors/Transforms/TotalsHavingTransform.cpp index 0b7797da24f..a636ba70ef9 100644 --- a/src/Processors/Transforms/TotalsHavingTransform.cpp +++ b/src/Processors/Transforms/TotalsHavingTransform.cpp @@ -119,6 +119,7 @@ TotalsHavingTransform::TotalsHavingTransform( IProcessor::Status TotalsHavingTransform::prepare() { + LOG_DEBUG(log, "TotalsHavingTransform::prepare()"); if (!finished_transform) { auto status = ISimpleTransform::prepare(); @@ -143,20 +144,24 @@ IProcessor::Status TotalsHavingTransform::prepare() totals_output.push(std::move(totals)); totals_output.finish(); + LOG_DEBUG(log, "exit TotalsHavingTransform::prepare()"); return Status::Finished; } void TotalsHavingTransform::work() { + LOG_DEBUG(log, "TotalsHavingTransform::work()"); if (finished_transform) prepareTotals(); else ISimpleTransform::work(); + LOG_DEBUG(log, "exit TotalsHavingTransform::work()"); } void TotalsHavingTransform::transform(Chunk & chunk) { /// Block with values not included in `max_rows_to_group_by`. We'll postpone it. + LOG_DEBUG(log, "TotalsHavingTransform::transform()"); if (overflow_row) { const auto & info = chunk.getChunkInfo(); @@ -249,10 +254,12 @@ void TotalsHavingTransform::transform(Chunk & chunk) } passed_keys += chunk.getNumRows(); + LOG_DEBUG(log, "exit TotalsHavingTransform::transform()"); } void TotalsHavingTransform::addToTotals(const Chunk & chunk, const IColumn::Filter * filter) { + LOG_DEBUG(log, "TotalsHavingTransform::addToTotals()"); auto num_columns = chunk.getNumColumns(); for (size_t col = 0; col < num_columns; ++col) { @@ -284,10 +291,12 @@ void TotalsHavingTransform::addToTotals(const Chunk & chunk, const IColumn::Filt } } } + LOG_DEBUG(log, "exit TotalsHavingTransform::addToTotals()"); } void TotalsHavingTransform::prepareTotals() { + LOG_DEBUG(log, "TotalsHavingTransform::prepareTotals()"); /// If totals_mode == AFTER_HAVING_AUTO, you need to decide whether to add aggregates to TOTALS for strings, /// not passed max_rows_to_group_by. if (overflow_aggregates) @@ -312,6 +321,7 @@ void TotalsHavingTransform::prepareTotals() /// Note: after expression totals may have several rows if `arrayJoin` was used in expression. totals = Chunk(block.getColumns(), num_rows); } + LOG_DEBUG(log, "exit TotalsHavingTransform::prepareTotals()"); } } diff --git a/src/Processors/Transforms/TotalsHavingTransform.h b/src/Processors/Transforms/TotalsHavingTransform.h index 03635054c65..59fa58edfbc 100644 --- a/src/Processors/Transforms/TotalsHavingTransform.h +++ b/src/Processors/Transforms/TotalsHavingTransform.h @@ -2,6 +2,8 @@ #include #include +#include // to be removed +// #include namespace DB { @@ -74,6 +76,8 @@ private: /// Here, total values are accumulated. After the work is finished, they will be placed in totals. MutableColumns current_totals; + + Poco::Logger * log = &Poco::Logger::get("TotalsHavingTransform"); }; void finalizeChunk(Chunk & chunk); From 390c8126a44156952c5b8bcbf3ee1201b47477c2 Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Sat, 15 May 2021 15:14:58 +0300 Subject: [PATCH 010/171] development --- src/Interpreters/ExpressionAnalyzer.cpp | 30 +++++++++++++++++++------ src/Interpreters/TreeOptimizer.cpp | 9 +++++--- src/Parsers/ExpressionListParsers.cpp | 6 +++++ src/Parsers/ExpressionListParsers.h | 8 +++++++ src/Parsers/ParserSelectQuery.cpp | 14 ++++++++++-- 5 files changed, 55 insertions(+), 12 deletions(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 7232c9eec80..dd8fbd8fe04 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -380,13 +380,16 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) /// Aggregation keys are uniqued. if (!unique_keys.count(key.name)) { + unique_keys.insert(key.name); if (select_query->group_by_with_grouping_sets) { aggregation_keys_list.push_back({key}); + LOG_DEBUG(poco_log, "pushed grouping set of 1 column: " + key.name); + } + else + { + aggregation_keys.push_back(key); } - - unique_keys.insert(key.name); - aggregation_keys.push_back(key); /// Key is no longer needed, therefore we can save a little by moving it. aggregated_columns.push_back(std::move(key)); @@ -1150,10 +1153,23 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain ExpressionActionsChain::Step & step = chain.lastStep(columns_after_join); ASTs asts = select_query->groupBy()->children; - for (const auto & ast : asts) + if (select_query->group_by_with_grouping_sets) { - step.addRequiredOutput(ast->getColumnName()); - getRootActions(ast, only_types, step.actions()); + for (const auto & inner_asts : asts) { + for (const auto & ast : inner_asts->children) + { + step.addRequiredOutput(ast->getColumnName()); + getRootActions(ast, only_types, step.actions()); + } + } + } + else + { + for (const auto & ast : asts) + { + step.addRequiredOutput(ast->getColumnName()); + getRootActions(ast, only_types, step.actions()); + } } if (optimize_aggregation_in_order) @@ -1644,7 +1660,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( /// TODO correct conditions optimize_aggregation_in_order = context->getSettingsRef().optimize_aggregation_in_order - && storage && query.groupBy(); + && storage && query.groupBy() && !query.group_by_with_grouping_sets; query_analyzer.appendGroupBy(chain, only_types || !first_stage, optimize_aggregation_in_order, group_by_elements_actions); query_analyzer.appendAggregateFunctionsArguments(chain, only_types || !first_stage); diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index 64b25ca9777..1d6ed76b9c4 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -84,6 +84,9 @@ void optimizeGroupBy(ASTSelectQuery * select_query, ContextPtr context) { const FunctionFactory & function_factory = FunctionFactory::instance(); + if (select_query->group_by_with_grouping_sets) + return; + if (!select_query->groupBy()) return; @@ -213,7 +216,7 @@ GroupByKeysInfo getGroupByKeysInfo(const ASTs & group_by_keys) ///eliminate functions of other GROUP BY keys void optimizeGroupByFunctionKeys(ASTSelectQuery * select_query) { - if (!select_query->groupBy()) + if (!select_query->groupBy() || select_query->group_by_with_grouping_sets) return; auto group_by = select_query->groupBy(); @@ -243,7 +246,7 @@ void optimizeGroupByFunctionKeys(ASTSelectQuery * select_query) /// Eliminates min/max/any-aggregators of functions of GROUP BY keys void optimizeAggregateFunctionsOfGroupByKeys(ASTSelectQuery * select_query, ASTPtr & node) { - if (!select_query->groupBy()) + if (!select_query->groupBy() || select_query->group_by_with_grouping_sets) return; const auto & group_by_keys = select_query->groupBy()->children; @@ -413,7 +416,7 @@ void optimizeMonotonousFunctionsInOrderBy(ASTSelectQuery * select_query, Context const TreeRewriterResult & result) { auto order_by = select_query->orderBy(); - if (!order_by) + if (!order_by || select_query->group_by_with_grouping_sets) return; /// Do not apply optimization for Distributed and Merge storages, diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index 96c1bad75c2..ce06485ae8e 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -756,6 +756,12 @@ bool ParserOrderByExpressionList::parseImpl(Pos & pos, ASTPtr & node, Expected & .parse(pos, node, expected); } +bool ParserGroupingSetsExpressionList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + return ParserList(std::make_unique(false), + std::make_unique(TokenType::Comma), false) + .parse(pos, node, expected); +} bool ParserTTLExpressionList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h index a035d4a2ef0..cd0f398c32d 100644 --- a/src/Parsers/ExpressionListParsers.h +++ b/src/Parsers/ExpressionListParsers.h @@ -508,6 +508,14 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; +class ParserGroupingSetsExpressionList : public IParserBase +{ +protected: + const char * getName() const override { return "grouping sets expression"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + + /// Parser for key-value pair, where value can be list of pairs. class ParserKeyValuePair : public IParserBase diff --git a/src/Parsers/ParserSelectQuery.cpp b/src/Parsers/ParserSelectQuery.cpp index 6a17993a704..f767ee88b96 100644 --- a/src/Parsers/ParserSelectQuery.cpp +++ b/src/Parsers/ParserSelectQuery.cpp @@ -66,6 +66,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserNotEmptyExpressionList exp_list_for_select_clause(true); /// Allows aliases without AS keyword. ParserExpressionWithOptionalAlias exp_elem(false); ParserOrderByExpressionList order_list; + ParserGroupingSetsExpressionList grouping_sets_list; ParserToken open_bracket(TokenType::OpeningRoundBracket); ParserToken close_bracket(TokenType::ClosingRoundBracket); @@ -192,8 +193,17 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) !open_bracket.ignore(pos, expected)) return false; - if (!exp_list.parse(pos, group_expression_list, expected)) - return false; + if (select_query->group_by_with_grouping_sets) + { + if (!grouping_sets_list.parse(pos, group_expression_list, expected)) + return false; + } + else + { + if (!exp_list.parse(pos, group_expression_list, expected)) + return false; + } + if ((select_query->group_by_with_rollup || select_query->group_by_with_cube || select_query->group_by_with_grouping_sets) && !close_bracket.ignore(pos, expected)) From abe09324c1a21dcb3ada9c8f4536d9917b11d5d3 Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Sat, 15 May 2021 19:41:22 +0300 Subject: [PATCH 011/171] grouping sets development --- src/Interpreters/Aggregator.h | 10 +- src/Interpreters/InterpreterSelectQuery.cpp | 142 ++++++++++++++------ src/Interpreters/InterpreterSelectQuery.h | 1 - 3 files changed, 111 insertions(+), 42 deletions(-) diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 7933d62dd84..b5d819be8be 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -948,15 +948,21 @@ public: size_t max_bytes_before_external_group_by_, bool empty_result_for_aggregation_by_empty_set_, VolumePtr tmp_volume_, size_t max_threads_, - size_t min_free_disk_space_) + size_t min_free_disk_space_, + bool compile_aggregate_expressions_, + size_t min_count_to_compile_aggregate_expression_, + const Block & intermediate_header_ = {}) : src_header(src_header_), + intermediate_header(intermediate_header_), keys_vector(keys_vector_), aggregates(aggregates_), keys_size(keys.size()), aggregates_size(aggregates.size()), overflow_row(overflow_row_), max_rows_to_group_by(max_rows_to_group_by_), group_by_overflow_mode(group_by_overflow_mode_), group_by_two_level_threshold(group_by_two_level_threshold_), group_by_two_level_threshold_bytes(group_by_two_level_threshold_bytes_), max_bytes_before_external_group_by(max_bytes_before_external_group_by_), empty_result_for_aggregation_by_empty_set(empty_result_for_aggregation_by_empty_set_), tmp_volume(tmp_volume_), max_threads(max_threads_), - min_free_disk_space(min_free_disk_space_) + min_free_disk_space(min_free_disk_space_), + compile_aggregate_expressions(compile_aggregate_expressions_), + min_count_to_compile_aggregate_expression(min_count_to_compile_aggregate_expression_) { } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index e69a4b767ec..a28e99d510a 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1186,18 +1186,10 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

aggregationKeys()) + ColumnNumbersTwoDimension keys_vector; + auto & query = getSelectQuery(); + if (query.group_by_with_grouping_sets) { - keys.push_back(header_before_aggregation.getPositionByName(key.name)); - LOG_DEBUG(log, "executeAggregation pushed back key with name {} and number {}", key.name, header_before_aggregation.getPositionByName(key.name)); + for (const auto & aggregation_keys : query_analyzer->aggregationKeysList()) + { + keys.clear(); + for (const auto & key : aggregation_keys) + { + keys.push_back(header_before_aggregation.getPositionByName(key.name)); + LOG_DEBUG( + log, + "GroupingSets add key with name {} and number {}", + key.name, + header_before_aggregation.getPositionByName(key.name)); + } + keys_vector.push_back(keys); + LOG_DEBUG( + log, + "GroupingSets add keys set of size {}", + keys.size()); + } + } + else + { + for (const auto & key : query_analyzer->aggregationKeys()) + { + keys.push_back(header_before_aggregation.getPositionByName(key.name)); + LOG_DEBUG(log, "executeAggregation pushed back key with name {} and number {}", key.name, header_before_aggregation.getPositionByName(key.name)); + } } AggregateDescriptions aggregates = query_analyzer->aggregates(); @@ -2082,29 +2100,72 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac const Settings & settings = context->getSettingsRef(); - Aggregator::Params params( - header_before_aggregation, - keys, - aggregates, - overflow_row, - settings.max_rows_to_group_by, - settings.group_by_overflow_mode, - settings.group_by_two_level_threshold, - settings.group_by_two_level_threshold_bytes, - settings.max_bytes_before_external_group_by, - settings.empty_result_for_aggregation_by_empty_set - || (settings.empty_result_for_aggregation_by_constant_keys_on_empty_set && keys.empty() - && query_analyzer->hasConstAggregationKeys()), - context->getTemporaryVolume(), - settings.max_threads, - settings.min_free_disk_space_for_temporary_data, - settings.compile_aggregate_expressions, - settings.min_count_to_compile_aggregate_expression); + // Aggregator::Params params( + // header_before_aggregation, + // keys, + // aggregates, + // overflow_row, + // settings.max_rows_to_group_by, + // settings.group_by_overflow_mode, + // settings.group_by_two_level_threshold, + // settings.group_by_two_level_threshold_bytes, + // settings.max_bytes_before_external_group_by, + // settings.empty_result_for_aggregation_by_empty_set + // || (settings.empty_result_for_aggregation_by_constant_keys_on_empty_set && keys.empty() + // && query_analyzer->hasConstAggregationKeys()), + // context->getTemporaryVolume(), + // settings.max_threads, + // settings.min_free_disk_space_for_temporary_data, + // settings.compile_aggregate_expressions, + // settings.min_count_to_compile_aggregate_expression); + std::shared_ptr params_ptr; + if (query.group_by_with_grouping_sets) + { + params_ptr = std::make_shared( + header_before_aggregation, + keys_vector, + aggregates, + overflow_row, + settings.max_rows_to_group_by, + settings.group_by_overflow_mode, + settings.group_by_two_level_threshold, + settings.group_by_two_level_threshold_bytes, + settings.max_bytes_before_external_group_by, + settings.empty_result_for_aggregation_by_empty_set + || (settings.empty_result_for_aggregation_by_constant_keys_on_empty_set && keys.empty() + && query_analyzer->hasConstAggregationKeys()), + context->getTemporaryVolume(), + settings.max_threads, + settings.min_free_disk_space_for_temporary_data, + settings.compile_aggregate_expressions, + settings.min_count_to_compile_aggregate_expression); + } + else + { + params_ptr = std::make_shared( + header_before_aggregation, + keys, + aggregates, + overflow_row, + settings.max_rows_to_group_by, + settings.group_by_overflow_mode, + settings.group_by_two_level_threshold, + settings.group_by_two_level_threshold_bytes, + settings.max_bytes_before_external_group_by, + settings.empty_result_for_aggregation_by_empty_set + || (settings.empty_result_for_aggregation_by_constant_keys_on_empty_set && keys.empty() + && query_analyzer->hasConstAggregationKeys()), + context->getTemporaryVolume(), + settings.max_threads, + settings.min_free_disk_space_for_temporary_data, + settings.compile_aggregate_expressions, + settings.min_count_to_compile_aggregate_expression); + } SortDescription group_by_sort_description; - if (group_by_info && settings.optimize_aggregation_in_order) - group_by_sort_description = getSortDescriptionFromGroupBy(getSelectQuery()); + if (group_by_info && settings.optimize_aggregation_in_order && !query.group_by_with_grouping_sets) + group_by_sort_description = getSortDescriptionFromGroupBy(query); else group_by_info = nullptr; @@ -2117,7 +2178,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac auto aggregating_step = std::make_unique( query_plan.getCurrentDataStream(), - params, + *params_ptr, final, settings.max_block_size, settings.aggregation_in_order_max_block_bytes, @@ -2218,9 +2279,10 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPlan & query_plan, Modific query_plan.addStep(std::move(step)); } -void InterpreterSelectQuery::executeGroupingSets(QueryPlan & query_plan, const ActionsDAGPtr & expression, bool overflow_row, bool final, InputOrderInfoPtr group_by_info) +/* +void InterpreterSelectQuery::executeGroupingSets(QueryPlan & query_plan, const ExpressionActionsPtr & expression, bool overflow_row, bool final, InputOrderInfoPtr group_by_info) { - /* + const auto & header_before_transform = query_plan.getCurrentDataStream().header; ColumnNumbers keys; @@ -2245,7 +2307,7 @@ void InterpreterSelectQuery::executeGroupingSets(QueryPlan & query_plan, const A step = std::make_unique(query_plan.getCurrentDataStream(), std::move(transform_params)); query_plan.addStep(std::move(step)); - */ + auto expression_before_aggregation = std::make_unique(query_plan.getCurrentDataStream(), expression); expression_before_aggregation->setStepDescription("Before GROUP BY"); @@ -2326,7 +2388,9 @@ void InterpreterSelectQuery::executeGroupingSets(QueryPlan & query_plan, const A std::move(group_by_sort_description)); query_plan.addStep(std::move(aggregating_step)); + } +*/ void InterpreterSelectQuery::executeExpression(QueryPlan & query_plan, const ActionsDAGPtr & expression, const std::string & description) { diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index db77a8bd8cb..34d56a7354b 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -161,7 +161,6 @@ private: }; void executeRollupOrCube(QueryPlan & query_plan, Modificator modificator); - void executeGroupingSets(QueryPlan & query_plan, const ActionsDAGPtr & expression, bool overflow_row, bool final, InputOrderInfoPtr group_by_info); /** If there is a SETTINGS section in the SELECT query, then apply settings from it. * From 4d1b354b5fbf6c549da5ab5d734fb44c2336316c Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Sat, 15 May 2021 22:58:41 +0300 Subject: [PATCH 012/171] grouping sets development --- src/Interpreters/Aggregator.h | 2 +- src/QueryPipeline/Pipe.cpp | 55 +++++++++++++++++++++- src/QueryPipeline/Pipe.h | 1 + src/QueryPipeline/QueryPipelineBuilder.cpp | 6 +++ src/QueryPipeline/QueryPipelineBuilder.h | 2 + 5 files changed, 64 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index b5d819be8be..1f722bad320 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -878,7 +878,7 @@ public: Block intermediate_header; /// What to count. - const ColumnNumbers keys; + ColumnNumbers keys; const ColumnNumbersTwoDimension keys_vector; const AggregateDescriptions aggregates; const size_t keys_size; diff --git a/src/QueryPipeline/Pipe.cpp b/src/QueryPipeline/Pipe.cpp index 6cef7cc28bd..093491eed2a 100644 --- a/src/QueryPipeline/Pipe.cpp +++ b/src/QueryPipeline/Pipe.cpp @@ -432,6 +432,59 @@ void Pipe::addTransform(ProcessorPtr transform) addTransform(std::move(transform), static_cast(nullptr), static_cast(nullptr)); } +void Pipe::addParallelTransforms(Processors transforms) +{ + if (output_ports.empty()) + throw Exception("Cannot add parallel transforms to empty Pipe.", ErrorCodes::LOGICAL_ERROR); + + std::vector inputs; + std::vector outputs; + for (const auto & transform : transforms) + { + auto current_transform_inputs = transform->getInputs(); + if (current_transform_inputs.size() != 1) + throw Exception("Each parallel transform should have one input port", ErrorCodes::LOGICAL_ERROR); + + inputs.push_back(current_transform_inputs.front()); + + auto current_transform_outputs = transform->getOutputs(); + if (current_transform_outputs.size() != 1) + throw Exception("Each parallel transform should have one output port", ErrorCodes::LOGICAL_ERROR); + + outputs.push_back(current_transform_outputs.front()); + } + + if (inputs.size() != output_ports.size()) + throw Exception("Cannot add parallel transforms to Pipes because " + + std::to_string(transforms.size()) + " transforms were passed, " + "but " + std::to_string(output_ports.size()) + " expected", ErrorCodes::LOGICAL_ERROR); + + size_t next_output = 0; + for (auto & input : inputs) + { + connect(*output_ports[next_output], input); + ++next_output; + } + + output_ports.clear(); + output_ports.reserve(outputs.size()); + + for (auto & output : outputs) + { + output_ports.emplace_back(&output); + } + + /// do not check output formats because they are different in case of parallel aggregations + + if (collected_processors) + collected_processors->insert(collected_processors->end(), transforms.begin(), transforms.end()); + + processors.insert(processors.end(), transforms.begin(), transforms.end()); + + /// Should not change streams number, so maybe not need max_parallel_streams update + max_parallel_streams = std::max(max_parallel_streams, output_ports.size()); +} + void Pipe::addTransform(ProcessorPtr transform, OutputPort * totals, OutputPort * extremes) { if (output_ports.empty()) @@ -497,7 +550,7 @@ void Pipe::addTransform(ProcessorPtr transform, OutputPort * totals, OutputPort for (size_t i = 1; i < output_ports.size(); ++i) assertBlocksHaveEqualStructure(header, output_ports[i]->getHeader(), "Pipes"); - // Temporarily skip this check. TotaslHavingTransform may return finalized totals but not finalized data. + // Temporarily skip this check. TotalsHavingTransform may return finalized totals but not finalized data. // if (totals_port) // assertBlocksHaveEqualStructure(header, totals_port->getHeader(), "Pipes"); diff --git a/src/QueryPipeline/Pipe.h b/src/QueryPipeline/Pipe.h index 613e92a782d..e655f807ec8 100644 --- a/src/QueryPipeline/Pipe.h +++ b/src/QueryPipeline/Pipe.h @@ -63,6 +63,7 @@ public: /// Output ports should have same headers. /// If totals or extremes are not empty, transform shouldn't change header. void addTransform(ProcessorPtr transform); + void addParallelTransforms(Processors transform); void addTransform(ProcessorPtr transform, OutputPort * totals, OutputPort * extremes); void addTransform(ProcessorPtr transform, InputPort * totals, InputPort * extremes); diff --git a/src/QueryPipeline/QueryPipelineBuilder.cpp b/src/QueryPipeline/QueryPipelineBuilder.cpp index dba7c7cb8f7..952900c772e 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.cpp +++ b/src/QueryPipeline/QueryPipelineBuilder.cpp @@ -155,6 +155,12 @@ void QueryPipelineBuilder::transform(const Transformer & transformer) pipe.transform(transformer); } +void QueryPipelineBuilder::addParallelTransforms(Processors transforms) +{ + checkInitializedAndNotCompleted(); + pipe.addParallelTransforms(transforms); +} + void QueryPipelineBuilder::setSinks(const Pipe::ProcessorGetterWithStreamKind & getter) { checkInitializedAndNotCompleted(); diff --git a/src/QueryPipeline/QueryPipelineBuilder.h b/src/QueryPipeline/QueryPipelineBuilder.h index 9e198f45e98..9428bbc5d0a 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.h +++ b/src/QueryPipeline/QueryPipelineBuilder.h @@ -67,6 +67,8 @@ public: /// Transform pipeline in general way. void transform(const Transformer & transformer); + /// Add transform and connects it to outputs[stream_index] stream + void addParallelTransforms(Processors transform); /// Add TotalsHavingTransform. Resize pipeline to single input. Adds totals port. void addTotalsHavingTransform(ProcessorPtr transform); /// Add transform which calculates extremes. This transform adds extremes port and doesn't change inputs number. From e6bd807f6074e9d68a1fc317727a34012cc67e6e Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Sun, 16 May 2021 22:44:20 +0300 Subject: [PATCH 013/171] grouping sets development --- src/Interpreters/Aggregator.h | 4 +- src/Interpreters/ExpressionAnalyzer.cpp | 1 + src/Interpreters/InterpreterSelectQuery.cpp | 45 +++++++++---------- .../Transforms/AggregatingTransform.cpp | 1 + src/QueryPipeline/Pipe.cpp | 22 ++++++++- src/QueryPipeline/Pipe.h | 3 ++ src/QueryPipeline/QueryPipelineBuilder.h | 2 +- 7 files changed, 52 insertions(+), 26 deletions(-) diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 1f722bad320..b9dca0b09ae 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -881,7 +881,7 @@ public: ColumnNumbers keys; const ColumnNumbersTwoDimension keys_vector; const AggregateDescriptions aggregates; - const size_t keys_size; + size_t keys_size; const size_t aggregates_size; /// The settings of approximate calculation of GROUP BY. @@ -942,6 +942,7 @@ public: /// two dimensional vector of aggregating keys in params Params( const Block & src_header_, + const ColumnNumbers & keys_, const ColumnNumbersTwoDimension & keys_vector_, const AggregateDescriptions & aggregates_, bool overflow_row_, size_t max_rows_to_group_by_, OverflowMode group_by_overflow_mode_, size_t group_by_two_level_threshold_, size_t group_by_two_level_threshold_bytes_, @@ -954,6 +955,7 @@ public: const Block & intermediate_header_ = {}) : src_header(src_header_), intermediate_header(intermediate_header_), + keys(keys_), keys_vector(keys_vector_), aggregates(aggregates_), keys_size(keys.size()), aggregates_size(aggregates.size()), overflow_row(overflow_row_), max_rows_to_group_by(max_rows_to_group_by_), group_by_overflow_mode(group_by_overflow_mode_), group_by_two_level_threshold(group_by_two_level_threshold_), group_by_two_level_threshold_bytes(group_by_two_level_threshold_bytes_), diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index dd8fbd8fe04..fcbe796032b 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -384,6 +384,7 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) if (select_query->group_by_with_grouping_sets) { aggregation_keys_list.push_back({key}); + aggregation_keys.push_back(key); LOG_DEBUG(poco_log, "pushed grouping set of 1 column: " + key.name); } else diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index a28e99d510a..cdf71f9894a 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1053,11 +1053,13 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

aggregates(); for (auto & descr : aggregates) if (descr.arguments.empty()) for (const auto & name : descr.argument_names) descr.arguments.push_back(header_before_aggregation.getPositionByName(name)); - + LOG_DEBUG(log, "GroupingSets debug 2"); const Settings & settings = context->getSettingsRef(); - - // Aggregator::Params params( - // header_before_aggregation, - // keys, - // aggregates, - // overflow_row, - // settings.max_rows_to_group_by, - // settings.group_by_overflow_mode, - // settings.group_by_two_level_threshold, - // settings.group_by_two_level_threshold_bytes, - // settings.max_bytes_before_external_group_by, - // settings.empty_result_for_aggregation_by_empty_set - // || (settings.empty_result_for_aggregation_by_constant_keys_on_empty_set && keys.empty() - // && query_analyzer->hasConstAggregationKeys()), - // context->getTemporaryVolume(), - // settings.max_threads, - // settings.min_free_disk_space_for_temporary_data, - // settings.compile_aggregate_expressions, - // settings.min_count_to_compile_aggregate_expression); + LOG_DEBUG(log, "GroupingSets debug 3"); std::shared_ptr params_ptr; if (query.group_by_with_grouping_sets) { + LOG_DEBUG(log, "GroupingSets debug 4"); params_ptr = std::make_shared( header_before_aggregation, + all_keys, keys_vector, aggregates, overflow_row, @@ -2161,7 +2157,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac settings.compile_aggregate_expressions, settings.min_count_to_compile_aggregate_expression); } - + LOG_DEBUG(log, "GroupingSets debug 5"); SortDescription group_by_sort_description; if (group_by_info && settings.optimize_aggregation_in_order && !query.group_by_with_grouping_sets) @@ -2175,7 +2171,8 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac : static_cast(settings.max_threads); bool storage_has_evenly_distributed_read = storage && storage->hasEvenlyDistributedRead(); - + LOG_DEBUG(log, "GroupingSets debug 6"); + LOG_DEBUG(log, "GroupingSets step header structure: {}", query_plan.getCurrentDataStream().header.dumpStructure()); auto aggregating_step = std::make_unique( query_plan.getCurrentDataStream(), *params_ptr, @@ -2187,8 +2184,10 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac storage_has_evenly_distributed_read, std::move(group_by_info), std::move(group_by_sort_description)); - + LOG_DEBUG(log, "GroupingSets step header structure: {}", aggregating_step->getOutputStream().header.dumpStructure()); + LOG_DEBUG(log, "GroupingSets debug 7"); query_plan.addStep(std::move(aggregating_step)); + LOG_DEBUG(log, "GroupingSets debug 8"); } void InterpreterSelectQuery::executeMergeAggregated(QueryPlan & query_plan, bool overflow_row, bool final) diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 5b58530f3d5..f1e292a92cc 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -501,6 +501,7 @@ void AggregatingTransform::work() Processors AggregatingTransform::expandPipeline() { + LOG_DEBUG(log, "in AggregatingTransform::expandPipeline"); auto & out = processors.back()->getOutputs().front(); inputs.emplace_back(out.getHeader(), this); connect(out, inputs.back()); diff --git a/src/QueryPipeline/Pipe.cpp b/src/QueryPipeline/Pipe.cpp index 093491eed2a..d9917b8636d 100644 --- a/src/QueryPipeline/Pipe.cpp +++ b/src/QueryPipeline/Pipe.cpp @@ -434,6 +434,8 @@ void Pipe::addTransform(ProcessorPtr transform) void Pipe::addParallelTransforms(Processors transforms) { + LOG_DEBUG(log, "Begin addParallelTransforms, have {} transforms", transforms.size()); + if (output_ports.empty()) throw Exception("Cannot add parallel transforms to empty Pipe.", ErrorCodes::LOGICAL_ERROR); @@ -452,6 +454,7 @@ void Pipe::addParallelTransforms(Processors transforms) throw Exception("Each parallel transform should have one output port", ErrorCodes::LOGICAL_ERROR); outputs.push_back(current_transform_outputs.front()); + LOG_DEBUG(log, "addParallelTransforms, added inputs and outputs for processor {}", transform->getName()); } if (inputs.size() != output_ports.size()) @@ -464,6 +467,7 @@ void Pipe::addParallelTransforms(Processors transforms) { connect(*output_ports[next_output], input); ++next_output; + LOG_DEBUG(log, "addParallelTransforms connect current output to new input {}", next_output); } output_ports.clear(); @@ -471,15 +475,22 @@ void Pipe::addParallelTransforms(Processors transforms) for (auto & output : outputs) { + LOG_DEBUG(log, "addParallelTransforms change outputs to new output"); + LOG_DEBUG(log, "addParallelTransforms is output connected: {}", output.isConnected()); output_ports.emplace_back(&output); } /// do not check output formats because they are different in case of parallel aggregations + LOG_DEBUG(log, "addParallelTransforms do not check format"); if (collected_processors) collected_processors->insert(collected_processors->end(), transforms.begin(), transforms.end()); - processors.insert(processors.end(), transforms.begin(), transforms.end()); + for (auto & transform_ptr : transforms) + { + processors.emplace_back(std::move(transform_ptr)); + } + LOG_DEBUG(log, "addParallelTransforms inserted processors, now processors is of size {}", processors.size()); /// Should not change streams number, so maybe not need max_parallel_streams update max_parallel_streams = std::max(max_parallel_streams, output_ports.size()); @@ -512,6 +523,15 @@ void Pipe::addTransform(ProcessorPtr transform, OutputPort * totals, OutputPort size_t next_output = 0; for (auto & input : inputs) { + LOG_DEBUG(log, "Pipe: is input connected {}", input.isConnected()); + LOG_DEBUG(log, "Pipe: is output connected {}", output_ports[next_output]->isConnected()); + if (output_ports[next_output]->isConnected()) + { + if (output_ports[next_output]->getHeader()) + LOG_DEBUG(log, "output header structure is: {}", output_ports[next_output]->getHeader().dumpStructure()); + else + LOG_DEBUG(log, "could not retrieve info about output"); + } connect(*output_ports[next_output], input); ++next_output; } diff --git a/src/QueryPipeline/Pipe.h b/src/QueryPipeline/Pipe.h index e655f807ec8..21e8fbfe039 100644 --- a/src/QueryPipeline/Pipe.h +++ b/src/QueryPipeline/Pipe.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB { @@ -137,6 +138,8 @@ private: /// It is needed for debug. See QueryPipelineProcessorsCollector. Processors * collected_processors = nullptr; + Poco::Logger * log = &Poco::Logger::get("Pipe"); + /// This methods are for QueryPipeline. It is allowed to complete graph only there. /// So, we may be sure that Pipe always has output port if not empty. bool isCompleted() const { return !empty() && output_ports.empty(); } diff --git a/src/QueryPipeline/QueryPipelineBuilder.h b/src/QueryPipeline/QueryPipelineBuilder.h index 9428bbc5d0a..d920f8a7e81 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.h +++ b/src/QueryPipeline/QueryPipelineBuilder.h @@ -67,7 +67,7 @@ public: /// Transform pipeline in general way. void transform(const Transformer & transformer); - /// Add transform and connects it to outputs[stream_index] stream + /// Add transforms and connect it to outputs streams void addParallelTransforms(Processors transform); /// Add TotalsHavingTransform. Resize pipeline to single input. Adds totals port. void addTotalsHavingTransform(ProcessorPtr transform); From 715d4c96986c821bd2309dedd48b57999434bc92 Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Sun, 16 May 2021 23:14:21 +0300 Subject: [PATCH 014/171] grouping sets cleanup --- src/Interpreters/InterpreterSelectQuery.cpp | 134 +----------------- src/Processors/QueryPlan/GroupingSetsStep.cpp | 46 ------ src/Processors/QueryPlan/GroupingSetsStep.h | 25 ---- .../Transforms/GroupingSetsTransform.cpp | 88 ------------ .../Transforms/GroupingSetsTransform.h | 37 ----- .../Transforms/TotalsHavingTransform.cpp | 10 -- .../Transforms/TotalsHavingTransform.h | 4 - 7 files changed, 2 insertions(+), 342 deletions(-) delete mode 100644 src/Processors/QueryPlan/GroupingSetsStep.cpp delete mode 100644 src/Processors/QueryPlan/GroupingSetsStep.h delete mode 100644 src/Processors/Transforms/GroupingSetsTransform.cpp delete mode 100644 src/Processors/Transforms/GroupingSetsTransform.h diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index cdf71f9894a..9ba084387fe 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -54,7 +54,6 @@ #include #include #include -#include #include #include #include @@ -1053,13 +1052,11 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

aggregates(); for (auto & descr : aggregates) if (descr.arguments.empty()) for (const auto & name : descr.argument_names) descr.arguments.push_back(header_before_aggregation.getPositionByName(name)); - LOG_DEBUG(log, "GroupingSets debug 2"); + const Settings & settings = context->getSettingsRef(); - LOG_DEBUG(log, "GroupingSets debug 3"); std::shared_ptr params_ptr; if (query.group_by_with_grouping_sets) { - LOG_DEBUG(log, "GroupingSets debug 4"); params_ptr = std::make_shared( header_before_aggregation, all_keys, @@ -2157,7 +2143,6 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac settings.compile_aggregate_expressions, settings.min_count_to_compile_aggregate_expression); } - LOG_DEBUG(log, "GroupingSets debug 5"); SortDescription group_by_sort_description; if (group_by_info && settings.optimize_aggregation_in_order && !query.group_by_with_grouping_sets) @@ -2171,7 +2156,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac : static_cast(settings.max_threads); bool storage_has_evenly_distributed_read = storage && storage->hasEvenlyDistributedRead(); - LOG_DEBUG(log, "GroupingSets debug 6"); + LOG_DEBUG(log, "GroupingSets step header structure: {}", query_plan.getCurrentDataStream().header.dumpStructure()); auto aggregating_step = std::make_unique( query_plan.getCurrentDataStream(), @@ -2185,9 +2170,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac std::move(group_by_info), std::move(group_by_sort_description)); LOG_DEBUG(log, "GroupingSets step header structure: {}", aggregating_step->getOutputStream().header.dumpStructure()); - LOG_DEBUG(log, "GroupingSets debug 7"); query_plan.addStep(std::move(aggregating_step)); - LOG_DEBUG(log, "GroupingSets debug 8"); } void InterpreterSelectQuery::executeMergeAggregated(QueryPlan & query_plan, bool overflow_row, bool final) @@ -2278,119 +2261,6 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPlan & query_plan, Modific query_plan.addStep(std::move(step)); } -/* -void InterpreterSelectQuery::executeGroupingSets(QueryPlan & query_plan, const ExpressionActionsPtr & expression, bool overflow_row, bool final, InputOrderInfoPtr group_by_info) -{ - - const auto & header_before_transform = query_plan.getCurrentDataStream().header; - - ColumnNumbers keys; - - for (const auto & key : query_analyzer->aggregationKeys()) - { - keys.push_back(header_before_transform.getPositionByName(key.name)); - LOG_DEBUG(log, "executeGroupingSets pushed back key with name {} and number {}", key.name, header_before_transform.getPositionByName(key.name)); - } - - const Settings & settings = context->getSettingsRef(); - - Aggregator::Params params(header_before_transform, keys, query_analyzer->aggregates(), - false, settings.max_rows_to_group_by, settings.group_by_overflow_mode, 0, 0, - settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, - context->getTemporaryVolume(), settings.max_threads, settings.min_free_disk_space_for_temporary_data, - settings.compile_aggregate_expressions, settings.min_count_to_compile_aggregate_expression); - - auto transform_params = std::make_shared(params, true); - - QueryPlanStepPtr step; - step = std::make_unique(query_plan.getCurrentDataStream(), std::move(transform_params)); - - query_plan.addStep(std::move(step)); - - - auto expression_before_aggregation = std::make_unique(query_plan.getCurrentDataStream(), expression); - expression_before_aggregation->setStepDescription("Before GROUP BY"); - query_plan.addStep(std::move(expression_before_aggregation)); - - const auto & header_before_aggregation = query_plan.getCurrentDataStream().header; - ColumnNumbers keys; - ColumnNumbersTwoDimension keys_vector; - for (const auto & aggregation_keys : query_analyzer->aggregationKeysList()) - { - keys.clear(); - for (const auto & key : aggregation_keys) - { - keys.push_back(header_before_aggregation.getPositionByName(key.name)); - LOG_DEBUG( - log, - "executeGroupingSets add key with name {} and number {}", - key.name, - header_before_aggregation.getPositionByName(key.name)); - } - keys_vector.push_back(keys); - } - - AggregateDescriptions aggregates = query_analyzer->aggregates(); - for (auto & descr : aggregates) - if (descr.arguments.empty()) - for (const auto & name : descr.argument_names) - { - descr.arguments.push_back(header_before_aggregation.getPositionByName(name)); - LOG_DEBUG( - log, - "executeGroupingSets add descr.atruments with name {} and number {}", - name, - header_before_aggregation.getPositionByName(name)); - } - - const Settings & settings = context->getSettingsRef(); - - Aggregator::Params params( - header_before_aggregation, - keys_vector, - aggregates, - overflow_row, - settings.max_rows_to_group_by, - settings.group_by_overflow_mode, - settings.group_by_two_level_threshold, - settings.group_by_two_level_threshold_bytes, - settings.max_bytes_before_external_group_by, - settings.empty_result_for_aggregation_by_empty_set, - context->getTemporaryVolume(), - settings.max_threads, - settings.min_free_disk_space_for_temporary_data); - - SortDescription group_by_sort_description; - - if (group_by_info && settings.optimize_aggregation_in_order) - group_by_sort_description = getSortDescriptionFromGroupBy(getSelectQuery()); - else - group_by_info = nullptr; - - auto merge_threads = max_streams; - auto temporary_data_merge_threads = settings.aggregation_memory_efficient_merge_threads - ? static_cast(settings.aggregation_memory_efficient_merge_threads) - : static_cast(settings.max_threads); - - bool storage_has_evenly_distributed_read = storage && storage->hasEvenlyDistributedRead(); - - auto aggregating_step = std::make_unique( - query_plan.getCurrentDataStream(), - params, - final, - settings.max_block_size, - settings.aggregation_in_order_max_block_bytes, - merge_threads, - temporary_data_merge_threads, - storage_has_evenly_distributed_read, - std::move(group_by_info), - std::move(group_by_sort_description)); - - query_plan.addStep(std::move(aggregating_step)); - -} -*/ - void InterpreterSelectQuery::executeExpression(QueryPlan & query_plan, const ActionsDAGPtr & expression, const std::string & description) { if (!expression) diff --git a/src/Processors/QueryPlan/GroupingSetsStep.cpp b/src/Processors/QueryPlan/GroupingSetsStep.cpp deleted file mode 100644 index d409e0e061a..00000000000 --- a/src/Processors/QueryPlan/GroupingSetsStep.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include -#include -#include - -namespace DB -{ - -static ITransformingStep::Traits getTraits() -{ - return ITransformingStep::Traits - { - { - .preserves_distinct_columns = false, - .returns_single_stream = true, - .preserves_number_of_streams = false, - .preserves_sorting = false, - }, - { - .preserves_number_of_rows = false, - } - }; -} - -GroupingSetsStep::GroupingSetsStep(const DataStream & input_stream_, AggregatingTransformParamsPtr params_) - : ITransformingStep(input_stream_, params_->getHeader(), getTraits()) - , params(std::move(params_)) -{ - /// Aggregation keys are distinct - for (auto key : params->params.keys) - output_stream->distinct_columns.insert(params->params.src_header.getByPosition(key).name); -} - -void GroupingSetsStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & /*settings*/) -{ - pipeline.resize(1); - - pipeline.addSimpleTransform([&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr - { - if (stream_type == QueryPipelineBuilder::StreamType::Totals) - return nullptr; - - return std::make_shared(header, std::move(params)); - }); -} - -} diff --git a/src/Processors/QueryPlan/GroupingSetsStep.h b/src/Processors/QueryPlan/GroupingSetsStep.h deleted file mode 100644 index f27d7f1280e..00000000000 --- a/src/Processors/QueryPlan/GroupingSetsStep.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once -#include -#include - -namespace DB -{ - -struct AggregatingTransformParams; -using AggregatingTransformParamsPtr = std::shared_ptr; - -/// WITH CUBE. See CubeTransform. -class GroupingSetsStep : public ITransformingStep -{ -public: - GroupingSetsStep(const DataStream & input_stream_, AggregatingTransformParamsPtr params_); - - String getName() const override { return "Grouping Sets"; } - - void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) override; - -private: - AggregatingTransformParamsPtr params; -}; - -} diff --git a/src/Processors/Transforms/GroupingSetsTransform.cpp b/src/Processors/Transforms/GroupingSetsTransform.cpp deleted file mode 100644 index ccbafac8e3e..00000000000 --- a/src/Processors/Transforms/GroupingSetsTransform.cpp +++ /dev/null @@ -1,88 +0,0 @@ -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - -GroupingSetsTransform::GroupingSetsTransform(Block header, AggregatingTransformParamsPtr params_) - : IAccumulatingTransform(std::move(header), params_->getHeader()) - , params(std::move(params_)) - , keys(params->params.keys) -{ -// if (keys.size() >= 8 * sizeof(mask)) -// throw Exception("Too many keys are used for CubeTransform.", ErrorCodes::LOGICAL_ERROR); -} - -Chunk GroupingSetsTransform::merge(Chunks && chunks, bool final) -{ - LOG_DEBUG(log, "merge {} blocks", chunks.size()); - BlocksList rollup_blocks; - for (auto & chunk : chunks) - rollup_blocks.emplace_back(getInputPort().getHeader().cloneWithColumns(chunk.detachColumns())); - - auto rollup_block = params->aggregator.mergeBlocks(rollup_blocks, final); - auto num_rows = rollup_block.rows(); - return Chunk(rollup_block.getColumns(), num_rows); -} - -void GroupingSetsTransform::consume(Chunk chunk) -{ - consumed_chunks.emplace_back(std::move(chunk)); - LOG_DEBUG(log, "consumed block, now consumed_chunks size is {}", consumed_chunks.size()); -} - -Chunk GroupingSetsTransform::generate() -{ - LOG_DEBUG(log, "generate start, mask = {}", mask); - if (!consumed_chunks.empty()) - { - LOG_DEBUG(log, "consumed_chunks not empty, size is {}", consumed_chunks.size()); - if (consumed_chunks.size() > 1) - grouping_sets_chunk = merge(std::move(consumed_chunks), false); - else - grouping_sets_chunk = std::move(consumed_chunks.front()); - - consumed_chunks.clear(); - - auto num_rows = grouping_sets_chunk.getNumRows(); - mask = (UInt64(1) << keys.size()); - LOG_DEBUG(log, "changed mask, mask = {}", mask); - - current_columns = grouping_sets_chunk.getColumns(); - current_zero_columns.clear(); - current_zero_columns.reserve(keys.size()); - - for (auto key : keys) - current_zero_columns.emplace_back(current_columns[key]->cloneEmpty()->cloneResized(num_rows)); - } - - // auto gen_chunk = std::move(cube_chunk); - LOG_DEBUG(log, "before if mask"); - if (mask > 1) - { - LOG_DEBUG(log, "in if mask > 1"); - mask = mask >> 1; - - auto columns = current_columns; - auto size = keys.size(); - for (size_t i = 0; i < size; ++i) - /// Reverse bit order to support previous behaviour. - if ((mask & (UInt64(1) << (size - i - 1))) == 0) - columns[keys[i]] = current_zero_columns[i]; - - Chunks chunks; - chunks.emplace_back(std::move(columns), current_columns.front()->size()); - grouping_sets_chunk = merge(std::move(chunks), false); - } - LOG_DEBUG(log, "before gen_chunk"); - auto gen_chunk = std::move(grouping_sets_chunk); - - finalizeChunk(gen_chunk); - return gen_chunk; -} - -} diff --git a/src/Processors/Transforms/GroupingSetsTransform.h b/src/Processors/Transforms/GroupingSetsTransform.h deleted file mode 100644 index 10108d3c1a7..00000000000 --- a/src/Processors/Transforms/GroupingSetsTransform.h +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once -#include -#include - - -namespace DB -{ - -/// Takes blocks after grouping, with non-finalized aggregate functions. -/// Calculates all subsets of columns and aggregates over them. -class GroupingSetsTransform : public IAccumulatingTransform -{ -public: - GroupingSetsTransform(Block header, AggregatingTransformParamsPtr params); - String getName() const override { return "GroupingSetsTransform"; } - -protected: - void consume(Chunk chunk) override; - Chunk generate() override; - -private: - AggregatingTransformParamsPtr params; - ColumnNumbers keys; - - Chunks consumed_chunks; - Chunk grouping_sets_chunk; - Columns current_columns; - Columns current_zero_columns; - - UInt64 mask = 0; - - Poco::Logger * log = &Poco::Logger::get("GroupingSetsTransform"); - - Chunk merge(Chunks && chunks, bool final); -}; - -} diff --git a/src/Processors/Transforms/TotalsHavingTransform.cpp b/src/Processors/Transforms/TotalsHavingTransform.cpp index a636ba70ef9..0b7797da24f 100644 --- a/src/Processors/Transforms/TotalsHavingTransform.cpp +++ b/src/Processors/Transforms/TotalsHavingTransform.cpp @@ -119,7 +119,6 @@ TotalsHavingTransform::TotalsHavingTransform( IProcessor::Status TotalsHavingTransform::prepare() { - LOG_DEBUG(log, "TotalsHavingTransform::prepare()"); if (!finished_transform) { auto status = ISimpleTransform::prepare(); @@ -144,24 +143,20 @@ IProcessor::Status TotalsHavingTransform::prepare() totals_output.push(std::move(totals)); totals_output.finish(); - LOG_DEBUG(log, "exit TotalsHavingTransform::prepare()"); return Status::Finished; } void TotalsHavingTransform::work() { - LOG_DEBUG(log, "TotalsHavingTransform::work()"); if (finished_transform) prepareTotals(); else ISimpleTransform::work(); - LOG_DEBUG(log, "exit TotalsHavingTransform::work()"); } void TotalsHavingTransform::transform(Chunk & chunk) { /// Block with values not included in `max_rows_to_group_by`. We'll postpone it. - LOG_DEBUG(log, "TotalsHavingTransform::transform()"); if (overflow_row) { const auto & info = chunk.getChunkInfo(); @@ -254,12 +249,10 @@ void TotalsHavingTransform::transform(Chunk & chunk) } passed_keys += chunk.getNumRows(); - LOG_DEBUG(log, "exit TotalsHavingTransform::transform()"); } void TotalsHavingTransform::addToTotals(const Chunk & chunk, const IColumn::Filter * filter) { - LOG_DEBUG(log, "TotalsHavingTransform::addToTotals()"); auto num_columns = chunk.getNumColumns(); for (size_t col = 0; col < num_columns; ++col) { @@ -291,12 +284,10 @@ void TotalsHavingTransform::addToTotals(const Chunk & chunk, const IColumn::Filt } } } - LOG_DEBUG(log, "exit TotalsHavingTransform::addToTotals()"); } void TotalsHavingTransform::prepareTotals() { - LOG_DEBUG(log, "TotalsHavingTransform::prepareTotals()"); /// If totals_mode == AFTER_HAVING_AUTO, you need to decide whether to add aggregates to TOTALS for strings, /// not passed max_rows_to_group_by. if (overflow_aggregates) @@ -321,7 +312,6 @@ void TotalsHavingTransform::prepareTotals() /// Note: after expression totals may have several rows if `arrayJoin` was used in expression. totals = Chunk(block.getColumns(), num_rows); } - LOG_DEBUG(log, "exit TotalsHavingTransform::prepareTotals()"); } } diff --git a/src/Processors/Transforms/TotalsHavingTransform.h b/src/Processors/Transforms/TotalsHavingTransform.h index 59fa58edfbc..03635054c65 100644 --- a/src/Processors/Transforms/TotalsHavingTransform.h +++ b/src/Processors/Transforms/TotalsHavingTransform.h @@ -2,8 +2,6 @@ #include #include -#include // to be removed -// #include namespace DB { @@ -76,8 +74,6 @@ private: /// Here, total values are accumulated. After the work is finished, they will be placed in totals. MutableColumns current_totals; - - Poco::Logger * log = &Poco::Logger::get("TotalsHavingTransform"); }; void finalizeChunk(Chunk & chunk); From daea6fcac3c04b0e525c1c89d39ea3f507b0d776 Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Mon, 17 May 2021 15:03:07 +0300 Subject: [PATCH 015/171] grouping sets: fix 'Port already connected' error --- src/Processors/ya.make | 2 +- src/QueryPipeline/Pipe.cpp | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/Processors/ya.make b/src/Processors/ya.make index a1386acb6d0..a6f8e274e9e 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -102,7 +102,7 @@ SRCS( QueryPlan/FillingStep.cpp QueryPlan/FilterStep.cpp QueryPlan/FinishSortingStep.cpp - QueryPlan/GroupingSetsStep.cpp + QueryPlan/IQueryPlanStep.cpp QueryPlan/ISourceStep.cpp QueryPlan/ITransformingStep.cpp diff --git a/src/QueryPipeline/Pipe.cpp b/src/QueryPipeline/Pipe.cpp index d9917b8636d..25cdf2c8fa9 100644 --- a/src/QueryPipeline/Pipe.cpp +++ b/src/QueryPipeline/Pipe.cpp @@ -439,21 +439,21 @@ void Pipe::addParallelTransforms(Processors transforms) if (output_ports.empty()) throw Exception("Cannot add parallel transforms to empty Pipe.", ErrorCodes::LOGICAL_ERROR); - std::vector inputs; - std::vector outputs; + std::vector inputs; + std::vector outputs; for (const auto & transform : transforms) { - auto current_transform_inputs = transform->getInputs(); + auto & current_transform_inputs = transform->getInputs(); if (current_transform_inputs.size() != 1) throw Exception("Each parallel transform should have one input port", ErrorCodes::LOGICAL_ERROR); - inputs.push_back(current_transform_inputs.front()); + inputs.push_back(&(current_transform_inputs.front())); - auto current_transform_outputs = transform->getOutputs(); + auto & current_transform_outputs = transform->getOutputs(); if (current_transform_outputs.size() != 1) throw Exception("Each parallel transform should have one output port", ErrorCodes::LOGICAL_ERROR); - outputs.push_back(current_transform_outputs.front()); + outputs.push_back(&(current_transform_outputs.front())); LOG_DEBUG(log, "addParallelTransforms, added inputs and outputs for processor {}", transform->getName()); } @@ -463,9 +463,11 @@ void Pipe::addParallelTransforms(Processors transforms) "but " + std::to_string(output_ports.size()) + " expected", ErrorCodes::LOGICAL_ERROR); size_t next_output = 0; - for (auto & input : inputs) + for (auto * input : inputs) { - connect(*output_ports[next_output], input); + LOG_DEBUG(log, "is input connected {}", input->isConnected()); + LOG_DEBUG(log, "is output connected {}", output_ports[next_output]->isConnected()); + connect(*output_ports[next_output], *input); ++next_output; LOG_DEBUG(log, "addParallelTransforms connect current output to new input {}", next_output); } @@ -473,11 +475,11 @@ void Pipe::addParallelTransforms(Processors transforms) output_ports.clear(); output_ports.reserve(outputs.size()); - for (auto & output : outputs) + for (auto * output : outputs) { LOG_DEBUG(log, "addParallelTransforms change outputs to new output"); - LOG_DEBUG(log, "addParallelTransforms is output connected: {}", output.isConnected()); - output_ports.emplace_back(&output); + LOG_DEBUG(log, "addParallelTransforms is output connected: {}", output->isConnected()); + output_ports.emplace_back(std::move(output)); } /// do not check output formats because they are different in case of parallel aggregations From e22cbb7cd82e073bf8046feb2ad130d0e99bd2da Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Fri, 21 May 2021 02:35:49 +0300 Subject: [PATCH 016/171] grouping sets dev: fix errors, something works --- src/Interpreters/ExpressionAnalyzer.cpp | 3 ++- src/Interpreters/InterpreterSelectQuery.cpp | 6 ++++-- src/Processors/QueryPlan/AggregatingStep.h | 2 ++ src/Processors/ya.make | 2 -- src/QueryPipeline/Pipe.cpp | 20 +------------------- 5 files changed, 9 insertions(+), 24 deletions(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index fcbe796032b..16e2258d4a8 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1156,7 +1156,8 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain ASTs asts = select_query->groupBy()->children; if (select_query->group_by_with_grouping_sets) { - for (const auto & inner_asts : asts) { + for (const auto & inner_asts : asts) + { for (const auto & ast : inner_asts->children) { step.addRequiredOutput(ast->getColumnName()); diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 9ba084387fe..5c3c69f535a 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -586,6 +586,8 @@ void InterpreterSelectQuery::buildQueryPlan(QueryPlan & query_plan) /// /// But if it's a projection query, plan header does not match result_header. /// TODO: add special stage for InterpreterSelectQuery? + LOG_DEBUG(log, "query_plan header: {}", query_plan.getCurrentDataStream().header.dumpStructure()); + LOG_DEBUG(log, "result header: {}", result_header.dumpStructure()); if (!options.is_projection_query && !blocksHaveEqualStructure(query_plan.getCurrentDataStream().header, result_header)) { auto convert_actions_dag = ActionsDAG::makeConvertingActions( @@ -2157,7 +2159,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac bool storage_has_evenly_distributed_read = storage && storage->hasEvenlyDistributedRead(); - LOG_DEBUG(log, "GroupingSets step header structure: {}", query_plan.getCurrentDataStream().header.dumpStructure()); + LOG_DEBUG(log, "GroupingSets step header before step structure: {}", query_plan.getCurrentDataStream().header.dumpStructure()); auto aggregating_step = std::make_unique( query_plan.getCurrentDataStream(), *params_ptr, @@ -2169,7 +2171,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac storage_has_evenly_distributed_read, std::move(group_by_info), std::move(group_by_sort_description)); - LOG_DEBUG(log, "GroupingSets step header structure: {}", aggregating_step->getOutputStream().header.dumpStructure()); + LOG_DEBUG(log, "GroupingSets step header after step structure: {}", aggregating_step->getOutputStream().header.dumpStructure()); query_plan.addStep(std::move(aggregating_step)); } diff --git a/src/Processors/QueryPlan/AggregatingStep.h b/src/Processors/QueryPlan/AggregatingStep.h index 154ff0abc0a..4c476a2841f 100644 --- a/src/Processors/QueryPlan/AggregatingStep.h +++ b/src/Processors/QueryPlan/AggregatingStep.h @@ -56,6 +56,8 @@ private: Processors aggregating; + Poco::Logger * log = &Poco::Logger::get("AggregatingStep"); + }; } diff --git a/src/Processors/ya.make b/src/Processors/ya.make index a6f8e274e9e..b2f8b9ba7c2 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -102,7 +102,6 @@ SRCS( QueryPlan/FillingStep.cpp QueryPlan/FilterStep.cpp QueryPlan/FinishSortingStep.cpp - QueryPlan/IQueryPlanStep.cpp QueryPlan/ISourceStep.cpp QueryPlan/ITransformingStep.cpp @@ -141,7 +140,6 @@ SRCS( Transforms/FillingTransform.cpp Transforms/FilterTransform.cpp Transforms/FinishSortingTransform.cpp - Transforms/GroupingSetsTransform.cpp Transforms/JoiningTransform.cpp Transforms/LimitByTransform.cpp Transforms/LimitsCheckingTransform.cpp diff --git a/src/QueryPipeline/Pipe.cpp b/src/QueryPipeline/Pipe.cpp index 25cdf2c8fa9..184cad18c17 100644 --- a/src/QueryPipeline/Pipe.cpp +++ b/src/QueryPipeline/Pipe.cpp @@ -455,6 +455,7 @@ void Pipe::addParallelTransforms(Processors transforms) outputs.push_back(&(current_transform_outputs.front())); LOG_DEBUG(log, "addParallelTransforms, added inputs and outputs for processor {}", transform->getName()); + LOG_DEBUG(log, "output structure: {}", transform->getOutputs().front().getHeader().dumpStructure()); } if (inputs.size() != output_ports.size()) @@ -465,22 +466,15 @@ void Pipe::addParallelTransforms(Processors transforms) size_t next_output = 0; for (auto * input : inputs) { - LOG_DEBUG(log, "is input connected {}", input->isConnected()); - LOG_DEBUG(log, "is output connected {}", output_ports[next_output]->isConnected()); connect(*output_ports[next_output], *input); ++next_output; - LOG_DEBUG(log, "addParallelTransforms connect current output to new input {}", next_output); } output_ports.clear(); output_ports.reserve(outputs.size()); for (auto * output : outputs) - { - LOG_DEBUG(log, "addParallelTransforms change outputs to new output"); - LOG_DEBUG(log, "addParallelTransforms is output connected: {}", output->isConnected()); output_ports.emplace_back(std::move(output)); - } /// do not check output formats because they are different in case of parallel aggregations LOG_DEBUG(log, "addParallelTransforms do not check format"); @@ -489,10 +483,7 @@ void Pipe::addParallelTransforms(Processors transforms) collected_processors->insert(collected_processors->end(), transforms.begin(), transforms.end()); for (auto & transform_ptr : transforms) - { processors.emplace_back(std::move(transform_ptr)); - } - LOG_DEBUG(log, "addParallelTransforms inserted processors, now processors is of size {}", processors.size()); /// Should not change streams number, so maybe not need max_parallel_streams update max_parallel_streams = std::max(max_parallel_streams, output_ports.size()); @@ -525,15 +516,6 @@ void Pipe::addTransform(ProcessorPtr transform, OutputPort * totals, OutputPort size_t next_output = 0; for (auto & input : inputs) { - LOG_DEBUG(log, "Pipe: is input connected {}", input.isConnected()); - LOG_DEBUG(log, "Pipe: is output connected {}", output_ports[next_output]->isConnected()); - if (output_ports[next_output]->isConnected()) - { - if (output_ports[next_output]->getHeader()) - LOG_DEBUG(log, "output header structure is: {}", output_ports[next_output]->getHeader().dumpStructure()); - else - LOG_DEBUG(log, "could not retrieve info about output"); - } connect(*output_ports[next_output], input); ++next_output; } From c7c2093bd9b9e4ff47897a0b7195d5e71b319d5d Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Fri, 21 May 2021 17:56:54 +0300 Subject: [PATCH 017/171] grouping sets: make simple aggregation with grouping sets to work --- src/Interpreters/ExpressionAnalyzer.cpp | 8 ++------ src/Interpreters/InterpreterSelectQuery.cpp | 20 +++++++++++--------- src/Interpreters/TreeOptimizer.cpp | 10 +++++----- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 16e2258d4a8..cbbd8885fd1 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -381,16 +381,12 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) if (!unique_keys.count(key.name)) { unique_keys.insert(key.name); + aggregation_keys.push_back(key); if (select_query->group_by_with_grouping_sets) { aggregation_keys_list.push_back({key}); - aggregation_keys.push_back(key); LOG_DEBUG(poco_log, "pushed grouping set of 1 column: " + key.name); } - else - { - aggregation_keys.push_back(key); - } /// Key is no longer needed, therefore we can save a little by moving it. aggregated_columns.push_back(std::move(key)); @@ -1662,7 +1658,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( /// TODO correct conditions optimize_aggregation_in_order = context->getSettingsRef().optimize_aggregation_in_order - && storage && query.groupBy() && !query.group_by_with_grouping_sets; + && storage && query.groupBy(); query_analyzer.appendGroupBy(chain, only_types || !first_stage, optimize_aggregation_in_order, group_by_elements_actions); query_analyzer.appendAggregateFunctionsArguments(chain, only_types || !first_stage); diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 5c3c69f535a..bea2c05d986 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -969,8 +969,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

QueryProcessingStage::WithMergeableState && - !query.group_by_with_totals && !query.group_by_with_rollup && !query.group_by_with_cube && - !query.group_by_with_grouping_sets; + !query.group_by_with_totals && !query.group_by_with_rollup && !query.group_by_with_cube; if (query_info.projection && query_info.projection->desc->type == ProjectionDescription::Type::Aggregate) { @@ -2074,14 +2073,14 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac all_keys.push_back(header_before_aggregation.getPositionByName(key.name)); LOG_DEBUG( log, - "GroupingSets add key with name {} and number {}", + "execute aggregation with grouping sets add key with name {} and number {}", key.name, header_before_aggregation.getPositionByName(key.name)); } keys_vector.push_back(keys); LOG_DEBUG( log, - "GroupingSets add keys set of size {}", + "execute aggregation with grouping sets add keys set of size {}", keys.size()); } } @@ -2090,7 +2089,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac for (const auto & key : query_analyzer->aggregationKeys()) { keys.push_back(header_before_aggregation.getPositionByName(key.name)); - LOG_DEBUG(log, "executeAggregation pushed back key with name {} and number {}", key.name, header_before_aggregation.getPositionByName(key.name)); + LOG_DEBUG(log, "execute aggregation without grouping sets pushed back key with name {} and number {}", key.name, header_before_aggregation.getPositionByName(key.name)); } } @@ -2147,10 +2146,13 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac } SortDescription group_by_sort_description; - if (group_by_info && settings.optimize_aggregation_in_order && !query.group_by_with_grouping_sets) + if (group_by_info && settings.optimize_aggregation_in_order && !query.group_by_with_grouping_sets) { group_by_sort_description = getSortDescriptionFromGroupBy(query); - else + LOG_DEBUG(log, "execute aggregation without grouping sets got group_by_sort_description"); + } else { group_by_info = nullptr; + LOG_DEBUG(log, "execute aggregation didn't get group_by_sort_description"); + } auto merge_threads = max_streams; auto temporary_data_merge_threads = settings.aggregation_memory_efficient_merge_threads @@ -2159,7 +2161,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac bool storage_has_evenly_distributed_read = storage && storage->hasEvenlyDistributedRead(); - LOG_DEBUG(log, "GroupingSets step header before step structure: {}", query_plan.getCurrentDataStream().header.dumpStructure()); + LOG_DEBUG(log, "execute aggregation header structure before step: {}", query_plan.getCurrentDataStream().header.dumpStructure()); auto aggregating_step = std::make_unique( query_plan.getCurrentDataStream(), *params_ptr, @@ -2171,7 +2173,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac storage_has_evenly_distributed_read, std::move(group_by_info), std::move(group_by_sort_description)); - LOG_DEBUG(log, "GroupingSets step header after step structure: {}", aggregating_step->getOutputStream().header.dumpStructure()); + LOG_DEBUG(log, "execute aggregation header structure after step: {}", aggregating_step->getOutputStream().header.dumpStructure()); query_plan.addStep(std::move(aggregating_step)); } diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index 1d6ed76b9c4..367ccc99547 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -84,8 +84,8 @@ void optimizeGroupBy(ASTSelectQuery * select_query, ContextPtr context) { const FunctionFactory & function_factory = FunctionFactory::instance(); - if (select_query->group_by_with_grouping_sets) - return; +// if (select_query->group_by_with_grouping_sets) +// return; if (!select_query->groupBy()) return; @@ -216,7 +216,7 @@ GroupByKeysInfo getGroupByKeysInfo(const ASTs & group_by_keys) ///eliminate functions of other GROUP BY keys void optimizeGroupByFunctionKeys(ASTSelectQuery * select_query) { - if (!select_query->groupBy() || select_query->group_by_with_grouping_sets) + if (!select_query->groupBy()) return; auto group_by = select_query->groupBy(); @@ -246,7 +246,7 @@ void optimizeGroupByFunctionKeys(ASTSelectQuery * select_query) /// Eliminates min/max/any-aggregators of functions of GROUP BY keys void optimizeAggregateFunctionsOfGroupByKeys(ASTSelectQuery * select_query, ASTPtr & node) { - if (!select_query->groupBy() || select_query->group_by_with_grouping_sets) + if (!select_query->groupBy()) return; const auto & group_by_keys = select_query->groupBy()->children; @@ -416,7 +416,7 @@ void optimizeMonotonousFunctionsInOrderBy(ASTSelectQuery * select_query, Context const TreeRewriterResult & result) { auto order_by = select_query->orderBy(); - if (!order_by || select_query->group_by_with_grouping_sets) + if (!order_by) return; /// Do not apply optimization for Distributed and Merge storages, From 597823369588968c41a3701298863e3dbd6cba11 Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Fri, 21 May 2021 23:47:57 +0300 Subject: [PATCH 018/171] grouping sets: fix --- src/Interpreters/InterpreterSelectQuery.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index bea2c05d986..7f39be5d1d0 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -586,8 +586,6 @@ void InterpreterSelectQuery::buildQueryPlan(QueryPlan & query_plan) /// /// But if it's a projection query, plan header does not match result_header. /// TODO: add special stage for InterpreterSelectQuery? - LOG_DEBUG(log, "query_plan header: {}", query_plan.getCurrentDataStream().header.dumpStructure()); - LOG_DEBUG(log, "result header: {}", result_header.dumpStructure()); if (!options.is_projection_query && !blocksHaveEqualStructure(query_plan.getCurrentDataStream().header, result_header)) { auto convert_actions_dag = ActionsDAG::makeConvertingActions( From 233471de5c865ae8f893558457447e3309218582 Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Thu, 27 May 2021 22:50:35 +0300 Subject: [PATCH 019/171] grouping sets: add tests, fix bug --- src/Interpreters/InterpreterSelectQuery.cpp | 3 +++ .../01883_with_grouping_sets.reference | 8 +++++++ .../0_stateless/01883_with_grouping_sets.sql | 22 +++++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 tests/queries/0_stateless/01883_with_grouping_sets.reference create mode 100644 tests/queries/0_stateless/01883_with_grouping_sets.sql diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 7f39be5d1d0..6c2d0246656 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -969,6 +969,9 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

QueryProcessingStage::WithMergeableState && !query.group_by_with_totals && !query.group_by_with_rollup && !query.group_by_with_cube; + if (query.group_by_with_grouping_sets && query.group_by_with_totals) + throw Exception("WITH TOTALS and GROUPING SETS are not supported together", ErrorCodes::NOT_IMPLEMENTED); + if (query_info.projection && query_info.projection->desc->type == ProjectionDescription::Type::Aggregate) { query_info.projection->aggregate_overflow_row = aggregate_overflow_row; diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.reference b/tests/queries/0_stateless/01883_with_grouping_sets.reference new file mode 100644 index 00000000000..60cc348f671 --- /dev/null +++ b/tests/queries/0_stateless/01883_with_grouping_sets.reference @@ -0,0 +1,8 @@ + 1 40 4 + 2 80 4 +a 0 70 4 +b 0 50 4 + 1 40 4 + 2 80 4 +a 0 70 4 +b 0 50 4 diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.sql b/tests/queries/0_stateless/01883_with_grouping_sets.sql new file mode 100644 index 00000000000..71f03209cce --- /dev/null +++ b/tests/queries/0_stateless/01883_with_grouping_sets.sql @@ -0,0 +1,22 @@ +DROP TABLE IF EXISTS grouping_sets; +CREATE TABLE grouping_sets(a String, b Int32, s Int32) ENGINE = Memory; + +INSERT INTO grouping_sets VALUES ('a', 1, 10), ('a', 1, 15), ('a', 2, 20); +INSERT INTO grouping_sets VALUES ('a', 2, 25), ('b', 1, 10), ('b', 1, 5); +INSERT INTO grouping_sets VALUES ('b', 2, 20), ('b', 2, 15); + +SELECT a, b, sum(s), count() from grouping_sets GROUP BY GROUPING SETS(a, b) ORDER BY a, b; + +-- doesn't work now +-- SELECT a, b, sum(s), count() from grouping_sets GROUP BY GROUPING SETS(a, b) WITH TOTALS ORDER BY a, b; + +SELECT a, b, sum(s), count() from grouping_sets GROUP BY a, b WITH GROUPING SETS ORDER BY a, b; + +-- doesn't work now +-- SELECT a, b, sum(s), count() from grouping_sets GROUP BY a, b WITH GROUPING SETS WITH TOTALS ORDER BY a, b; + +-- not sure that always works +-- SET group_by_two_level_threshold = 1; +-- SELECT a, b, sum(s), count() from grouping_sets GROUP BY a, b WITH GROUPING SETS ORDER BY a, b; + +DROP TABLE grouping_sets; From fa323500f7f4d4f33b7c27803d596978209bc06f Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Sun, 20 Jun 2021 12:36:00 +0300 Subject: [PATCH 020/171] grouping-sets: rearrange result columns so that resize is possible after grouping sets --- src/Interpreters/InterpreterSelectQuery.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 6c2d0246656..e17e5987b51 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -969,8 +969,8 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

QueryProcessingStage::WithMergeableState && !query.group_by_with_totals && !query.group_by_with_rollup && !query.group_by_with_cube; - if (query.group_by_with_grouping_sets && query.group_by_with_totals) - throw Exception("WITH TOTALS and GROUPING SETS are not supported together", ErrorCodes::NOT_IMPLEMENTED); +// if (query.group_by_with_grouping_sets && query.group_by_with_totals) +// throw Exception("WITH TOTALS and GROUPING SETS are not supported together", ErrorCodes::NOT_IMPLEMENTED); if (query_info.projection && query_info.projection->desc->type == ProjectionDescription::Type::Aggregate) { From a2d2868996412546f93a254679acf6b6c9326cbb Mon Sep 17 00:00:00 2001 From: MaxTheHuman Date: Sun, 20 Jun 2021 13:04:18 +0300 Subject: [PATCH 021/171] make grouping sets work with total --- .../0_stateless/01883_with_grouping_sets.reference | 12 ++++++++++++ .../queries/0_stateless/01883_with_grouping_sets.sql | 6 ++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.reference b/tests/queries/0_stateless/01883_with_grouping_sets.reference index 60cc348f671..fd128444ebc 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.reference +++ b/tests/queries/0_stateless/01883_with_grouping_sets.reference @@ -6,3 +6,15 @@ b 0 50 4 2 80 4 a 0 70 4 b 0 50 4 + + 0 240 16 + 1 40 4 + 2 80 4 +a 0 70 4 +b 0 50 4 + 1 40 4 + 2 80 4 +a 0 70 4 +b 0 50 4 + + 0 240 16 diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.sql b/tests/queries/0_stateless/01883_with_grouping_sets.sql index 71f03209cce..6270b772328 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.sql +++ b/tests/queries/0_stateless/01883_with_grouping_sets.sql @@ -7,13 +7,11 @@ INSERT INTO grouping_sets VALUES ('b', 2, 20), ('b', 2, 15); SELECT a, b, sum(s), count() from grouping_sets GROUP BY GROUPING SETS(a, b) ORDER BY a, b; --- doesn't work now --- SELECT a, b, sum(s), count() from grouping_sets GROUP BY GROUPING SETS(a, b) WITH TOTALS ORDER BY a, b; +SELECT a, b, sum(s), count() from grouping_sets GROUP BY GROUPING SETS(a, b) WITH TOTALS ORDER BY a, b; SELECT a, b, sum(s), count() from grouping_sets GROUP BY a, b WITH GROUPING SETS ORDER BY a, b; --- doesn't work now --- SELECT a, b, sum(s), count() from grouping_sets GROUP BY a, b WITH GROUPING SETS WITH TOTALS ORDER BY a, b; +SELECT a, b, sum(s), count() from grouping_sets GROUP BY a, b WITH GROUPING SETS WITH TOTALS ORDER BY a, b; -- not sure that always works -- SET group_by_two_level_threshold = 1; From 912338629ec20ad981b7d1b02c640069e5502612 Mon Sep 17 00:00:00 2001 From: fanzhou Date: Wed, 28 Jul 2021 15:02:10 +0800 Subject: [PATCH 022/171] grouping sets support multiple group --- src/Interpreters/ActionsDAG.h | 2 +- src/Interpreters/ExpressionAnalyzer.cpp | 127 ++++++++++++++------ src/Interpreters/InterpreterSelectQuery.cpp | 14 ++- src/Interpreters/TreeOptimizer.cpp | 18 ++- src/Parsers/ExpressionListParsers.cpp | 47 +++++++- src/Parsers/ExpressionListParsers.h | 6 + 6 files changed, 164 insertions(+), 50 deletions(-) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 9a5ad01a252..cfb85cd5369 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -183,7 +183,7 @@ public: ActionsDAGPtr clone() const; /// Execute actions for header. Input block must have empty columns. - /// Result should be equal to the execution of ExpressionActions build form this DAG. + /// Result should be equal to the execution of ExpressionActions built from this DAG. /// Actions are not changed, no expressions are compiled. /// /// In addition, check that result constants are constants according to DAG. diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index cbbd8885fd1..cd2e00c832b 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -40,6 +40,7 @@ #include #include +#include "Core/NamesAndTypes.h" #include @@ -339,57 +340,103 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) if (getContext()->getSettingsRef().enable_positional_arguments) replaceForPositionalArguments(group_asts[i], select_query, ASTSelectQuery::Expression::GROUP_BY); - if (select_query->group_by_with_grouping_sets) - { - LOG_DEBUG(poco_log, "analyzeAggregation: detect group by with grouping sets"); - /// TODO - } - getRootActionsNoMakeSet(group_asts[i], true, temp_actions, false); - const auto & column_name = group_asts[i]->getColumnName(); - - const auto * node = temp_actions->tryFindInIndex(column_name); - if (!node) - throw Exception("Unknown identifier (in GROUP BY): " + column_name, ErrorCodes::UNKNOWN_IDENTIFIER); - - /// Only removes constant keys if it's an initiator or distributed_group_by_no_merge is enabled. - if (getContext()->getClientInfo().distributed_depth == 0 || settings.distributed_group_by_no_merge > 0) + if (select_query->group_by_with_grouping_sets) { - /// Constant expressions have non-null column pointer at this stage. - if (node->column && isColumnConst(*node->column)) + ASTs group_elements_ast; + if (group_asts[i]->as()) { - select_query->group_by_with_constant_keys = true; + group_elements_ast = group_asts[i]->as()->children; + } + else + { + const auto id_ast = group_asts[i]->as(); + group_elements_ast.push_back(std::make_shared(id_ast)); + } - /// But don't remove last key column if no aggregate functions, otherwise aggregation will not work. - if (!aggregate_descriptions.empty() || size > 1) + NamesAndTypesList grouping_set_list; + + for (ssize_t j = 0; j < ssize_t(group_elements_ast.size()); ++j) + { + const auto & column_name = group_elements_ast[j]->getColumnName(); + const auto * node = temp_actions->tryFindInIndex(column_name); + if (!node) + throw Exception("Unknown identifier (in GROUP BY): " + column_name, ErrorCodes::UNKNOWN_IDENTIFIER); + + /// Constant expressions have non-null column pointer at this stage. + if (node->column && isColumnConst(*node->column)) { - if (i + 1 < static_cast(size)) - group_asts[i] = std::move(group_asts.back()); + /// But don't remove last key column if no aggregate functions, otherwise aggregation will not work. + if (!aggregate_descriptions.empty() || size > 1) + { + if (j + 1 < static_cast(size)) + group_asts[i] = std::move(group_asts.back()); - group_asts.pop_back(); + group_asts.pop_back(); - --i; - continue; + --j; + continue; + } + } + + NameAndTypePair key{column_name, node->result_type}; + + grouping_set_list.push_back(key); + + /// Aggregation keys are unique. + if (!unique_keys.count(key.name)) + { + unique_keys.insert(key.name); + aggregation_keys.push_back(key); + + /// Key is no longer needed, therefore we can save a little by moving it. + aggregated_columns.push_back(std::move(key)); } } + + aggregation_keys_list.push_back(grouping_set_list); } - - NameAndTypePair key{column_name, node->result_type}; - - /// Aggregation keys are uniqued. - if (!unique_keys.count(key.name)) + else { - unique_keys.insert(key.name); - aggregation_keys.push_back(key); - if (select_query->group_by_with_grouping_sets) + const auto & column_name = group_asts[i]->getColumnName(); + const auto * node = temp_actions->tryFindInIndex(column_name); + if (!node) + throw Exception("Unknown identifier (in GROUP BY): " + column_name, ErrorCodes::UNKNOWN_IDENTIFIER); + + /// Only removes constant keys if it's an initiator or distributed_group_by_no_merge is enabled. + if (getContext()->getClientInfo().distributed_depth == 0 || settings.distributed_group_by_no_merge > 0) { - aggregation_keys_list.push_back({key}); - LOG_DEBUG(poco_log, "pushed grouping set of 1 column: " + key.name); + /// Constant expressions have non-null column pointer at this stage. + if (node->column && isColumnConst(*node->column)) + { + select_query->group_by_with_constant_keys = true; + + /// But don't remove last key column if no aggregate functions, otherwise aggregation will not work. + if (!aggregate_descriptions.empty() || size > 1) + { + if (i + 1 < static_cast(size)) + group_asts[i] = std::move(group_asts.back()); + + group_asts.pop_back(); + + --i; + continue; + } + } } - /// Key is no longer needed, therefore we can save a little by moving it. - aggregated_columns.push_back(std::move(key)); + NameAndTypePair key{column_name, node->result_type}; + + /// Aggregation keys are uniqued. + if (!unique_keys.count(key.name)) + { + unique_keys.insert(key.name); + aggregation_keys.push_back(key); + + /// Key is no longer needed, therefore we can save a little by moving it. + aggregated_columns.push_back(std::move(key)); + } } } @@ -1152,12 +1199,12 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain ASTs asts = select_query->groupBy()->children; if (select_query->group_by_with_grouping_sets) { - for (const auto & inner_asts : asts) + for (const auto & ast : asts) { - for (const auto & ast : inner_asts->children) + for(const auto & ast_element : ast->children) { - step.addRequiredOutput(ast->getColumnName()); - getRootActions(ast, only_types, step.actions()); + step.addRequiredOutput(ast_element->getColumnName()); + getRootActions(ast_element, only_types, step.actions()); } } } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index e17e5987b51..28147cec58e 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2065,13 +2065,18 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac auto & query = getSelectQuery(); if (query.group_by_with_grouping_sets) { + std::set keys_set; for (const auto & aggregation_keys : query_analyzer->aggregationKeysList()) { keys.clear(); for (const auto & key : aggregation_keys) { - keys.push_back(header_before_aggregation.getPositionByName(key.name)); - all_keys.push_back(header_before_aggregation.getPositionByName(key.name)); + size_t key_name_pos = header_before_aggregation.getPositionByName(key.name); + if (!keys_set.count(key_name_pos)) + { + keys_set.insert(key_name_pos); + } + keys.push_back(key_name_pos); LOG_DEBUG( log, "execute aggregation with grouping sets add key with name {} and number {}", @@ -2084,6 +2089,11 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac "execute aggregation with grouping sets add keys set of size {}", keys.size()); } + all_keys.assign(keys_set.begin(), keys_set.end()); + LOG_DEBUG( + log, + "execute aggregation with grouping sets add all keys of size {}", + all_keys.size()); } else { diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index 367ccc99547..4b5e093d61a 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -204,10 +204,22 @@ GroupByKeysInfo getGroupByKeysInfo(const ASTs & group_by_keys) /// filling set with short names of keys for (const auto & group_key : group_by_keys) { - if (group_key->as()) - data.has_function = true; + /// for grouping sets case + if (group_key->as()) + { + const auto express_list_ast = group_key->as(); + for (const auto & group_elem : express_list_ast.children) + { + data.key_names.insert(group_elem->getColumnName()); + } + } + else + { + if (group_key->as()) + data.has_function = true; - data.key_names.insert(group_key->getColumnName()); + data.key_names.insert(group_key->getColumnName()); + } } return data; diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index ce06485ae8e..5a0fb03bd61 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -749,18 +749,57 @@ bool ParserNotEmptyExpressionList::parseImpl(Pos & pos, ASTPtr & node, Expected return nested_parser.parse(pos, node, expected) && !node->children.empty(); } - bool ParserOrderByExpressionList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { return ParserList(std::make_unique(), std::make_unique(TokenType::Comma), false) .parse(pos, node, expected); } +bool ParserGroupingSetsExpressionListElements::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + auto command_list = std::make_shared(); + node = command_list; + + ParserToken s_comma(TokenType::Comma); + ParserToken s_open(TokenType::OpeningRoundBracket); + ParserToken s_close(TokenType::ClosingRoundBracket); + ParserExpressionWithOptionalAlias p_expression(false); + ParserList p_command(std::make_unique(false), + std::make_unique(TokenType::Comma), false); + + do + { + Pos begin = pos; + ASTPtr command; + if (!s_open.ignore(pos, expected)) + { + pos = begin; + if (!p_expression.parse(pos, command, expected)) + { + return false; + } + } + else + { + if (!p_command.parse(pos, command, expected)) + return false; + + if (!s_close.ignore(pos, expected)) + break; + } + + command_list->children.push_back(command); + } + while (s_comma.ignore(pos, expected)); + + return true; +} + bool ParserGroupingSetsExpressionList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - return ParserList(std::make_unique(false), - std::make_unique(TokenType::Comma), false) - .parse(pos, node, expected); + ParserGroupingSetsExpressionListElements grouping_sets_elements; + return grouping_sets_elements.parse(pos, node, expected); + } bool ParserTTLExpressionList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h index cd0f398c32d..5fdf4b69960 100644 --- a/src/Parsers/ExpressionListParsers.h +++ b/src/Parsers/ExpressionListParsers.h @@ -515,6 +515,12 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; +class ParserGroupingSetsExpressionListElements : public IParserBase +{ +protected: + const char * getName() const override { return "grouping sets expression elements"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; /// Parser for key-value pair, where value can be list of pairs. From 20d8614dc7de3e113d93a00e3919ab5e7fdd8f93 Mon Sep 17 00:00:00 2001 From: fanzhou Date: Mon, 23 Aug 2021 14:28:49 +0800 Subject: [PATCH 023/171] grouing sets transformer instead of fork processor in aggregating transformer --- src/Interpreters/InterpreterSelectQuery.cpp | 108 ++++++++++++++---- src/Interpreters/InterpreterSelectQuery.h | 3 +- src/Processors/QueryPlan/GroupingSetsStep.cpp | 46 ++++++++ src/Processors/QueryPlan/GroupingSetsStep.h | 25 ++++ .../Transforms/GroupingSetsTransform.cpp | 76 ++++++++++++ .../Transforms/GroupingSetsTransform.h | 35 ++++++ .../01883_with_grouping_sets.reference | 85 +++++++++++--- .../0_stateless/01883_with_grouping_sets.sql | 40 +++++-- 8 files changed, 365 insertions(+), 53 deletions(-) create mode 100644 src/Processors/QueryPlan/GroupingSetsStep.cpp create mode 100644 src/Processors/QueryPlan/GroupingSetsStep.h create mode 100644 src/Processors/Transforms/GroupingSetsTransform.cpp create mode 100644 src/Processors/Transforms/GroupingSetsTransform.h diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 28147cec58e..995f25816bb 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -967,7 +968,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

QueryProcessingStage::WithMergeableState && - !query.group_by_with_totals && !query.group_by_with_rollup && !query.group_by_with_cube; + !query.group_by_with_totals && !query.group_by_with_rollup && !query.group_by_with_cube && !query.group_by_with_grouping_sets; // if (query.group_by_with_grouping_sets && query.group_by_with_totals) // throw Exception("WITH TOTALS and GROUPING SETS are not supported together", ErrorCodes::NOT_IMPLEMENTED); @@ -1262,6 +1263,8 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

aggregationKeys()) - keys.push_back(header_before_transform.getPositionByName(key.name)); + ColumnNumbers all_keys; + ColumnNumbersTwoDimension keys_vector; + auto & query = getSelectQuery(); + if (query.group_by_with_grouping_sets) + { + std::set keys_set; + for (const auto & aggregation_keys : query_analyzer->aggregationKeysList()) + { + keys.clear(); + for (const auto & key : aggregation_keys) + { + size_t key_name_pos = header_before_transform.getPositionByName(key.name); + if (!keys_set.count(key_name_pos)) + { + keys_set.insert(key_name_pos); + } + keys.push_back(key_name_pos); + } + keys_vector.push_back(keys); + LOG_DEBUG( + log, + "execute aggregation with grouping sets add keys set of size {}", + keys.size()); + } + all_keys.assign(keys_set.begin(), keys_set.end()); + LOG_DEBUG( + log, + "execute aggregation with grouping sets add all keys of size {}", + all_keys.size()); + } + else + { + for (const auto & key : query_analyzer->aggregationKeys()) + { + keys.push_back(header_before_transform.getPositionByName(key.name)); + } + } const Settings & settings = context->getSettingsRef(); - Aggregator::Params params( - header_before_transform, - keys, - query_analyzer->aggregates(), - false, - settings.max_rows_to_group_by, - settings.group_by_overflow_mode, - 0, - 0, - settings.max_bytes_before_external_group_by, - settings.empty_result_for_aggregation_by_empty_set, - context->getTemporaryVolume(), - settings.max_threads, - settings.min_free_disk_space_for_temporary_data, - settings.compile_aggregate_expressions, - settings.min_count_to_compile_aggregate_expression); + std::shared_ptr params_ptr; + if (query.group_by_with_grouping_sets) + { + params_ptr = std::make_shared( + header_before_transform, + all_keys, + keys_vector, + query_analyzer->aggregates(), + false, + settings.max_rows_to_group_by, + settings.group_by_overflow_mode, + 0, + 0, + settings.max_bytes_before_external_group_by, + settings.empty_result_for_aggregation_by_empty_set, + context->getTemporaryVolume(), + settings.max_threads, + settings.min_free_disk_space_for_temporary_data, + settings.compile_aggregate_expressions, + settings.min_count_to_compile_aggregate_expression); + } + else + { + params_ptr = std::make_shared( + header_before_transform, + keys, + query_analyzer->aggregates(), + false, + settings.max_rows_to_group_by, + settings.group_by_overflow_mode, + 0, + 0, + settings.max_bytes_before_external_group_by, + settings.empty_result_for_aggregation_by_empty_set, + context->getTemporaryVolume(), + settings.max_threads, + settings.min_free_disk_space_for_temporary_data, + settings.compile_aggregate_expressions, + settings.min_count_to_compile_aggregate_expression); + } - auto transform_params = std::make_shared(params, true); + auto transform_params = std::make_shared(*params_ptr, true); QueryPlanStepPtr step; if (modificator == Modificator::ROLLUP) step = std::make_unique(query_plan.getCurrentDataStream(), std::move(transform_params)); - else + else if (modificator == Modificator::CUBE) step = std::make_unique(query_plan.getCurrentDataStream(), std::move(transform_params)); + else + step = std::make_unique(query_plan.getCurrentDataStream(), std::move(transform_params)); query_plan.addStep(std::move(step)); } diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 34d56a7354b..71a8ac5e1d5 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -157,7 +157,8 @@ private: enum class Modificator { ROLLUP = 0, - CUBE = 1 + CUBE = 1, + GROUPING_SETS = 2 }; void executeRollupOrCube(QueryPlan & query_plan, Modificator modificator); diff --git a/src/Processors/QueryPlan/GroupingSetsStep.cpp b/src/Processors/QueryPlan/GroupingSetsStep.cpp new file mode 100644 index 00000000000..383ecf5ea1b --- /dev/null +++ b/src/Processors/QueryPlan/GroupingSetsStep.cpp @@ -0,0 +1,46 @@ +#include +#include +#include "QueryPipeline/QueryPipelineBuilder.h" + +namespace DB +{ + +static ITransformingStep::Traits getTraits() +{ + return ITransformingStep::Traits + { + { + .preserves_distinct_columns = false, + .returns_single_stream = true, + .preserves_number_of_streams = false, + .preserves_sorting = false, + }, + { + .preserves_number_of_rows = false, + } + }; +} + +GroupingSetsStep::GroupingSetsStep(const DataStream & input_stream_, AggregatingTransformParamsPtr params_) + : ITransformingStep(input_stream_, params_->getHeader(), getTraits()) + , params(std::move(params_)) +{ + /// Aggregation keys are distinct + for (auto key : params->params.keys) + output_stream->distinct_columns.insert(params->params.src_header.getByPosition(key).name); +} + +void GroupingSetsStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) +{ + pipeline.resize(1); + + pipeline.addSimpleTransform([&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr + { + if (stream_type == QueryPipelineBuilder::StreamType::Totals) + return nullptr; + + return std::make_shared(header, std::move(params)); + }); +} + +} diff --git a/src/Processors/QueryPlan/GroupingSetsStep.h b/src/Processors/QueryPlan/GroupingSetsStep.h new file mode 100644 index 00000000000..3dc7c3b5a99 --- /dev/null +++ b/src/Processors/QueryPlan/GroupingSetsStep.h @@ -0,0 +1,25 @@ +#pragma once +#include +#include +#include "QueryPipeline/QueryPipelineBuilder.h" + +namespace DB +{ + +struct AggregatingTransformParams; +using AggregatingTransformParamsPtr = std::shared_ptr; + +class GroupingSetsStep : public ITransformingStep +{ +public: + GroupingSetsStep(const DataStream & input_stream_, AggregatingTransformParamsPtr params_); + + String getName() const override { return "GroupingSets"; } + + void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override; + +private: + AggregatingTransformParamsPtr params; +}; + +} diff --git a/src/Processors/Transforms/GroupingSetsTransform.cpp b/src/Processors/Transforms/GroupingSetsTransform.cpp new file mode 100644 index 00000000000..9672b8e32c2 --- /dev/null +++ b/src/Processors/Transforms/GroupingSetsTransform.cpp @@ -0,0 +1,76 @@ +#include +#include + +namespace DB +{ + +GroupingSetsTransform::GroupingSetsTransform(Block header, AggregatingTransformParamsPtr params_) + : IAccumulatingTransform(std::move(header), params_->getHeader()) + , params(std::move(params_)) + , keys(params->params.keys) + , keys_vector(params->params.keys_vector) + , keys_vector_idx(0) +{ +} + +void GroupingSetsTransform::consume(Chunk chunk) +{ + consumed_chunks.emplace_back(std::move(chunk)); +} + +Chunk GroupingSetsTransform::merge(Chunks && chunks, bool final) +{ + BlocksList grouping_sets_blocks; + for (auto & chunk : chunks) + grouping_sets_blocks.emplace_back(getInputPort().getHeader().cloneWithColumns(chunk.detachColumns())); + + auto grouping_sets_block = params->aggregator.mergeBlocks(grouping_sets_blocks, final); + auto num_rows = grouping_sets_block.rows(); + return Chunk(grouping_sets_block.getColumns(), num_rows); +} + +Chunk GroupingSetsTransform::generate() +{ + if (!consumed_chunks.empty()) + { + if (consumed_chunks.size() > 1) + grouping_sets_chunk = merge(std::move(consumed_chunks), false); + else + grouping_sets_chunk = std::move(consumed_chunks.front()); + + consumed_chunks.clear(); + + auto num_rows = grouping_sets_chunk.getNumRows(); + + current_columns = grouping_sets_chunk.getColumns(); + current_zero_columns.clear(); + + for (auto key : keys) + current_zero_columns.emplace(key, current_columns[key]->cloneEmpty()->cloneResized(num_rows)); + } + + Chunk gen_chunk; + + if (keys_vector_idx < keys_vector.size()) + { + auto columns = current_columns; + std::set key_vector(keys_vector[keys_vector_idx].begin(), keys_vector[keys_vector_idx].end()); + + for (auto key : keys) + { + if (!key_vector.contains(key)) + columns[key] = current_zero_columns[key]; + } + + Chunks chunks; + chunks.emplace_back(std::move(columns), current_columns.front()->size()); + gen_chunk = merge(std::move(chunks), false); + + ++keys_vector_idx; + } + + finalizeChunk(gen_chunk); + return gen_chunk; +} + +} diff --git a/src/Processors/Transforms/GroupingSetsTransform.h b/src/Processors/Transforms/GroupingSetsTransform.h new file mode 100644 index 00000000000..4019b2ffa52 --- /dev/null +++ b/src/Processors/Transforms/GroupingSetsTransform.h @@ -0,0 +1,35 @@ +#pragma once +#include +#include + +namespace DB +{ + +class GroupingSetsTransform : public IAccumulatingTransform +{ +public: + GroupingSetsTransform(Block header, AggregatingTransformParamsPtr params); + String getName() const override { return "GroupingSetsTransform"; } + +protected: + void consume(Chunk chunk) override; + Chunk generate() override; + +private: + AggregatingTransformParamsPtr params; + ColumnNumbers keys; + ColumnNumbersTwoDimension keys_vector; + + Chunks consumed_chunks; + Chunk grouping_sets_chunk; + Columns current_columns; + std::unordered_map current_zero_columns; + + UInt64 keys_vector_idx = 0; + + Poco::Logger * log = &Poco::Logger::get("GroupingSetsTransform"); + + Chunk merge(Chunks && chunks, bool final); +}; + +} diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.reference b/tests/queries/0_stateless/01883_with_grouping_sets.reference index fd128444ebc..258fa6f5abd 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.reference +++ b/tests/queries/0_stateless/01883_with_grouping_sets.reference @@ -1,20 +1,69 @@ - 1 40 4 - 2 80 4 -a 0 70 4 -b 0 50 4 - 1 40 4 - 2 80 4 -a 0 70 4 -b 0 50 4 +1 0 1 4500 +1 0 3 4700 +1 0 5 4900 +1 0 7 5100 +1 0 9 5300 +1 1 0 4500 +1 2 0 5100 +1 3 0 4700 +1 4 0 5300 +1 5 0 4900 +2 0 2 4600 +2 0 4 4800 +2 0 6 5000 +2 0 8 5200 +2 0 10 5400 +2 1 0 5000 +2 2 0 4600 +2 3 0 5200 +2 4 0 4800 +2 5 0 5400 - 0 240 16 - 1 40 4 - 2 80 4 -a 0 70 4 -b 0 50 4 - 1 40 4 - 2 80 4 -a 0 70 4 -b 0 50 4 +0 0 1 1 4500 +0 0 2 2 4600 +0 0 3 3 4700 +0 0 4 4 4800 +0 0 5 5 4900 +0 0 6 6 5000 +0 0 7 7 5100 +0 0 8 8 5200 +0 0 9 9 5300 +0 0 10 10 5400 +1 1 0 0 4500 +1 2 0 0 5100 +1 3 0 0 4700 +1 4 0 0 5300 +1 5 0 0 4900 +2 1 0 0 5000 +2 2 0 0 4600 +2 3 0 0 5200 +2 4 0 0 4800 +2 5 0 0 5400 - 0 240 16 +1 0 24500 +1 1 4500 +1 3 4700 +1 5 4900 +1 7 5100 +1 9 5300 +2 0 25000 +2 2 4600 +2 4 4800 +2 6 5000 +2 8 5200 +2 10 5400 +0 0 49500 + +1 0 24500 +1 1 4500 +1 3 4700 +1 5 4900 +1 7 5100 +1 9 5300 +2 0 25000 +2 2 4600 +2 4 4800 +2 6 5000 +2 8 5200 +2 10 5400 +0 0 49500 diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.sql b/tests/queries/0_stateless/01883_with_grouping_sets.sql index 6270b772328..274eebef5ed 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.sql +++ b/tests/queries/0_stateless/01883_with_grouping_sets.sql @@ -1,20 +1,38 @@ DROP TABLE IF EXISTS grouping_sets; -CREATE TABLE grouping_sets(a String, b Int32, s Int32) ENGINE = Memory; -INSERT INTO grouping_sets VALUES ('a', 1, 10), ('a', 1, 15), ('a', 2, 20); -INSERT INTO grouping_sets VALUES ('a', 2, 25), ('b', 1, 10), ('b', 1, 5); -INSERT INTO grouping_sets VALUES ('b', 2, 20), ('b', 2, 15); +CREATE TABLE grouping_sets(fact_1_id Int32, fact_2_id Int32, fact_3_id Int32, fact_4_id Int32, sales_value Int32) ENGINE = Memory; -SELECT a, b, sum(s), count() from grouping_sets GROUP BY GROUPING SETS(a, b) ORDER BY a, b; +INSERT INTO grouping_sets +SELECT + number % 2 + 1 AS fact_1_id, + number % 5 + 1 AS fact_2_id, + number % 10 + 1 AS fact_3_id, + number % 10 + 1 AS fact_4_id, + number % 100 AS sales_value +FROM system.numbers limit 1000; -SELECT a, b, sum(s), count() from grouping_sets GROUP BY GROUPING SETS(a, b) WITH TOTALS ORDER BY a, b; +SELECT fact_1_id, fact_2_id, fact_3_id, SUM(sales_value) AS sales_value from grouping_sets +GROUP BY GROUPING SETS((fact_1_id, fact_2_id), (fact_1_id, fact_3_id)) +ORDER BY fact_1_id, fact_2_id, fact_3_id; -SELECT a, b, sum(s), count() from grouping_sets GROUP BY a, b WITH GROUPING SETS ORDER BY a, b; +SELECT fact_1_id, fact_2_id, fact_3_id, fact_4_id, SUM(sales_value) AS sales_value from grouping_sets +GROUP BY GROUPING SETS((fact_1_id, fact_2_id), (fact_3_id, fact_4_id)) +ORDER BY fact_1_id, fact_2_id, fact_3_id, fact_4_id; -SELECT a, b, sum(s), count() from grouping_sets GROUP BY a, b WITH GROUPING SETS WITH TOTALS ORDER BY a, b; +SELECT + fact_1_id, + fact_3_id, + SUM(sales_value) AS sales_value +FROM grouping_sets +GROUP BY grouping sets((fact_1_id), (fact_1_id, fact_3_id)) WITH TOTALS +ORDER BY fact_1_id, fact_3_id; --- not sure that always works --- SET group_by_two_level_threshold = 1; --- SELECT a, b, sum(s), count() from grouping_sets GROUP BY a, b WITH GROUPING SETS ORDER BY a, b; +SELECT + fact_1_id, + fact_3_id, + SUM(sales_value) AS sales_value +FROM grouping_sets +GROUP BY grouping sets(fact_1_id, (fact_1_id, fact_3_id)) WITH TOTALS +ORDER BY fact_1_id, fact_3_id; DROP TABLE grouping_sets; From 8d4d48362daf1744ffe8440b81ff5d82f4ef2dc6 Mon Sep 17 00:00:00 2001 From: fanzhou Date: Sat, 28 Aug 2021 12:10:41 +0800 Subject: [PATCH 024/171] fix bug when no data in grouping sets transformer --- src/Processors/Transforms/GroupingSetsTransform.cpp | 2 +- tests/queries/0_stateless/01883_with_grouping_sets.reference | 2 ++ tests/queries/0_stateless/01883_with_grouping_sets.sql | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Processors/Transforms/GroupingSetsTransform.cpp b/src/Processors/Transforms/GroupingSetsTransform.cpp index 9672b8e32c2..960ea0e5d76 100644 --- a/src/Processors/Transforms/GroupingSetsTransform.cpp +++ b/src/Processors/Transforms/GroupingSetsTransform.cpp @@ -51,7 +51,7 @@ Chunk GroupingSetsTransform::generate() Chunk gen_chunk; - if (keys_vector_idx < keys_vector.size()) + if (!current_columns.empty() && keys_vector_idx < keys_vector.size()) { auto columns = current_columns; std::set key_vector(keys_vector[keys_vector_idx].begin(), keys_vector[keys_vector_idx].end()); diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.reference b/tests/queries/0_stateless/01883_with_grouping_sets.reference index 258fa6f5abd..5d5341dd093 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.reference +++ b/tests/queries/0_stateless/01883_with_grouping_sets.reference @@ -67,3 +67,5 @@ 2 8 5200 2 10 5400 0 0 49500 + + diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.sql b/tests/queries/0_stateless/01883_with_grouping_sets.sql index 274eebef5ed..24350143849 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.sql +++ b/tests/queries/0_stateless/01883_with_grouping_sets.sql @@ -35,4 +35,8 @@ FROM grouping_sets GROUP BY grouping sets(fact_1_id, (fact_1_id, fact_3_id)) WITH TOTALS ORDER BY fact_1_id, fact_3_id; +truncate grouping_sets; + +SELECT a, b, sum(s), count() from grouping_sets GROUP BY GROUPING SETS(a, b) ORDER BY a, b; + DROP TABLE grouping_sets; From 4b25d3078304e52f03b4f961f65df0248c03d7d2 Mon Sep 17 00:00:00 2001 From: fanzhou Date: Sun, 29 Aug 2021 15:13:53 +0800 Subject: [PATCH 025/171] correct test case results --- .../queries/0_stateless/01883_with_grouping_sets.reference | 7 ++----- tests/queries/0_stateless/01883_with_grouping_sets.sql | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.reference b/tests/queries/0_stateless/01883_with_grouping_sets.reference index 5d5341dd093..a1d55448f3f 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.reference +++ b/tests/queries/0_stateless/01883_with_grouping_sets.reference @@ -18,7 +18,6 @@ 2 3 0 5200 2 4 0 4800 2 5 0 5400 - 0 0 1 1 4500 0 0 2 2 4600 0 0 3 3 4700 @@ -39,7 +38,6 @@ 2 3 0 0 5200 2 4 0 0 4800 2 5 0 0 5400 - 1 0 24500 1 1 4500 1 3 4700 @@ -52,8 +50,8 @@ 2 6 5000 2 8 5200 2 10 5400 -0 0 49500 +0 0 49500 1 0 24500 1 1 4500 1 3 4700 @@ -66,6 +64,5 @@ 2 6 5000 2 8 5200 2 10 5400 -0 0 49500 - +0 0 49500 \ No newline at end of file diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.sql b/tests/queries/0_stateless/01883_with_grouping_sets.sql index 24350143849..fffe4314706 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.sql +++ b/tests/queries/0_stateless/01883_with_grouping_sets.sql @@ -4,7 +4,7 @@ CREATE TABLE grouping_sets(fact_1_id Int32, fact_2_id Int32, fact_3_id Int32, fa INSERT INTO grouping_sets SELECT - number % 2 + 1 AS fact_1_id, + number % 2 + 1 AS fact_1_id, number % 5 + 1 AS fact_2_id, number % 10 + 1 AS fact_3_id, number % 10 + 1 AS fact_4_id, @@ -37,6 +37,6 @@ ORDER BY fact_1_id, fact_3_id; truncate grouping_sets; -SELECT a, b, sum(s), count() from grouping_sets GROUP BY GROUPING SETS(a, b) ORDER BY a, b; +SELECT fact_1_id, fact_3_id, sum(sales_value), count() from grouping_sets GROUP BY GROUPING SETS(fact_1_id, fact_3_id) ORDER BY fact_1_id, fact_3_id; DROP TABLE grouping_sets; From 83e9e5d0e59bfaeacf6353543a999408812fc247 Mon Sep 17 00:00:00 2001 From: fanzhou Date: Mon, 6 Sep 2021 10:18:42 +0800 Subject: [PATCH 026/171] some changes --- src/Core/ColumnNumbers.h | 2 +- src/Core/NamesAndTypes.h | 2 +- src/Interpreters/Aggregator.h | 4 +- src/Interpreters/ExpressionAnalyzer.cpp | 2 +- src/Interpreters/ExpressionAnalyzer.h | 4 +- src/Interpreters/InterpreterSelectQuery.cpp | 79 ++++++------------- .../Transforms/GroupingSetsTransform.h | 2 +- src/QueryPipeline/Pipe.cpp | 4 +- .../01883_with_grouping_sets.reference | 7 +- 9 files changed, 40 insertions(+), 66 deletions(-) diff --git a/src/Core/ColumnNumbers.h b/src/Core/ColumnNumbers.h index 82868ef3cd3..29b4c49dc83 100644 --- a/src/Core/ColumnNumbers.h +++ b/src/Core/ColumnNumbers.h @@ -8,6 +8,6 @@ namespace DB { using ColumnNumbers = std::vector; -using ColumnNumbersTwoDimension = std::vector; +using ColumnNumbersList = std::vector; } diff --git a/src/Core/NamesAndTypes.h b/src/Core/NamesAndTypes.h index dacd5a459dc..2719017a726 100644 --- a/src/Core/NamesAndTypes.h +++ b/src/Core/NamesAndTypes.h @@ -109,7 +109,7 @@ public: std::optional tryGetByName(const std::string & name) const; }; -using TwoDimensionNamesAndTypesList = std::list; +using NamesAndTypesLists = std::vector; } diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index b9dca0b09ae..e8635c4e31a 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -879,7 +879,7 @@ public: /// What to count. ColumnNumbers keys; - const ColumnNumbersTwoDimension keys_vector; + const ColumnNumbersList keys_vector; const AggregateDescriptions aggregates; size_t keys_size; const size_t aggregates_size; @@ -943,7 +943,7 @@ public: Params( const Block & src_header_, const ColumnNumbers & keys_, - const ColumnNumbersTwoDimension & keys_vector_, const AggregateDescriptions & aggregates_, + const ColumnNumbersList & keys_vector_, const AggregateDescriptions & aggregates_, bool overflow_row_, size_t max_rows_to_group_by_, OverflowMode group_by_overflow_mode_, size_t group_by_two_level_threshold_, size_t group_by_two_level_threshold_bytes_, size_t max_bytes_before_external_group_by_, diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index cd2e00c832b..1834479df69 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -395,7 +395,7 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) } } - aggregation_keys_list.push_back(grouping_set_list); + aggregation_keys_list.push_back(std::move(grouping_set_list)); } else { diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index 7598efa40c6..19054e795a6 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -64,8 +64,8 @@ struct ExpressionAnalyzerData bool has_aggregation = false; NamesAndTypesList aggregation_keys; + NamesAndTypesLists aggregation_keys_list; bool has_const_aggregation_keys = false; - TwoDimensionNamesAndTypesList aggregation_keys_list; AggregateDescriptions aggregate_descriptions; WindowDescriptions window_descriptions; @@ -324,7 +324,7 @@ public: const NamesAndTypesList & aggregationKeys() const { return aggregation_keys; } bool hasConstAggregationKeys() const { return has_const_aggregation_keys; } - const TwoDimensionNamesAndTypesList & aggregationKeysList() const { return aggregation_keys_list; } + const NamesAndTypesLists & aggregationKeysList() const { return aggregation_keys_list; } const AggregateDescriptions & aggregates() const { return aggregate_descriptions; } const PreparedSets & getPreparedSets() const { return prepared_sets; } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 995f25816bb..970a1362699 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2062,61 +2062,35 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac return; const auto & header_before_aggregation = query_plan.getCurrentDataStream().header; - ColumnNumbers keys; - ColumnNumbers all_keys; - ColumnNumbersTwoDimension keys_vector; - auto & query = getSelectQuery(); - if (query.group_by_with_grouping_sets) - { - std::set keys_set; - for (const auto & aggregation_keys : query_analyzer->aggregationKeysList()) - { - keys.clear(); - for (const auto & key : aggregation_keys) - { - size_t key_name_pos = header_before_aggregation.getPositionByName(key.name); - if (!keys_set.count(key_name_pos)) - { - keys_set.insert(key_name_pos); - } - keys.push_back(key_name_pos); - LOG_DEBUG( - log, - "execute aggregation with grouping sets add key with name {} and number {}", - key.name, - header_before_aggregation.getPositionByName(key.name)); - } - keys_vector.push_back(keys); - LOG_DEBUG( - log, - "execute aggregation with grouping sets add keys set of size {}", - keys.size()); - } - all_keys.assign(keys_set.begin(), keys_set.end()); - LOG_DEBUG( - log, - "execute aggregation with grouping sets add all keys of size {}", - all_keys.size()); - } - else - { - for (const auto & key : query_analyzer->aggregationKeys()) - { - keys.push_back(header_before_aggregation.getPositionByName(key.name)); - LOG_DEBUG(log, "execute aggregation without grouping sets pushed back key with name {} and number {}", key.name, header_before_aggregation.getPositionByName(key.name)); - } - } AggregateDescriptions aggregates = query_analyzer->aggregates(); for (auto & descr : aggregates) if (descr.arguments.empty()) for (const auto & name : descr.argument_names) descr.arguments.push_back(header_before_aggregation.getPositionByName(name)); - const Settings & settings = context->getSettingsRef(); std::shared_ptr params_ptr; + + auto & query = getSelectQuery(); if (query.group_by_with_grouping_sets) { + ColumnNumbers keys; + ColumnNumbers all_keys; + ColumnNumbersList keys_vector; + std::unordered_set keys_set; + for (const auto & aggregation_keys : query_analyzer->aggregationKeysList()) + { + keys.clear(); + for (const auto & key : aggregation_keys) + { + size_t key_name_pos = header_before_aggregation.getPositionByName(key.name); + keys_set.insert(key_name_pos); + keys.push_back(key_name_pos); + } + keys_vector.push_back(keys); + } + all_keys.assign(keys_set.begin(), keys_set.end()); + params_ptr = std::make_shared( header_before_aggregation, all_keys, @@ -2139,6 +2113,10 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac } else { + ColumnNumbers keys; + for (const auto & key : query_analyzer->aggregationKeys()) + keys.push_back(header_before_aggregation.getPositionByName(key.name)); + params_ptr = std::make_shared( header_before_aggregation, keys, @@ -2160,13 +2138,10 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac } SortDescription group_by_sort_description; - if (group_by_info && settings.optimize_aggregation_in_order && !query.group_by_with_grouping_sets) { + if (group_by_info && settings.optimize_aggregation_in_order && !query.group_by_with_grouping_sets) group_by_sort_description = getSortDescriptionFromGroupBy(query); - LOG_DEBUG(log, "execute aggregation without grouping sets got group_by_sort_description"); - } else { + else group_by_info = nullptr; - LOG_DEBUG(log, "execute aggregation didn't get group_by_sort_description"); - } auto merge_threads = max_streams; auto temporary_data_merge_threads = settings.aggregation_memory_efficient_merge_threads @@ -2175,7 +2150,6 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac bool storage_has_evenly_distributed_read = storage && storage->hasEvenlyDistributedRead(); - LOG_DEBUG(log, "execute aggregation header structure before step: {}", query_plan.getCurrentDataStream().header.dumpStructure()); auto aggregating_step = std::make_unique( query_plan.getCurrentDataStream(), *params_ptr, @@ -2187,7 +2161,6 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac storage_has_evenly_distributed_read, std::move(group_by_info), std::move(group_by_sort_description)); - LOG_DEBUG(log, "execute aggregation header structure after step: {}", aggregating_step->getOutputStream().header.dumpStructure()); query_plan.addStep(std::move(aggregating_step)); } @@ -2245,7 +2218,7 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPlan & query_plan, Modific const auto & header_before_transform = query_plan.getCurrentDataStream().header; ColumnNumbers keys; ColumnNumbers all_keys; - ColumnNumbersTwoDimension keys_vector; + ColumnNumbersList keys_vector; auto & query = getSelectQuery(); if (query.group_by_with_grouping_sets) { diff --git a/src/Processors/Transforms/GroupingSetsTransform.h b/src/Processors/Transforms/GroupingSetsTransform.h index 4019b2ffa52..f9c688b2cc8 100644 --- a/src/Processors/Transforms/GroupingSetsTransform.h +++ b/src/Processors/Transforms/GroupingSetsTransform.h @@ -18,7 +18,7 @@ protected: private: AggregatingTransformParamsPtr params; ColumnNumbers keys; - ColumnNumbersTwoDimension keys_vector; + ColumnNumbersList keys_vector; Chunks consumed_chunks; Chunk grouping_sets_chunk; diff --git a/src/QueryPipeline/Pipe.cpp b/src/QueryPipeline/Pipe.cpp index 184cad18c17..dd4a2cb6c77 100644 --- a/src/QueryPipeline/Pipe.cpp +++ b/src/QueryPipeline/Pipe.cpp @@ -464,9 +464,9 @@ void Pipe::addParallelTransforms(Processors transforms) "but " + std::to_string(output_ports.size()) + " expected", ErrorCodes::LOGICAL_ERROR); size_t next_output = 0; - for (auto * input : inputs) + for (size_t i = 0; i < inputs.size(); ++i) { - connect(*output_ports[next_output], *input); + connect(*output_ports[next_output], *inputs[i]); ++next_output; } diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.reference b/tests/queries/0_stateless/01883_with_grouping_sets.reference index a1d55448f3f..258fa6f5abd 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.reference +++ b/tests/queries/0_stateless/01883_with_grouping_sets.reference @@ -18,6 +18,7 @@ 2 3 0 5200 2 4 0 4800 2 5 0 5400 + 0 0 1 1 4500 0 0 2 2 4600 0 0 3 3 4700 @@ -38,6 +39,7 @@ 2 3 0 0 5200 2 4 0 0 4800 2 5 0 0 5400 + 1 0 24500 1 1 4500 1 3 4700 @@ -50,8 +52,8 @@ 2 6 5000 2 8 5200 2 10 5400 - 0 0 49500 + 1 0 24500 1 1 4500 1 3 4700 @@ -64,5 +66,4 @@ 2 6 5000 2 8 5200 2 10 5400 - -0 0 49500 \ No newline at end of file +0 0 49500 From a2c2317d75402dc900a1c78ca06e636ea443b1f0 Mon Sep 17 00:00:00 2001 From: fanzhou Date: Mon, 6 Sep 2021 11:05:05 +0800 Subject: [PATCH 027/171] refactor --- src/Interpreters/InterpreterSelectQuery.cpp | 42 +++++++-------------- 1 file changed, 13 insertions(+), 29 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 970a1362699..41098747f41 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2216,50 +2216,30 @@ void InterpreterSelectQuery::executeTotalsAndHaving( void InterpreterSelectQuery::executeRollupOrCube(QueryPlan & query_plan, Modificator modificator) { const auto & header_before_transform = query_plan.getCurrentDataStream().header; - ColumnNumbers keys; - ColumnNumbers all_keys; - ColumnNumbersList keys_vector; + + const Settings & settings = context->getSettingsRef(); + std::shared_ptr params_ptr; + auto & query = getSelectQuery(); if (query.group_by_with_grouping_sets) { - std::set keys_set; + ColumnNumbers keys; + ColumnNumbers all_keys; + ColumnNumbersList keys_vector; + std::unordered_set keys_set; for (const auto & aggregation_keys : query_analyzer->aggregationKeysList()) { keys.clear(); for (const auto & key : aggregation_keys) { size_t key_name_pos = header_before_transform.getPositionByName(key.name); - if (!keys_set.count(key_name_pos)) - { - keys_set.insert(key_name_pos); - } + keys_set.insert(key_name_pos); keys.push_back(key_name_pos); } keys_vector.push_back(keys); - LOG_DEBUG( - log, - "execute aggregation with grouping sets add keys set of size {}", - keys.size()); } all_keys.assign(keys_set.begin(), keys_set.end()); - LOG_DEBUG( - log, - "execute aggregation with grouping sets add all keys of size {}", - all_keys.size()); - } - else - { - for (const auto & key : query_analyzer->aggregationKeys()) - { - keys.push_back(header_before_transform.getPositionByName(key.name)); - } - } - const Settings & settings = context->getSettingsRef(); - - std::shared_ptr params_ptr; - if (query.group_by_with_grouping_sets) - { params_ptr = std::make_shared( header_before_transform, all_keys, @@ -2280,6 +2260,10 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPlan & query_plan, Modific } else { + ColumnNumbers keys; + for (const auto & key : query_analyzer->aggregationKeys()) + keys.push_back(header_before_transform.getPositionByName(key.name)); + params_ptr = std::make_shared( header_before_transform, keys, From 6c6d3015de683fe6b4545d1e09f640357db836f0 Mon Sep 17 00:00:00 2001 From: fanzhou Date: Sat, 11 Sep 2021 11:08:22 +0800 Subject: [PATCH 028/171] 1. refactor ExpressionAnalyzer 2. Update formatAST to support grouping sets query with distributed table 3. modify astExpression to support function in grouping sets --- src/Interpreters/ExpressionAnalyzer.cpp | 13 +- src/Interpreters/InterpreterSelectQuery.cpp | 213 +++++++----------- src/Interpreters/InterpreterSelectQuery.h | 7 + src/Parsers/ASTExpressionList.cpp | 87 +++++-- src/Parsers/ASTSelectQuery.cpp | 15 +- src/Parsers/ExpressionListParsers.cpp | 5 +- .../01883_with_grouping_sets.reference | 24 ++ .../0_stateless/01883_with_grouping_sets.sql | 18 +- 8 files changed, 214 insertions(+), 168 deletions(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 1834479df69..5b207051029 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -345,15 +345,10 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) if (select_query->group_by_with_grouping_sets) { ASTs group_elements_ast; - if (group_asts[i]->as()) - { - group_elements_ast = group_asts[i]->as()->children; - } - else - { - const auto id_ast = group_asts[i]->as(); - group_elements_ast.push_back(std::make_shared(id_ast)); - } + const ASTExpressionList * group_ast_element = group_asts[i]->as(); + if (!group_ast_element) + throw Exception("Grouping Sets element " + group_asts[i]->getColumnName() + " should be an expression type", ErrorCodes::UNKNOWN_IDENTIFIER); + group_elements_ast = group_ast_element->children; NamesAndTypesList grouping_set_list; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 41098747f41..c04816cb794 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2041,7 +2041,6 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc } } - void InterpreterSelectQuery::executeWhere(QueryPlan & query_plan, const ActionsDAGPtr & expression, bool remove_filter) { auto where_step = std::make_unique( @@ -2051,6 +2050,79 @@ void InterpreterSelectQuery::executeWhere(QueryPlan & query_plan, const ActionsD query_plan.addStep(std::move(where_step)); } +void InterpreterSelectQuery::initAggregatorParams( + const Block & current_data_stream_header, + AggregatorParamsPtr & params_ptr, + const AggregateDescriptions & aggregates, + bool overflow_row, const Settings & settings, + size_t group_by_two_level_threshold, size_t group_by_two_level_threshold_bytes) +{ + auto & query = getSelectQuery(); + if (query.group_by_with_grouping_sets) + { + ColumnNumbers keys; + ColumnNumbers all_keys; + ColumnNumbersList keys_vector; + std::unordered_set keys_set; + for (const auto & aggregation_keys : query_analyzer->aggregationKeysList()) + { + keys.clear(); + for (const auto & key : aggregation_keys) + { + size_t key_name_pos = current_data_stream_header.getPositionByName(key.name); + keys_set.insert(key_name_pos); + keys.push_back(key_name_pos); + } + keys_vector.push_back(keys); + } + all_keys.assign(keys_set.begin(), keys_set.end()); + + params_ptr = std::make_unique( + current_data_stream_header, + all_keys, + keys_vector, + aggregates, + overflow_row, + settings.max_rows_to_group_by, + settings.group_by_overflow_mode, + group_by_two_level_threshold, + group_by_two_level_threshold_bytes, + settings.max_bytes_before_external_group_by, + settings.empty_result_for_aggregation_by_empty_set + || (settings.empty_result_for_aggregation_by_constant_keys_on_empty_set && keys.empty() + && query_analyzer->hasConstAggregationKeys()), + context->getTemporaryVolume(), + settings.max_threads, + settings.min_free_disk_space_for_temporary_data, + settings.compile_aggregate_expressions, + settings.min_count_to_compile_aggregate_expression); + } + else + { + ColumnNumbers keys; + for (const auto & key : query_analyzer->aggregationKeys()) + keys.push_back(current_data_stream_header.getPositionByName(key.name)); + + params_ptr = std::make_unique( + current_data_stream_header, + keys, + aggregates, + overflow_row, + settings.max_rows_to_group_by, + settings.group_by_overflow_mode, + group_by_two_level_threshold, + group_by_two_level_threshold_bytes, + settings.max_bytes_before_external_group_by, + settings.empty_result_for_aggregation_by_empty_set + || (settings.empty_result_for_aggregation_by_constant_keys_on_empty_set && keys.empty() + && query_analyzer->hasConstAggregationKeys()), + context->getTemporaryVolume(), + settings.max_threads, + settings.min_free_disk_space_for_temporary_data, + settings.compile_aggregate_expressions, + settings.min_count_to_compile_aggregate_expression); + } +} void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const ActionsDAGPtr & expression, bool overflow_row, bool final, InputOrderInfoPtr group_by_info) { @@ -2068,78 +2140,17 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac if (descr.arguments.empty()) for (const auto & name : descr.argument_names) descr.arguments.push_back(header_before_aggregation.getPositionByName(name)); + const Settings & settings = context->getSettingsRef(); - std::shared_ptr params_ptr; - auto & query = getSelectQuery(); - if (query.group_by_with_grouping_sets) - { - ColumnNumbers keys; - ColumnNumbers all_keys; - ColumnNumbersList keys_vector; - std::unordered_set keys_set; - for (const auto & aggregation_keys : query_analyzer->aggregationKeysList()) - { - keys.clear(); - for (const auto & key : aggregation_keys) - { - size_t key_name_pos = header_before_aggregation.getPositionByName(key.name); - keys_set.insert(key_name_pos); - keys.push_back(key_name_pos); - } - keys_vector.push_back(keys); - } - all_keys.assign(keys_set.begin(), keys_set.end()); + AggregatorParamsPtr params_ptr; + initAggregatorParams(header_before_aggregation, params_ptr, aggregates, overflow_row, settings, + settings.group_by_two_level_threshold, settings.group_by_two_level_threshold_bytes); - params_ptr = std::make_shared( - header_before_aggregation, - all_keys, - keys_vector, - aggregates, - overflow_row, - settings.max_rows_to_group_by, - settings.group_by_overflow_mode, - settings.group_by_two_level_threshold, - settings.group_by_two_level_threshold_bytes, - settings.max_bytes_before_external_group_by, - settings.empty_result_for_aggregation_by_empty_set - || (settings.empty_result_for_aggregation_by_constant_keys_on_empty_set && keys.empty() - && query_analyzer->hasConstAggregationKeys()), - context->getTemporaryVolume(), - settings.max_threads, - settings.min_free_disk_space_for_temporary_data, - settings.compile_aggregate_expressions, - settings.min_count_to_compile_aggregate_expression); - } - else - { - ColumnNumbers keys; - for (const auto & key : query_analyzer->aggregationKeys()) - keys.push_back(header_before_aggregation.getPositionByName(key.name)); - - params_ptr = std::make_shared( - header_before_aggregation, - keys, - aggregates, - overflow_row, - settings.max_rows_to_group_by, - settings.group_by_overflow_mode, - settings.group_by_two_level_threshold, - settings.group_by_two_level_threshold_bytes, - settings.max_bytes_before_external_group_by, - settings.empty_result_for_aggregation_by_empty_set - || (settings.empty_result_for_aggregation_by_constant_keys_on_empty_set && keys.empty() - && query_analyzer->hasConstAggregationKeys()), - context->getTemporaryVolume(), - settings.max_threads, - settings.min_free_disk_space_for_temporary_data, - settings.compile_aggregate_expressions, - settings.min_count_to_compile_aggregate_expression); - } SortDescription group_by_sort_description; - if (group_by_info && settings.optimize_aggregation_in_order && !query.group_by_with_grouping_sets) - group_by_sort_description = getSortDescriptionFromGroupBy(query); + if (group_by_info && settings.optimize_aggregation_in_order) + group_by_sort_description = getSortDescriptionFromGroupBy(getSelectQuery()); else group_by_info = nullptr; @@ -2212,76 +2223,14 @@ void InterpreterSelectQuery::executeTotalsAndHaving( query_plan.addStep(std::move(totals_having_step)); } - void InterpreterSelectQuery::executeRollupOrCube(QueryPlan & query_plan, Modificator modificator) { const auto & header_before_transform = query_plan.getCurrentDataStream().header; const Settings & settings = context->getSettingsRef(); - std::shared_ptr params_ptr; - - auto & query = getSelectQuery(); - if (query.group_by_with_grouping_sets) - { - ColumnNumbers keys; - ColumnNumbers all_keys; - ColumnNumbersList keys_vector; - std::unordered_set keys_set; - for (const auto & aggregation_keys : query_analyzer->aggregationKeysList()) - { - keys.clear(); - for (const auto & key : aggregation_keys) - { - size_t key_name_pos = header_before_transform.getPositionByName(key.name); - keys_set.insert(key_name_pos); - keys.push_back(key_name_pos); - } - keys_vector.push_back(keys); - } - all_keys.assign(keys_set.begin(), keys_set.end()); - - params_ptr = std::make_shared( - header_before_transform, - all_keys, - keys_vector, - query_analyzer->aggregates(), - false, - settings.max_rows_to_group_by, - settings.group_by_overflow_mode, - 0, - 0, - settings.max_bytes_before_external_group_by, - settings.empty_result_for_aggregation_by_empty_set, - context->getTemporaryVolume(), - settings.max_threads, - settings.min_free_disk_space_for_temporary_data, - settings.compile_aggregate_expressions, - settings.min_count_to_compile_aggregate_expression); - } - else - { - ColumnNumbers keys; - for (const auto & key : query_analyzer->aggregationKeys()) - keys.push_back(header_before_transform.getPositionByName(key.name)); - - params_ptr = std::make_shared( - header_before_transform, - keys, - query_analyzer->aggregates(), - false, - settings.max_rows_to_group_by, - settings.group_by_overflow_mode, - 0, - 0, - settings.max_bytes_before_external_group_by, - settings.empty_result_for_aggregation_by_empty_set, - context->getTemporaryVolume(), - settings.max_threads, - settings.min_free_disk_space_for_temporary_data, - settings.compile_aggregate_expressions, - settings.min_count_to_compile_aggregate_expression); - } + AggregatorParamsPtr params_ptr; + initAggregatorParams(header_before_transform, params_ptr, query_analyzer->aggregates(), false, settings, 0, 0); auto transform_params = std::make_shared(*params_ptr, true); QueryPlanStepPtr step; diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 71a8ac5e1d5..7f75c4ea41f 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -30,6 +30,7 @@ class QueryPlan; struct TreeRewriterResult; using TreeRewriterResultPtr = std::shared_ptr; +using AggregatorParamsPtr = std::unique_ptr; /** Interprets the SELECT query. Returns the stream of blocks with the results of the query before `to_stage` stage. @@ -127,6 +128,12 @@ private: /// Different stages of query execution. + void initAggregatorParams( + const Block & current_data_stream_header, + AggregatorParamsPtr & params_ptr, + const AggregateDescriptions & aggregates, + bool overflow_row, const Settings & settings, + size_t group_by_two_level_threshold, size_t group_by_two_level_threshold_bytes); void executeFetchColumns(QueryProcessingStage::Enum processing_stage, QueryPlan & query_plan); void executeWhere(QueryPlan & query_plan, const ActionsDAGPtr & expression, bool remove_filter); void executeAggregation( diff --git a/src/Parsers/ASTExpressionList.cpp b/src/Parsers/ASTExpressionList.cpp index 2724465537f..453624aa794 100644 --- a/src/Parsers/ASTExpressionList.cpp +++ b/src/Parsers/ASTExpressionList.cpp @@ -17,16 +17,38 @@ void ASTExpressionList::formatImpl(const FormatSettings & settings, FormatState if (frame.expression_list_prepend_whitespace) settings.ostr << ' '; - for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it) + if (frame.need_parens) { - if (it != children.begin()) + settings.ostr << "("; + for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it) { - if (separator) - settings.ostr << separator; - settings.ostr << ' '; - } + if (it != children.begin()) + { + if (separator) + settings.ostr << separator; + settings.ostr << ' '; + } - (*it)->formatImpl(settings, state, frame); + settings.ostr << "("; + FormatStateStacked frame_nested = frame; + frame_nested.need_parens = false; + (*it)->formatImpl(settings, state, frame_nested); + settings.ostr << ")"; + } + settings.ostr << ")"; + } + else + { + for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it) + { + if (it != children.begin()) + { + if (separator) + settings.ostr << separator; + settings.ostr << ' '; + } + (*it)->formatImpl(settings, state, frame); + } } } @@ -41,20 +63,53 @@ void ASTExpressionList::formatImplMultiline(const FormatSettings & settings, For } ++frame.indent; - for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it) + + if (frame.need_parens) { - if (it != children.begin()) + for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it) { - if (separator) - settings.ostr << separator; + if (it != children.begin()) + { + if (separator) + settings.ostr << separator; + } + + if (children.size() > 1 || frame.expression_list_always_start_on_new_line) + settings.ostr << indent_str; + + if (it == children.begin()) + { + settings.ostr << "("; + } + + FormatStateStacked frame_nested = frame; + frame_nested.expression_list_always_start_on_new_line = false; + frame_nested.expression_list_prepend_whitespace = false; + frame_nested.need_parens = false; + settings.ostr << "("; + (*it)->formatImpl(settings, state, frame_nested); + settings.ostr << ")"; } + settings.ostr << ")"; + } + else + { + for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it) + { + if (it != children.begin()) + { + if (separator) + settings.ostr << separator; + } - if (children.size() > 1 || frame.expression_list_always_start_on_new_line) - settings.ostr << indent_str; + if (children.size() > 1 || frame.expression_list_always_start_on_new_line) + settings.ostr << indent_str; - FormatStateStacked frame_nested = frame; - frame_nested.expression_list_always_start_on_new_line = false; - (*it)->formatImpl(settings, state, frame_nested); + FormatStateStacked frame_nested = frame; + frame_nested.expression_list_always_start_on_new_line = false; + + (*it)->formatImpl(settings, state, frame_nested); + } } } diff --git a/src/Parsers/ASTSelectQuery.cpp b/src/Parsers/ASTSelectQuery.cpp index f07a4a328e2..3a4f8d79662 100644 --- a/src/Parsers/ASTSelectQuery.cpp +++ b/src/Parsers/ASTSelectQuery.cpp @@ -114,9 +114,12 @@ void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, F if (groupBy()) { s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "GROUP BY" << (s.hilite ? hilite_none : ""); - s.one_line + if (!group_by_with_grouping_sets) + { + s.one_line ? groupBy()->formatImpl(s, state, frame) : groupBy()->as().formatImplMultiline(s, state, frame); + } } if (group_by_with_rollup) @@ -126,7 +129,15 @@ void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, F s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << (s.one_line ? "" : " ") << "WITH CUBE" << (s.hilite ? hilite_none : ""); if (group_by_with_grouping_sets) - s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << (s.one_line ? "" : " ") << "WITH GROUPING SETS" << (s.hilite ? hilite_none : ""); + { + bool tmp_need_parens = frame.need_parens; + frame.need_parens = true; + s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << (s.one_line ? "" : " ") << "GROUPING SETS" << (s.hilite ? hilite_none : ""); + s.one_line + ? groupBy()->formatImpl(s, state, frame) + : groupBy()->as().formatImplMultiline(s, state, frame); + frame.need_parens = tmp_need_parens; + } if (group_by_with_totals) s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << (s.one_line ? "" : " ") << "WITH TOTALS" << (s.hilite ? hilite_none : ""); diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index 5a0fb03bd61..109daca98e4 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -765,7 +765,7 @@ bool ParserGroupingSetsExpressionListElements::parseImpl(Pos & pos, ASTPtr & nod ParserToken s_close(TokenType::ClosingRoundBracket); ParserExpressionWithOptionalAlias p_expression(false); ParserList p_command(std::make_unique(false), - std::make_unique(TokenType::Comma), false); + std::make_unique(TokenType::Comma), true); do { @@ -778,6 +778,9 @@ bool ParserGroupingSetsExpressionListElements::parseImpl(Pos & pos, ASTPtr & nod { return false; } + auto list = std::make_shared(','); + list->children.push_back(command); + command = std::move(list); } else { diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.reference b/tests/queries/0_stateless/01883_with_grouping_sets.reference index 258fa6f5abd..82fa6f9e57d 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.reference +++ b/tests/queries/0_stateless/01883_with_grouping_sets.reference @@ -1,3 +1,5 @@ + + 1 0 1 4500 1 0 3 4700 1 0 5 4900 @@ -40,6 +42,28 @@ 2 4 0 0 4800 2 5 0 0 5400 +0 0 0 49500 +0 0 1 4500 +0 0 2 4600 +0 0 3 4700 +0 0 4 4800 +0 0 5 4900 +0 0 6 5000 +0 0 7 5100 +0 0 8 5200 +0 0 9 5300 +0 0 10 5400 +1 1 0 4500 +1 2 0 5100 +1 3 0 4700 +1 4 0 5300 +1 5 0 4900 +2 1 0 5000 +2 2 0 4600 +2 3 0 5200 +2 4 0 4800 +2 5 0 5400 + 1 0 24500 1 1 4500 1 3 4700 diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.sql b/tests/queries/0_stateless/01883_with_grouping_sets.sql index fffe4314706..92a56be0a3f 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.sql +++ b/tests/queries/0_stateless/01883_with_grouping_sets.sql @@ -2,6 +2,8 @@ DROP TABLE IF EXISTS grouping_sets; CREATE TABLE grouping_sets(fact_1_id Int32, fact_2_id Int32, fact_3_id Int32, fact_4_id Int32, sales_value Int32) ENGINE = Memory; +SELECT fact_1_id, fact_3_id, sum(sales_value), count() from grouping_sets GROUP BY GROUPING SETS(fact_1_id, fact_3_id) ORDER BY fact_1_id, fact_3_id; + INSERT INTO grouping_sets SELECT number % 2 + 1 AS fact_1_id, @@ -12,19 +14,23 @@ SELECT FROM system.numbers limit 1000; SELECT fact_1_id, fact_2_id, fact_3_id, SUM(sales_value) AS sales_value from grouping_sets -GROUP BY GROUPING SETS((fact_1_id, fact_2_id), (fact_1_id, fact_3_id)) +GROUP BY GROUPING SETS ((fact_1_id, fact_2_id), (fact_1_id, fact_3_id)) ORDER BY fact_1_id, fact_2_id, fact_3_id; SELECT fact_1_id, fact_2_id, fact_3_id, fact_4_id, SUM(sales_value) AS sales_value from grouping_sets -GROUP BY GROUPING SETS((fact_1_id, fact_2_id), (fact_3_id, fact_4_id)) +GROUP BY GROUPING SETS ((fact_1_id, fact_2_id), (fact_3_id, fact_4_id)) ORDER BY fact_1_id, fact_2_id, fact_3_id, fact_4_id; +SELECT fact_1_id, fact_2_id, fact_3_id, SUM(sales_value) AS sales_value from grouping_sets +GROUP BY GROUPING SETS ((fact_1_id, fact_2_id), (fact_3_id), ()) +ORDER BY fact_1_id, fact_2_id, fact_3_id; + SELECT fact_1_id, fact_3_id, SUM(sales_value) AS sales_value FROM grouping_sets -GROUP BY grouping sets((fact_1_id), (fact_1_id, fact_3_id)) WITH TOTALS +GROUP BY grouping sets ((fact_1_id), (fact_1_id, fact_3_id)) WITH TOTALS ORDER BY fact_1_id, fact_3_id; SELECT @@ -32,11 +38,7 @@ SELECT fact_3_id, SUM(sales_value) AS sales_value FROM grouping_sets -GROUP BY grouping sets(fact_1_id, (fact_1_id, fact_3_id)) WITH TOTALS +GROUP BY grouping sets (fact_1_id, (fact_1_id, fact_3_id)) WITH TOTALS ORDER BY fact_1_id, fact_3_id; -truncate grouping_sets; - -SELECT fact_1_id, fact_3_id, sum(sales_value), count() from grouping_sets GROUP BY GROUPING SETS(fact_1_id, fact_3_id) ORDER BY fact_1_id, fact_3_id; - DROP TABLE grouping_sets; From b81a958c776e4f75a66d69a2f856965d89e463fc Mon Sep 17 00:00:00 2001 From: fanzhou Date: Mon, 13 Sep 2021 15:47:28 +0800 Subject: [PATCH 029/171] update test reference --- .../0_stateless/01883_with_grouping_sets.reference | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.reference b/tests/queries/0_stateless/01883_with_grouping_sets.reference index 82fa6f9e57d..7a69064f567 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.reference +++ b/tests/queries/0_stateless/01883_with_grouping_sets.reference @@ -1,5 +1,3 @@ - - 1 0 1 4500 1 0 3 4700 1 0 5 4900 @@ -20,7 +18,6 @@ 2 3 0 5200 2 4 0 4800 2 5 0 5400 - 0 0 1 1 4500 0 0 2 2 4600 0 0 3 3 4700 @@ -41,7 +38,6 @@ 2 3 0 0 5200 2 4 0 0 4800 2 5 0 0 5400 - 0 0 0 49500 0 0 1 4500 0 0 2 4600 @@ -63,7 +59,6 @@ 2 3 0 5200 2 4 0 4800 2 5 0 5400 - 1 0 24500 1 1 4500 1 3 4700 @@ -76,8 +71,8 @@ 2 6 5000 2 8 5200 2 10 5400 -0 0 49500 +0 0 49500 1 0 24500 1 1 4500 1 3 4700 @@ -90,4 +85,5 @@ 2 6 5000 2 8 5200 2 10 5400 -0 0 49500 + +0 0 49500 \ No newline at end of file From 668a70829435e1776fc7a13997ef5329c54245b8 Mon Sep 17 00:00:00 2001 From: fanzhou Date: Mon, 13 Sep 2021 16:31:46 +0800 Subject: [PATCH 030/171] update test reference --- tests/queries/0_stateless/01883_with_grouping_sets.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.reference b/tests/queries/0_stateless/01883_with_grouping_sets.reference index 7a69064f567..e52f72b84b9 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.reference +++ b/tests/queries/0_stateless/01883_with_grouping_sets.reference @@ -86,4 +86,4 @@ 2 8 5200 2 10 5400 -0 0 49500 \ No newline at end of file +0 0 49500 From e957054409c5cfd742393478d6c383053a52d108 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 7 Dec 2021 16:44:09 +0300 Subject: [PATCH 031/171] fix GROUPING SETS formating --- src/Parsers/ASTExpressionList.cpp | 94 +++++++++---------------------- src/Parsers/ASTSelectQuery.cpp | 7 ++- src/Parsers/IAST.h | 1 + 3 files changed, 33 insertions(+), 69 deletions(-) diff --git a/src/Parsers/ASTExpressionList.cpp b/src/Parsers/ASTExpressionList.cpp index 453624aa794..2590c6b2941 100644 --- a/src/Parsers/ASTExpressionList.cpp +++ b/src/Parsers/ASTExpressionList.cpp @@ -17,38 +17,24 @@ void ASTExpressionList::formatImpl(const FormatSettings & settings, FormatState if (frame.expression_list_prepend_whitespace) settings.ostr << ' '; - if (frame.need_parens) + for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it) { - settings.ostr << "("; - for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it) + if (it != children.begin()) { - if (it != children.begin()) - { - if (separator) - settings.ostr << separator; - settings.ostr << ' '; - } + if (separator) + settings.ostr << separator; + settings.ostr << ' '; + } + if (frame.surround_each_list_element_with_parens) settings.ostr << "("; - FormatStateStacked frame_nested = frame; - frame_nested.need_parens = false; - (*it)->formatImpl(settings, state, frame_nested); + + FormatStateStacked frame_nested = frame; + frame_nested.surround_each_list_element_with_parens = false; + (*it)->formatImpl(settings, state, frame_nested); + + if (frame.surround_each_list_element_with_parens) settings.ostr << ")"; - } - settings.ostr << ")"; - } - else - { - for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it) - { - if (it != children.begin()) - { - if (separator) - settings.ostr << separator; - settings.ostr << ' '; - } - (*it)->formatImpl(settings, state, frame); - } } } @@ -64,52 +50,28 @@ void ASTExpressionList::formatImplMultiline(const FormatSettings & settings, For ++frame.indent; - if (frame.need_parens) + for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it) { - for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it) + if (it != children.begin()) { - if (it != children.begin()) - { - if (separator) - settings.ostr << separator; - } + if (separator) + settings.ostr << separator; + } - if (children.size() > 1 || frame.expression_list_always_start_on_new_line) - settings.ostr << indent_str; + if (children.size() > 1 || frame.expression_list_always_start_on_new_line) + settings.ostr << indent_str; - if (it == children.begin()) - { - settings.ostr << "("; - } + FormatStateStacked frame_nested = frame; + frame_nested.expression_list_always_start_on_new_line = false; + frame_nested.surround_each_list_element_with_parens = false; - FormatStateStacked frame_nested = frame; - frame_nested.expression_list_always_start_on_new_line = false; - frame_nested.expression_list_prepend_whitespace = false; - frame_nested.need_parens = false; + if (frame.surround_each_list_element_with_parens) settings.ostr << "("; - (*it)->formatImpl(settings, state, frame_nested); + + (*it)->formatImpl(settings, state, frame_nested); + + if (frame.surround_each_list_element_with_parens) settings.ostr << ")"; - } - settings.ostr << ")"; - } - else - { - for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it) - { - if (it != children.begin()) - { - if (separator) - settings.ostr << separator; - } - - if (children.size() > 1 || frame.expression_list_always_start_on_new_line) - settings.ostr << indent_str; - - FormatStateStacked frame_nested = frame; - frame_nested.expression_list_always_start_on_new_line = false; - - (*it)->formatImpl(settings, state, frame_nested); - } } } diff --git a/src/Parsers/ASTSelectQuery.cpp b/src/Parsers/ASTSelectQuery.cpp index 3a4f8d79662..5a96e1d4df9 100644 --- a/src/Parsers/ASTSelectQuery.cpp +++ b/src/Parsers/ASTSelectQuery.cpp @@ -130,13 +130,14 @@ void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, F if (group_by_with_grouping_sets) { - bool tmp_need_parens = frame.need_parens; - frame.need_parens = true; + frame.surround_each_list_element_with_parens = true; s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << (s.one_line ? "" : " ") << "GROUPING SETS" << (s.hilite ? hilite_none : ""); + s.ostr << " ("; s.one_line ? groupBy()->formatImpl(s, state, frame) : groupBy()->as().formatImplMultiline(s, state, frame); - frame.need_parens = tmp_need_parens; + s.ostr << ")"; + frame.surround_each_list_element_with_parens = false; } if (group_by_with_totals) diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index ed3c54624ba..93498562d23 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -224,6 +224,7 @@ public: bool need_parens = false; bool expression_list_always_start_on_new_line = false; /// Line feed and indent before expression list even if it's of single element. bool expression_list_prepend_whitespace = false; /// Prepend whitespace (if it is required) + bool surround_each_list_element_with_parens = false; const IAST * current_select = nullptr; }; From 29d98cf00d658e15bbe3b5c343fcd6ffd2ce8ce1 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 7 Dec 2021 16:47:42 +0300 Subject: [PATCH 032/171] remove ya.make --- src/Processors/ya.make | 158 ----------------------------------------- 1 file changed, 158 deletions(-) delete mode 100644 src/Processors/ya.make diff --git a/src/Processors/ya.make b/src/Processors/ya.make deleted file mode 100644 index b2f8b9ba7c2..00000000000 --- a/src/Processors/ya.make +++ /dev/null @@ -1,158 +0,0 @@ -# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it. -LIBRARY() - -PEERDIR( - clickhouse/src/Common - contrib/libs/msgpack - contrib/libs/protobuf -) - -CFLAGS(-g0) - -SRCS( - Chunk.cpp - ConcatProcessor.cpp - DelayedPortsProcessor.cpp - Executors/ExecutingGraph.cpp - Executors/PipelineExecutingBlockInputStream.cpp - Executors/PipelineExecutor.cpp - Executors/PullingAsyncPipelineExecutor.cpp - Executors/PullingPipelineExecutor.cpp - ForkProcessor.cpp - Formats/IInputFormat.cpp - Formats/Impl/BinaryRowInputFormat.cpp - Formats/Impl/BinaryRowOutputFormat.cpp - Formats/Impl/ConstantExpressionTemplate.cpp - Formats/Impl/CSVRowInputFormat.cpp - Formats/Impl/CSVRowOutputFormat.cpp - Formats/Impl/JSONAsStringRowInputFormat.cpp - Formats/Impl/JSONCompactEachRowRowInputFormat.cpp - Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp - Formats/Impl/JSONCompactRowOutputFormat.cpp - Formats/Impl/JSONEachRowRowInputFormat.cpp - Formats/Impl/JSONEachRowRowOutputFormat.cpp - Formats/Impl/JSONEachRowWithProgressRowOutputFormat.cpp - Formats/Impl/JSONRowOutputFormat.cpp - Formats/Impl/LineAsStringRowInputFormat.cpp - Formats/Impl/MarkdownRowOutputFormat.cpp - Formats/Impl/MsgPackRowInputFormat.cpp - Formats/Impl/MsgPackRowOutputFormat.cpp - Formats/Impl/MySQLOutputFormat.cpp - Formats/Impl/NativeFormat.cpp - Formats/Impl/NullFormat.cpp - Formats/Impl/ODBCDriver2BlockOutputFormat.cpp - Formats/Impl/PostgreSQLOutputFormat.cpp - Formats/Impl/PrettyBlockOutputFormat.cpp - Formats/Impl/PrettyCompactBlockOutputFormat.cpp - Formats/Impl/PrettySpaceBlockOutputFormat.cpp - Formats/Impl/ProtobufRowInputFormat.cpp - Formats/Impl/ProtobufRowOutputFormat.cpp - Formats/Impl/RawBLOBRowInputFormat.cpp - Formats/Impl/RawBLOBRowOutputFormat.cpp - Formats/Impl/RegexpRowInputFormat.cpp - Formats/Impl/TabSeparatedRowInputFormat.cpp - Formats/Impl/TabSeparatedRowOutputFormat.cpp - Formats/Impl/TemplateBlockOutputFormat.cpp - Formats/Impl/TemplateRowInputFormat.cpp - Formats/Impl/TSKVRowInputFormat.cpp - Formats/Impl/TSKVRowOutputFormat.cpp - Formats/Impl/ValuesBlockInputFormat.cpp - Formats/Impl/ValuesRowOutputFormat.cpp - Formats/Impl/VerticalRowOutputFormat.cpp - Formats/Impl/XMLRowOutputFormat.cpp - Formats/IOutputFormat.cpp - Formats/IRowInputFormat.cpp - Formats/IRowOutputFormat.cpp - Formats/LazyOutputFormat.cpp - Formats/OutputStreamToOutputFormat.cpp - Formats/PullingOutputFormat.cpp - Formats/RowInputFormatWithDiagnosticInfo.cpp - IAccumulatingTransform.cpp - IInflatingTransform.cpp - IProcessor.cpp - ISimpleTransform.cpp - ISink.cpp - ISource.cpp - LimitTransform.cpp - Merges/Algorithms/AggregatingSortedAlgorithm.cpp - Merges/Algorithms/CollapsingSortedAlgorithm.cpp - Merges/Algorithms/GraphiteRollupSortedAlgorithm.cpp - Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.cpp - Merges/Algorithms/IMergingAlgorithmWithSharedChunks.cpp - Merges/Algorithms/MergingSortedAlgorithm.cpp - Merges/Algorithms/ReplacingSortedAlgorithm.cpp - Merges/Algorithms/SummingSortedAlgorithm.cpp - Merges/Algorithms/VersionedCollapsingAlgorithm.cpp - Merges/IMergingTransform.cpp - Merges/MergingSortedTransform.cpp - OffsetTransform.cpp - Pipe.cpp - Port.cpp - printPipeline.cpp - QueryPipeline.cpp - QueryPlan/AddingDelayedSourceStep.cpp - QueryPlan/AggregatingStep.cpp - QueryPlan/ArrayJoinStep.cpp - QueryPlan/ConvertingStep.cpp - QueryPlan/CreatingSetsStep.cpp - QueryPlan/CubeStep.cpp - QueryPlan/DistinctStep.cpp - QueryPlan/ExpressionStep.cpp - QueryPlan/ExtremesStep.cpp - QueryPlan/FillingStep.cpp - QueryPlan/FilterStep.cpp - QueryPlan/FinishSortingStep.cpp - QueryPlan/IQueryPlanStep.cpp - QueryPlan/ISourceStep.cpp - QueryPlan/ITransformingStep.cpp - QueryPlan/LimitByStep.cpp - QueryPlan/LimitStep.cpp - QueryPlan/MergeSortingStep.cpp - QueryPlan/MergingAggregatedStep.cpp - QueryPlan/MergingSortedStep.cpp - QueryPlan/OffsetStep.cpp - QueryPlan/PartialSortingStep.cpp - QueryPlan/QueryPlan.cpp - QueryPlan/ReadFromPreparedSource.cpp - QueryPlan/ReadFromStorageStep.cpp - QueryPlan/ReadNothingStep.cpp - QueryPlan/RollupStep.cpp - QueryPlan/TotalsHavingStep.cpp - QueryPlan/UnionStep.cpp - ResizeProcessor.cpp - Sources/DelayedSource.cpp - Sources/RemoteSource.cpp - Sources/SinkToOutputStream.cpp - Sources/SourceFromInputStream.cpp - Sources/SourceWithProgress.cpp - Transforms/AddingMissedTransform.cpp - Transforms/AddingSelectorTransform.cpp - Transforms/AggregatingInOrderTransform.cpp - Transforms/AggregatingTransform.cpp - Transforms/ArrayJoinTransform.cpp - Transforms/ConvertingTransform.cpp - Transforms/CopyTransform.cpp - Transforms/CreatingSetsTransform.cpp - Transforms/CubeTransform.cpp - Transforms/DistinctTransform.cpp - Transforms/ExpressionTransform.cpp - Transforms/ExtremesTransform.cpp - Transforms/FillingTransform.cpp - Transforms/FilterTransform.cpp - Transforms/FinishSortingTransform.cpp - Transforms/JoiningTransform.cpp - Transforms/LimitByTransform.cpp - Transforms/LimitsCheckingTransform.cpp - Transforms/MaterializingTransform.cpp - Transforms/MergeSortingTransform.cpp - Transforms/MergingAggregatedMemoryEfficientTransform.cpp - Transforms/MergingAggregatedTransform.cpp - Transforms/PartialSortingTransform.cpp - Transforms/ReverseTransform.cpp - Transforms/RollupTransform.cpp - Transforms/SortingTransform.cpp - Transforms/TotalsHavingTransform.cpp - -) - -END() From 1095814bbf5027c8f4f14923a472d2688ab6209e Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 14 Dec 2021 22:15:14 +0300 Subject: [PATCH 033/171] Cleanup code --- src/Interpreters/Aggregator.h | 47 ++++++++++++------- src/Interpreters/ExpressionAnalyzer.cpp | 2 +- src/Interpreters/InterpreterSelectQuery.cpp | 7 --- src/Interpreters/TreeOptimizer.cpp | 3 -- .../Transforms/AggregatingTransform.cpp | 1 - .../Transforms/GroupingSetsTransform.h | 4 +- src/QueryPipeline/Pipe.cpp | 12 +---- 7 files changed, 34 insertions(+), 42 deletions(-) diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index e8635c4e31a..587659c7eeb 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -943,30 +943,41 @@ public: Params( const Block & src_header_, const ColumnNumbers & keys_, - const ColumnNumbersList & keys_vector_, const AggregateDescriptions & aggregates_, - bool overflow_row_, size_t max_rows_to_group_by_, OverflowMode group_by_overflow_mode_, - size_t group_by_two_level_threshold_, size_t group_by_two_level_threshold_bytes_, + const ColumnNumbersList & keys_vector_, + const AggregateDescriptions & aggregates_, + bool overflow_row_, + size_t max_rows_to_group_by_, + OverflowMode group_by_overflow_mode_, + size_t group_by_two_level_threshold_, + size_t group_by_two_level_threshold_bytes_, size_t max_bytes_before_external_group_by_, bool empty_result_for_aggregation_by_empty_set_, - VolumePtr tmp_volume_, size_t max_threads_, + VolumePtr tmp_volume_, + size_t max_threads_, size_t min_free_disk_space_, bool compile_aggregate_expressions_, size_t min_count_to_compile_aggregate_expression_, const Block & intermediate_header_ = {}) - : src_header(src_header_), - intermediate_header(intermediate_header_), - keys(keys_), - keys_vector(keys_vector_), aggregates(aggregates_), keys_size(keys.size()), aggregates_size(aggregates.size()), - overflow_row(overflow_row_), max_rows_to_group_by(max_rows_to_group_by_), group_by_overflow_mode(group_by_overflow_mode_), - group_by_two_level_threshold(group_by_two_level_threshold_), group_by_two_level_threshold_bytes(group_by_two_level_threshold_bytes_), - max_bytes_before_external_group_by(max_bytes_before_external_group_by_), - empty_result_for_aggregation_by_empty_set(empty_result_for_aggregation_by_empty_set_), - tmp_volume(tmp_volume_), max_threads(max_threads_), - min_free_disk_space(min_free_disk_space_), - compile_aggregate_expressions(compile_aggregate_expressions_), - min_count_to_compile_aggregate_expression(min_count_to_compile_aggregate_expression_) - { - } + : src_header(src_header_) + , intermediate_header(intermediate_header_) + , keys(keys_) + , keys_vector(keys_vector_) + , aggregates(aggregates_) + , keys_size(keys.size()) + , aggregates_size(aggregates.size()) + , overflow_row(overflow_row_) + , max_rows_to_group_by(max_rows_to_group_by_) + , group_by_overflow_mode(group_by_overflow_mode_) + , group_by_two_level_threshold(group_by_two_level_threshold_) + , group_by_two_level_threshold_bytes(group_by_two_level_threshold_bytes_) + , max_bytes_before_external_group_by(max_bytes_before_external_group_by_) + , empty_result_for_aggregation_by_empty_set(empty_result_for_aggregation_by_empty_set_) + , tmp_volume(tmp_volume_) + , max_threads(max_threads_) + , min_free_disk_space(min_free_disk_space_) + , compile_aggregate_expressions(compile_aggregate_expressions_) + , min_count_to_compile_aggregate_expression(min_count_to_compile_aggregate_expression_) + {} /// Only parameters that matter during merge. Params(const Block & intermediate_header_, diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 5b207051029..665e28b138c 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1196,7 +1196,7 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain { for (const auto & ast : asts) { - for(const auto & ast_element : ast->children) + for (const auto & ast_element : ast->children) { step.addRequiredOutput(ast_element->getColumnName()); getRootActions(ast_element, only_types, step.actions()); diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index c04816cb794..74d7448fdc1 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -50,13 +50,6 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include #include #include #include diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index 4b5e093d61a..a61d41f78fb 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -84,9 +84,6 @@ void optimizeGroupBy(ASTSelectQuery * select_query, ContextPtr context) { const FunctionFactory & function_factory = FunctionFactory::instance(); -// if (select_query->group_by_with_grouping_sets) -// return; - if (!select_query->groupBy()) return; diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index f1e292a92cc..5b58530f3d5 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -501,7 +501,6 @@ void AggregatingTransform::work() Processors AggregatingTransform::expandPipeline() { - LOG_DEBUG(log, "in AggregatingTransform::expandPipeline"); auto & out = processors.back()->getOutputs().front(); inputs.emplace_back(out.getHeader(), this); connect(out, inputs.back()); diff --git a/src/Processors/Transforms/GroupingSetsTransform.h b/src/Processors/Transforms/GroupingSetsTransform.h index f9c688b2cc8..e9b20867ef4 100644 --- a/src/Processors/Transforms/GroupingSetsTransform.h +++ b/src/Processors/Transforms/GroupingSetsTransform.h @@ -17,8 +17,8 @@ protected: private: AggregatingTransformParamsPtr params; - ColumnNumbers keys; - ColumnNumbersList keys_vector; + const ColumnNumbers & keys; + const ColumnNumbersList & keys_vector; Chunks consumed_chunks; Chunk grouping_sets_chunk; diff --git a/src/QueryPipeline/Pipe.cpp b/src/QueryPipeline/Pipe.cpp index dd4a2cb6c77..6aa10b87328 100644 --- a/src/QueryPipeline/Pipe.cpp +++ b/src/QueryPipeline/Pipe.cpp @@ -463,18 +463,10 @@ void Pipe::addParallelTransforms(Processors transforms) std::to_string(transforms.size()) + " transforms were passed, " "but " + std::to_string(output_ports.size()) + " expected", ErrorCodes::LOGICAL_ERROR); - size_t next_output = 0; for (size_t i = 0; i < inputs.size(); ++i) - { - connect(*output_ports[next_output], *inputs[i]); - ++next_output; - } + connect(*output_ports[i], *inputs[i]); - output_ports.clear(); - output_ports.reserve(outputs.size()); - - for (auto * output : outputs) - output_ports.emplace_back(std::move(output)); + output_ports = std::move(outputs); /// do not check output formats because they are different in case of parallel aggregations LOG_DEBUG(log, "addParallelTransforms do not check format"); From 17681ecea45be4a0f28e535477f41d045d4611bd Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 14 Dec 2021 22:44:54 +0300 Subject: [PATCH 034/171] Remove unused code --- src/QueryPipeline/Pipe.cpp | 49 ---------------------- src/QueryPipeline/Pipe.h | 1 - src/QueryPipeline/QueryPipelineBuilder.cpp | 6 --- src/QueryPipeline/QueryPipelineBuilder.h | 2 - 4 files changed, 58 deletions(-) diff --git a/src/QueryPipeline/Pipe.cpp b/src/QueryPipeline/Pipe.cpp index 6aa10b87328..25c5f7c0781 100644 --- a/src/QueryPipeline/Pipe.cpp +++ b/src/QueryPipeline/Pipe.cpp @@ -432,55 +432,6 @@ void Pipe::addTransform(ProcessorPtr transform) addTransform(std::move(transform), static_cast(nullptr), static_cast(nullptr)); } -void Pipe::addParallelTransforms(Processors transforms) -{ - LOG_DEBUG(log, "Begin addParallelTransforms, have {} transforms", transforms.size()); - - if (output_ports.empty()) - throw Exception("Cannot add parallel transforms to empty Pipe.", ErrorCodes::LOGICAL_ERROR); - - std::vector inputs; - std::vector outputs; - for (const auto & transform : transforms) - { - auto & current_transform_inputs = transform->getInputs(); - if (current_transform_inputs.size() != 1) - throw Exception("Each parallel transform should have one input port", ErrorCodes::LOGICAL_ERROR); - - inputs.push_back(&(current_transform_inputs.front())); - - auto & current_transform_outputs = transform->getOutputs(); - if (current_transform_outputs.size() != 1) - throw Exception("Each parallel transform should have one output port", ErrorCodes::LOGICAL_ERROR); - - outputs.push_back(&(current_transform_outputs.front())); - LOG_DEBUG(log, "addParallelTransforms, added inputs and outputs for processor {}", transform->getName()); - LOG_DEBUG(log, "output structure: {}", transform->getOutputs().front().getHeader().dumpStructure()); - } - - if (inputs.size() != output_ports.size()) - throw Exception("Cannot add parallel transforms to Pipes because " + - std::to_string(transforms.size()) + " transforms were passed, " - "but " + std::to_string(output_ports.size()) + " expected", ErrorCodes::LOGICAL_ERROR); - - for (size_t i = 0; i < inputs.size(); ++i) - connect(*output_ports[i], *inputs[i]); - - output_ports = std::move(outputs); - - /// do not check output formats because they are different in case of parallel aggregations - LOG_DEBUG(log, "addParallelTransforms do not check format"); - - if (collected_processors) - collected_processors->insert(collected_processors->end(), transforms.begin(), transforms.end()); - - for (auto & transform_ptr : transforms) - processors.emplace_back(std::move(transform_ptr)); - - /// Should not change streams number, so maybe not need max_parallel_streams update - max_parallel_streams = std::max(max_parallel_streams, output_ports.size()); -} - void Pipe::addTransform(ProcessorPtr transform, OutputPort * totals, OutputPort * extremes) { if (output_ports.empty()) diff --git a/src/QueryPipeline/Pipe.h b/src/QueryPipeline/Pipe.h index 21e8fbfe039..01a5d8bb961 100644 --- a/src/QueryPipeline/Pipe.h +++ b/src/QueryPipeline/Pipe.h @@ -64,7 +64,6 @@ public: /// Output ports should have same headers. /// If totals or extremes are not empty, transform shouldn't change header. void addTransform(ProcessorPtr transform); - void addParallelTransforms(Processors transform); void addTransform(ProcessorPtr transform, OutputPort * totals, OutputPort * extremes); void addTransform(ProcessorPtr transform, InputPort * totals, InputPort * extremes); diff --git a/src/QueryPipeline/QueryPipelineBuilder.cpp b/src/QueryPipeline/QueryPipelineBuilder.cpp index 952900c772e..dba7c7cb8f7 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.cpp +++ b/src/QueryPipeline/QueryPipelineBuilder.cpp @@ -155,12 +155,6 @@ void QueryPipelineBuilder::transform(const Transformer & transformer) pipe.transform(transformer); } -void QueryPipelineBuilder::addParallelTransforms(Processors transforms) -{ - checkInitializedAndNotCompleted(); - pipe.addParallelTransforms(transforms); -} - void QueryPipelineBuilder::setSinks(const Pipe::ProcessorGetterWithStreamKind & getter) { checkInitializedAndNotCompleted(); diff --git a/src/QueryPipeline/QueryPipelineBuilder.h b/src/QueryPipeline/QueryPipelineBuilder.h index d920f8a7e81..9e198f45e98 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.h +++ b/src/QueryPipeline/QueryPipelineBuilder.h @@ -67,8 +67,6 @@ public: /// Transform pipeline in general way. void transform(const Transformer & transformer); - /// Add transforms and connect it to outputs streams - void addParallelTransforms(Processors transform); /// Add TotalsHavingTransform. Resize pipeline to single input. Adds totals port. void addTotalsHavingTransform(ProcessorPtr transform); /// Add transform which calculates extremes. This transform adds extremes port and doesn't change inputs number. From b27af9020292525cd2875327d4a70b1dac6b0af3 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 16 Dec 2021 15:39:25 +0300 Subject: [PATCH 035/171] Fix removing constant columns --- src/Interpreters/ExpressionAnalyzer.cpp | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 665e28b138c..f1d04e4b3a7 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -354,24 +354,29 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) for (ssize_t j = 0; j < ssize_t(group_elements_ast.size()); ++j) { + ssize_t group_size = group_elements_ast.size(); const auto & column_name = group_elements_ast[j]->getColumnName(); const auto * node = temp_actions->tryFindInIndex(column_name); if (!node) throw Exception("Unknown identifier (in GROUP BY): " + column_name, ErrorCodes::UNKNOWN_IDENTIFIER); - /// Constant expressions have non-null column pointer at this stage. - if (node->column && isColumnConst(*node->column)) + /// Only removes constant keys if it's an initiator or distributed_group_by_no_merge is enabled. + if (getContext()->getClientInfo().distributed_depth == 0 || settings.distributed_group_by_no_merge > 0) { - /// But don't remove last key column if no aggregate functions, otherwise aggregation will not work. - if (!aggregate_descriptions.empty() || size > 1) + /// Constant expressions have non-null column pointer at this stage. + if (node->column && isColumnConst(*node->column)) { - if (j + 1 < static_cast(size)) - group_asts[i] = std::move(group_asts.back()); + /// But don't remove last key column if no aggregate functions, otherwise aggregation will not work. + if (!aggregate_descriptions.empty() || group_size > 1) + { + if (j + 1 < static_cast(group_size)) + group_elements_ast[j] = std::move(group_elements_ast.back()); - group_asts.pop_back(); + group_elements_ast.pop_back(); - --j; - continue; + --j; + continue; + } } } From 9ae9048184615835bb27c3ebd1a9b969abec46a9 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 14 Jan 2022 13:18:05 +0000 Subject: [PATCH 036/171] Fix crash in GROUPING SETS --- src/Interpreters/ExpressionAnalyzer.cpp | 8 +- src/Interpreters/InterpreterSelectQuery.cpp | 7 +- .../01883_grouping_sets_crash.reference | 156 ++++++++++++++++++ .../0_stateless/01883_grouping_sets_crash.sql | 64 +++++++ 4 files changed, 231 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/01883_grouping_sets_crash.reference create mode 100644 tests/queries/0_stateless/01883_grouping_sets_crash.sql diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index f1d04e4b3a7..48dec020b1e 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -340,8 +340,6 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) if (getContext()->getSettingsRef().enable_positional_arguments) replaceForPositionalArguments(group_asts[i], select_query, ASTSelectQuery::Expression::GROUP_BY); - getRootActionsNoMakeSet(group_asts[i], true, temp_actions, false); - if (select_query->group_by_with_grouping_sets) { ASTs group_elements_ast; @@ -354,6 +352,8 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) for (ssize_t j = 0; j < ssize_t(group_elements_ast.size()); ++j) { + getRootActionsNoMakeSet(group_elements_ast[j], true, temp_actions, false); + ssize_t group_size = group_elements_ast.size(); const auto & column_name = group_elements_ast[j]->getColumnName(); const auto * node = temp_actions->tryFindInIndex(column_name); @@ -366,6 +366,8 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) /// Constant expressions have non-null column pointer at this stage. if (node->column && isColumnConst(*node->column)) { + select_query->group_by_with_constant_keys = true; + /// But don't remove last key column if no aggregate functions, otherwise aggregation will not work. if (!aggregate_descriptions.empty() || group_size > 1) { @@ -399,6 +401,8 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) } else { + getRootActionsNoMakeSet(group_asts[i], true, temp_actions, false); + const auto & column_name = group_asts[i]->getColumnName(); const auto * node = temp_actions->tryFindInIndex(column_name); if (!node) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 74d7448fdc1..16eb2fadf27 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2063,12 +2063,15 @@ void InterpreterSelectQuery::initAggregatorParams( for (const auto & key : aggregation_keys) { size_t key_name_pos = current_data_stream_header.getPositionByName(key.name); - keys_set.insert(key_name_pos); + if (!keys_set.contains(key_name_pos)) + { + keys_set.insert(key_name_pos); + all_keys.push_back(key_name_pos); + } keys.push_back(key_name_pos); } keys_vector.push_back(keys); } - all_keys.assign(keys_set.begin(), keys_set.end()); params_ptr = std::make_unique( current_data_stream_header, diff --git a/tests/queries/0_stateless/01883_grouping_sets_crash.reference b/tests/queries/0_stateless/01883_grouping_sets_crash.reference new file mode 100644 index 00000000000..e88d4543b51 --- /dev/null +++ b/tests/queries/0_stateless/01883_grouping_sets_crash.reference @@ -0,0 +1,156 @@ +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +SECOND QUERY: +0 0 +0 0 +0 0 +0 0 +0 0 +0 0 +0 0 +0 0 +0 0 +0 0 +0 0 +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +THIRD QUERY: +\N 1 0 0 +\N 2 0 0 +\N 3 0 0 +\N 4 0 0 +\N 5 0 0 +\N 1 0 0 +\N 2 0 0 +\N 3 0 0 +\N 4 0 0 +\N 5 0 0 +\N 0 10 10 +\N 0 9 9 +\N 0 8 8 +\N 0 7 7 +\N 0 6 6 +\N 0 5 5 +\N 0 4 4 +\N 0 3 3 +\N 0 2 2 +\N 0 1 1 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 +\N 0 0 0 diff --git a/tests/queries/0_stateless/01883_grouping_sets_crash.sql b/tests/queries/0_stateless/01883_grouping_sets_crash.sql new file mode 100644 index 00000000000..bfa45cfaff1 --- /dev/null +++ b/tests/queries/0_stateless/01883_grouping_sets_crash.sql @@ -0,0 +1,64 @@ +DROP TABLE IF EXISTS grouping_sets; + +CREATE TABLE grouping_sets(fact_1_id Int32, fact_2_id Int32, fact_3_id Int32, fact_4_id Int32, sales_value Int32) ENGINE = Memory; + +INSERT INTO grouping_sets +SELECT + number % 2 + 1 AS fact_1_id, + number % 5 + 1 AS fact_2_id, + number % 10 + 1 AS fact_3_id, + number % 10 + 1 AS fact_4_id, + number % 100 AS sales_value +FROM system.numbers limit 1000; + +SELECT + fact_3_id, + fact_4_id +FROM grouping_sets +GROUP BY + GROUPING SETS ( + ('wo\0ldworldwo\0ldworld'), + (fact_3_id, fact_4_id)) +ORDER BY + fact_3_id, fact_4_id; + +SELECT 'SECOND QUERY:'; + +SELECT + fact_3_id, + fact_4_id +FROM grouping_sets +GROUP BY + GROUPING SETS ( + (fact_1_id, fact_2_id), + ((-9223372036854775808, NULL, (tuple(1.), (tuple(1.), 1048576), 65535))), + ((tuple(3.4028234663852886e38), (tuple(1024), -2147483647), NULL)), + (fact_3_id, fact_4_id)) +ORDER BY + (NULL, ('256', (tuple(NULL), NULL), NULL, NULL), NULL) ASC, + fact_1_id DESC NULLS FIRST, + fact_2_id DESC NULLS FIRST, + fact_4_id ASC; + +SELECT 'THIRD QUERY:'; + +SELECT + extractAllGroups(NULL, 'worldworldworldwo\0ldworldworldworldwo\0ld'), + fact_2_id, + fact_3_id, + fact_4_id +FROM grouping_sets +GROUP BY + GROUPING SETS ( + (sales_value), + (fact_1_id, fact_2_id), + ('wo\0ldworldwo\0ldworld'), + (fact_3_id, fact_4_id)) +ORDER BY + fact_1_id DESC NULLS LAST, + fact_1_id DESC NULLS FIRST, + fact_2_id ASC, + fact_3_id DESC NULLS FIRST, + fact_4_id ASC; + +DROP TABLE IF EXISTS grouping_sets; \ No newline at end of file From 91bc253f864f22d5728a2a2287a799ffa4b7c920 Mon Sep 17 00:00:00 2001 From: qieqieplus Date: Tue, 22 Mar 2022 15:04:25 +0800 Subject: [PATCH 037/171] impl distance functions for array --- .gitmodules | 3 + contrib/CMakeLists.txt | 1 + contrib/eigen | 1 + contrib/eigen-cmake/CMakeLists.txt | 5 + src/Functions/array/CMakeLists.txt | 2 +- src/Functions/array/arrayDistance.cpp | 241 ++++++++++++++++++ .../array/registerFunctionsArray.cpp | 2 + 7 files changed, 254 insertions(+), 1 deletion(-) create mode 160000 contrib/eigen create mode 100644 contrib/eigen-cmake/CMakeLists.txt create mode 100644 src/Functions/array/arrayDistance.cpp diff --git a/.gitmodules b/.gitmodules index 6c9e66f9cbc..84507a0935b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -262,3 +262,6 @@ [submodule "contrib/minizip-ng"] path = contrib/minizip-ng url = https://github.com/zlib-ng/minizip-ng +[submodule "contrib/eigen"] + path = contrib/eigen + url = https://gitlab.com/libeigen/eigen.git diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 9cf307c473e..33afd901e1d 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -147,6 +147,7 @@ endif() add_contrib (sqlite-cmake sqlite-amalgamation) add_contrib (s2geometry-cmake s2geometry) +add_contrib (eigen-cmake eigen) # Put all targets defined here and in subdirectories under "contrib/" folders in GUI-based IDEs. # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear diff --git a/contrib/eigen b/contrib/eigen new file mode 160000 index 00000000000..51a0b4e2d2d --- /dev/null +++ b/contrib/eigen @@ -0,0 +1 @@ +Subproject commit 51a0b4e2d2d6f22ec88529b520e9d954615fec1d diff --git a/contrib/eigen-cmake/CMakeLists.txt b/contrib/eigen-cmake/CMakeLists.txt new file mode 100644 index 00000000000..661fd3056b6 --- /dev/null +++ b/contrib/eigen-cmake/CMakeLists.txt @@ -0,0 +1,5 @@ +set(EIGEN_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/eigen") + +add_library (_eigen INTERFACE) +target_include_directories (_eigen SYSTEM INTERFACE ${EIGEN_LIBRARY_DIR}) +add_library(ch_contrib::eigen ALIAS _eigen) diff --git a/src/Functions/array/CMakeLists.txt b/src/Functions/array/CMakeLists.txt index 9762674d6e9..c98f4430078 100644 --- a/src/Functions/array/CMakeLists.txt +++ b/src/Functions/array/CMakeLists.txt @@ -1,7 +1,7 @@ include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") add_headers_and_sources(clickhouse_functions_array .) add_library(clickhouse_functions_array ${clickhouse_functions_array_sources} ${clickhouse_functions_array_headers}) -target_link_libraries(clickhouse_functions_array PRIVATE dbms clickhouse_functions_gatherutils) +target_link_libraries(clickhouse_functions_array PRIVATE dbms clickhouse_functions_gatherutils ch_contrib::eigen) if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) target_compile_options(clickhouse_functions_array PRIVATE "-g0") diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp new file mode 100644 index 00000000000..93185b2ecb3 --- /dev/null +++ b/src/Functions/array/arrayDistance.cpp @@ -0,0 +1,241 @@ +#include +#include +#include +#include +#include + +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +template +struct LpDistance +{ + static inline String name = "L" + std::to_string(N); + template + static void compute(const Eigen::MatrixX & left, const Eigen::MatrixX & right, MutableColumnPtr & column) + { + auto & data = assert_cast(*column).getData(); + for (auto col : left.colwise()) + { + auto norms = (right.colwise() - col).colwise().template lpNorm(); + for (auto n : norms) + { + data.emplace_back(n); + } + } + } +}; + +struct L2Distance +{ + static inline String name = "L2"; + template + static void compute(const Eigen::MatrixX & left, const Eigen::MatrixX & right, MutableColumnPtr & column) + { + auto & data = assert_cast(*column).getData(); + for (auto col : left.colwise()) + { + auto norms = (right.colwise() - col).colwise().norm(); + for (auto n : norms) + { + data.emplace_back(n); + } + } + } +}; + +struct LinfDistance : LpDistance +{ + static inline String name = "Linf"; +}; + +struct CosineDistance +{ + static inline String name = "Cosine"; + template + static void compute(const Eigen::MatrixX & left, const Eigen::MatrixX & right, MutableColumnPtr & column) + { + auto & data = assert_cast(*column).getData(); + auto x = left.colwise().normalized(); + auto y = right.colwise().normalized(); + auto & prod = x.cwiseProduct(y).colwise().sum(); + for (auto p : prod) + { + data.emplace_back(1.0 - p); + } + } +}; + +template +class FunctionArrayDistance : public IFunction +{ +public: + static inline auto name = "array" + Label::name + "Distance"; + String getName() const override { return name; } + static FunctionPtr create(ContextPtr) { return std::make_shared>(); } + size_t getNumberOfArguments() const override { return 2; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + DataTypePtr first_nested_type = nullptr; + for (const auto & argument : arguments) + { + const DataTypeArray * array_type = checkAndGetDataType(argument.type.get()); + if (!array_type) + { + throw Exception( + "Arguments of function " + getName() + " must be array. Found " + argument.type->getName() + " instead.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + + const auto & nested_type = array_type->getNestedType(); + + if (!first_nested_type) + { + first_nested_type = nested_type; + } + else if (!nested_type->equals(*first_nested_type)) + { + throw Exception("Arguments of function " + getName() + " must have identical type.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + } + + switch (first_nested_type->getTypeId()) + { + case TypeIndex::UInt8: + case TypeIndex::UInt32: + case TypeIndex::Int8: + case TypeIndex::Int32: + case TypeIndex::Float32: + case TypeIndex::Float64: + break; + default: + throw Exception( + "Array nested type " + first_nested_type->getName() + ". Support: UInt8, UInt32, Int8, Int32, Float32, Float64.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + + //return std::make_shared(std::make_shared()); + return std::make_shared(); + } + + ColumnPtr + executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override + { + auto res_ptr = result_type->createColumn(); + //auto & res = assert_cast(*res_ptr).getData(); + const auto & first_col = arguments[0].column; + const auto & second_col = arguments[1].column; + const auto & nested_type = checkAndGetDataType(arguments[0].type.get())->getNestedType(); + switch (nested_type->getTypeId()) + { + case TypeIndex::UInt8: + executeDistance(first_col, second_col, res_ptr); + break; + case TypeIndex::UInt32: + executeDistance(first_col, second_col, res_ptr); + break; + case TypeIndex::Int8: + executeDistance(first_col, second_col, res_ptr); + break; + case TypeIndex::Int32: + executeDistance(first_col, second_col, res_ptr); + break; + case TypeIndex::Float32: + executeDistance(first_col, second_col, res_ptr); + break; + case TypeIndex::Float64: + executeDistance(first_col, second_col, res_ptr); + break; + default: + throw Exception( + "Array nested type " + nested_type->getName() + ". Support: UInt8, UInt32, Int8, Int32, Float32, Float64.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + + return res_ptr; + } + +private: + template + static void executeDistance(const ColumnPtr & first, const ColumnPtr & second, MutableColumnPtr & column) + { + Eigen::MatrixX mx, my; + columnToMatrix(first, mx); + columnToMatrix(second, my); + + if (mx.rows() && my.rows() && mx.rows() != my.rows()) + { + throw Exception( + "Arguments of function " + String(name) + " have different array sizes.", ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH); + } + Label::compute(mx, my, column); + } + + template + static void columnToMatrix(const ColumnPtr column, Eigen::MatrixX & mat) + { + ColumnPtr offsets; + const ColumnVector * vec; + + if (const ColumnArray * column_array = checkAndGetColumn(column.get())) + { + offsets = column_array->getOffsetsPtr(); + vec = checkAndGetColumn>(column_array->getData()); + } + else if (const ColumnArray * const_array = checkAndGetColumnConstData(column.get())) + { + offsets = const_array->getOffsetsPtr(); + vec = checkAndGetColumn>(const_array->getData()); + } + else + { + throw Exception("Argument of function " + String(name) + " must be array. ", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + + const auto & off = assert_cast(*offsets).getData(); + fillMatrix(mat, vec->getData(), off); + } + + template + static void fillMatrix(Eigen::MatrixX & mat, const PaddedPODArray & data, const ColumnArray::Offsets & offsets) + { + auto row_size = offsets.front(); + mat.resize(row_size, offsets.size()); + ColumnArray::Offset prev = 0, col = 0; + for (auto offset : offsets) + { + /* + if (offset - prev != row_size) { + throw Exception("Array must have equal sizes", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + */ + for (ColumnArray::Offset row = 0; row < offset - prev; ++row) + { + mat(row, col) = X(data[prev + row]); + } + ++col; + prev = offset; + } + } +}; + +void registerFunctionArrayDistance(FunctionFactory & factory) +{ + factory.registerFunction>(); + factory.registerFunction>>(); + factory.registerFunction>(); + factory.registerFunction>(); +} + +} diff --git a/src/Functions/array/registerFunctionsArray.cpp b/src/Functions/array/registerFunctionsArray.cpp index 3bb27cbadf9..f24a2023d40 100644 --- a/src/Functions/array/registerFunctionsArray.cpp +++ b/src/Functions/array/registerFunctionsArray.cpp @@ -37,6 +37,7 @@ void registerFunctionArrayAUC(FunctionFactory &); void registerFunctionArrayReduceInRanges(FunctionFactory &); void registerFunctionMapOp(FunctionFactory &); void registerFunctionMapPopulateSeries(FunctionFactory &); +void registerFunctionArrayDistance(FunctionFactory &); void registerFunctionsArray(FunctionFactory & factory) { @@ -75,6 +76,7 @@ void registerFunctionsArray(FunctionFactory & factory) registerFunctionArrayAUC(factory); registerFunctionMapOp(factory); registerFunctionMapPopulateSeries(factory); + registerFunctionArrayDistance(factory); } } From c4b5c4574049e9887104accb2d275f1a63ecc915 Mon Sep 17 00:00:00 2001 From: qieqieplus Date: Wed, 23 Mar 2022 18:17:55 +0800 Subject: [PATCH 038/171] refactor & add tests --- contrib/eigen-cmake/CMakeLists.txt | 15 + src/Functions/array/arrayDistance.cpp | 285 ++++++++---------- .../02280_array_distance.reference | 30 ++ .../0_stateless/02280_array_distance.sql | 29 ++ 4 files changed, 206 insertions(+), 153 deletions(-) create mode 100644 tests/queries/0_stateless/02280_array_distance.reference create mode 100644 tests/queries/0_stateless/02280_array_distance.sql diff --git a/contrib/eigen-cmake/CMakeLists.txt b/contrib/eigen-cmake/CMakeLists.txt index 661fd3056b6..58b41d122e0 100644 --- a/contrib/eigen-cmake/CMakeLists.txt +++ b/contrib/eigen-cmake/CMakeLists.txt @@ -1,5 +1,20 @@ set(EIGEN_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/eigen") add_library (_eigen INTERFACE) + +option (ENABLE_MKL "Build Eigen with Intel MKL" OFF) +if (ENABLE_MKL) + set(MKL_THREADING sequential) + set(MKL_INTERFACE lp64) + find_package(MKL REQUIRED) + if (MKL_FOUND) + message("MKL INCLUDE: ${MKL_INCLUDE}") + message("MKL LIBRARIES: ${MKL_LIBRARIES}") + target_compile_definitions(_eigen INTERFACE EIGEN_USE_MKL_ALL) + target_include_directories(_eigen PRIVATE ${MKL_INCLUDE}) + target_link_libraries(_eigen INTERFACE ${MKL_LIBRARIES}) + endif() +endif() + target_include_directories (_eigen SYSTEM INTERFACE ${EIGEN_LIBRARY_DIR}) add_library(ch_contrib::eigen ALIAS _eigen) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index 93185b2ecb3..8004552a840 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -10,7 +11,6 @@ namespace DB { namespace ErrorCodes { - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; extern const int ILLEGAL_TYPE_OF_ARGUMENT; } @@ -19,36 +19,12 @@ template struct LpDistance { static inline String name = "L" + std::to_string(N); - template - static void compute(const Eigen::MatrixX & left, const Eigen::MatrixX & right, MutableColumnPtr & column) + template + static void compute(const Eigen::MatrixBase & left, const Eigen::MatrixBase & right, PaddedPODArray & array) { - auto & data = assert_cast(*column).getData(); - for (auto col : left.colwise()) - { - auto norms = (right.colwise() - col).colwise().template lpNorm(); - for (auto n : norms) - { - data.emplace_back(n); - } - } - } -}; - -struct L2Distance -{ - static inline String name = "L2"; - template - static void compute(const Eigen::MatrixX & left, const Eigen::MatrixX & right, MutableColumnPtr & column) - { - auto & data = assert_cast(*column).getData(); - for (auto col : left.colwise()) - { - auto norms = (right.colwise() - col).colwise().norm(); - for (auto n : norms) - { - data.emplace_back(n); - } - } + auto & norms = (left - right).colwise().template lpNorm(); + for (auto n : norms) + array.emplace_back(n); } }; @@ -60,180 +36,183 @@ struct LinfDistance : LpDistance struct CosineDistance { static inline String name = "Cosine"; - template - static void compute(const Eigen::MatrixX & left, const Eigen::MatrixX & right, MutableColumnPtr & column) + template + static void compute(const Eigen::MatrixBase & left, const Eigen::MatrixBase & right, PaddedPODArray & array) { - auto & data = assert_cast(*column).getData(); - auto x = left.colwise().normalized(); - auto y = right.colwise().normalized(); - auto & prod = x.cwiseProduct(y).colwise().sum(); - for (auto p : prod) - { - data.emplace_back(1.0 - p); - } + // auto & nx = left.colwise().normalized().eval(); + // auto & ny = right.colwise().normalized().eval(); + // auto & dist = 1.0 - x.cwiseProduct(y).colwise().sum().array(); + auto & prod = left.cwiseProduct(right).colwise().sum(); + auto & nx = left.colwise().norm(); + auto & ny = right.colwise().norm(); + auto & nm = nx.cwiseProduct(ny).cwiseInverse(); + auto & dist = 1.0 - prod.cwiseProduct(nm).array(); + for (auto d : dist) + array.emplace_back(d); } }; -template +template class FunctionArrayDistance : public IFunction { public: - static inline auto name = "array" + Label::name + "Distance"; + static inline auto name = "array" + Kernel::name + "Distance"; String getName() const override { return name; } - static FunctionPtr create(ContextPtr) { return std::make_shared>(); } + static FunctionPtr create(ContextPtr) { return std::make_shared>(); } size_t getNumberOfArguments() const override { return 2; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } bool useDefaultImplementationForConstants() const override { return true; } - DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & /*arguments*/) const override { - DataTypePtr first_nested_type = nullptr; - for (const auto & argument : arguments) - { - const DataTypeArray * array_type = checkAndGetDataType(argument.type.get()); - if (!array_type) - { - throw Exception( - "Arguments of function " + getName() + " must be array. Found " + argument.type->getName() + " instead.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - - const auto & nested_type = array_type->getNestedType(); - - if (!first_nested_type) - { - first_nested_type = nested_type; - } - else if (!nested_type->equals(*first_nested_type)) - { - throw Exception("Arguments of function " + getName() + " must have identical type.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - } - - switch (first_nested_type->getTypeId()) - { - case TypeIndex::UInt8: - case TypeIndex::UInt32: - case TypeIndex::Int8: - case TypeIndex::Int32: - case TypeIndex::Float32: - case TypeIndex::Float64: - break; - default: - throw Exception( - "Array nested type " + first_nested_type->getName() + ". Support: UInt8, UInt32, Int8, Int32, Float32, Float64.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - - //return std::make_shared(std::make_shared()); - return std::make_shared(); + return std::make_shared(); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override { - auto res_ptr = result_type->createColumn(); - //auto & res = assert_cast(*res_ptr).getData(); - const auto & first_col = arguments[0].column; - const auto & second_col = arguments[1].column; - const auto & nested_type = checkAndGetDataType(arguments[0].type.get())->getNestedType(); - switch (nested_type->getTypeId()) + const auto & left = arguments[0]; + const auto & right = arguments[1]; + + const auto & col_x = left.column->convertToFullColumnIfConst(); + const auto & col_y = right.column->convertToFullColumnIfConst(); + const auto * arr_x = checkAndGetColumn(col_x.get()); + const auto * arr_y = checkAndGetColumn(col_y.get()); + if (!arr_x || !arr_y) { - case TypeIndex::UInt8: - executeDistance(first_col, second_col, res_ptr); - break; - case TypeIndex::UInt32: - executeDistance(first_col, second_col, res_ptr); - break; - case TypeIndex::Int8: - executeDistance(first_col, second_col, res_ptr); - break; - case TypeIndex::Int32: - executeDistance(first_col, second_col, res_ptr); - break; - case TypeIndex::Float32: - executeDistance(first_col, second_col, res_ptr); - break; - case TypeIndex::Float64: - executeDistance(first_col, second_col, res_ptr); - break; - default: - throw Exception( - "Array nested type " + nested_type->getName() + ". Support: UInt8, UInt32, Int8, Int32, Float32, Float64.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception("Argument of function " + String(name) + " must be array. ", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } - return res_ptr; + const auto & type_x = checkAndGetDataType(left.type.get())->getNestedType(); + const auto & type_y = checkAndGetDataType(right.type.get())->getNestedType(); + + const auto & common_type = getLeastSupertype(DataTypes{type_x, type_y}); + switch (common_type->getTypeId()) + { + case TypeIndex::UInt8: + case TypeIndex::UInt16: + case TypeIndex::UInt32: + case TypeIndex::Int8: + case TypeIndex::Int16: + case TypeIndex::Int32: + case TypeIndex::Float32: + return executeWithType(*arr_x, *arr_y, type_x, type_y, result_type); + case TypeIndex::UInt64: + case TypeIndex::Int64: + case TypeIndex::Float64: + return executeWithType(*arr_x, *arr_y, type_x, type_y, result_type); + default: + throw Exception( + "Array nested type " + common_type->getName() + + ". Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } } private: - template - static void executeDistance(const ColumnPtr & first, const ColumnPtr & second, MutableColumnPtr & column) + template + static ColumnPtr executeWithType( + const ColumnArray & array_x, + const ColumnArray & array_y, + const DataTypePtr & type_x, + const DataTypePtr & type_y, + const DataTypePtr & result_type) { - Eigen::MatrixX mx, my; - columnToMatrix(first, mx); - columnToMatrix(second, my); + auto result = result_type->createColumn(); + auto & array = typeid_cast(*result).getData(); + + Eigen::MatrixX mx, my; + columnToMatrix(array_x, type_x, mx); + columnToMatrix(array_y, type_y, my); if (mx.rows() && my.rows() && mx.rows() != my.rows()) { throw Exception( "Arguments of function " + String(name) + " have different array sizes.", ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH); } - Label::compute(mx, my, column); + + Kernel::compute(mx, my, array); + return result; } - template - static void columnToMatrix(const ColumnPtr column, Eigen::MatrixX & mat) + template + static void columnToMatrix(const ColumnArray & array, const DataTypePtr & nested_type, Eigen::MatrixX & mat) { - ColumnPtr offsets; - const ColumnVector * vec; - - if (const ColumnArray * column_array = checkAndGetColumn(column.get())) + switch (nested_type->getTypeId()) { - offsets = column_array->getOffsetsPtr(); - vec = checkAndGetColumn>(column_array->getData()); + case TypeIndex::UInt8: + fillMatrix(mat, array); + break; + case TypeIndex::UInt16: + fillMatrix(mat, array); + break; + case TypeIndex::UInt32: + fillMatrix(mat, array); + break; + case TypeIndex::UInt64: + fillMatrix(mat, array); + break; + case TypeIndex::Int8: + fillMatrix(mat, array); + break; + case TypeIndex::Int16: + fillMatrix(mat, array); + break; + case TypeIndex::Int32: + fillMatrix(mat, array); + break; + case TypeIndex::Int64: + fillMatrix(mat, array); + break; + case TypeIndex::Float32: + fillMatrix(mat, array); + break; + case TypeIndex::Float64: + fillMatrix(mat, array); + break; + default: + throw Exception( + "Array nested type " + nested_type->getName() + + ". Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } - else if (const ColumnArray * const_array = checkAndGetColumnConstData(column.get())) - { - offsets = const_array->getOffsetsPtr(); - vec = checkAndGetColumn>(const_array->getData()); - } - else - { - throw Exception("Argument of function " + String(name) + " must be array. ", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - - const auto & off = assert_cast(*offsets).getData(); - fillMatrix(mat, vec->getData(), off); } - template - static void fillMatrix(Eigen::MatrixX & mat, const PaddedPODArray & data, const ColumnArray::Offsets & offsets) + // optimize for float/ double + template ::value>::type> + static void fillMatrix(Eigen::MatrixX & mat, const ColumnArray & array) { - auto row_size = offsets.front(); - mat.resize(row_size, offsets.size()); + const auto & vec = typeid_cast &>(array.getData()); + const auto & data = vec.getData(); + const auto & offsets = array.getOffsets(); + mat = Eigen::Map>(data.data(), offsets.front(), offsets.size()); + } + + template + static void fillMatrix(Eigen::MatrixX & mat, const ColumnArray & array) + { + const auto & vec = typeid_cast &>(array.getData()); + const auto & data = vec.getData(); + const auto & offsets = array.getOffsets(); + mat.resize(offsets.front(), offsets.size()); + ColumnArray::Offset prev = 0, col = 0; - for (auto offset : offsets) + for (auto off : offsets) { - /* - if (offset - prev != row_size) { - throw Exception("Array must have equal sizes", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - */ - for (ColumnArray::Offset row = 0; row < offset - prev; ++row) + for (ColumnArray::Offset row = 0; row < off - prev; ++row) { - mat(row, col) = X(data[prev + row]); + mat(row, col) = static_cast(data[prev + row]); } ++col; - prev = offset; + prev = off; } } }; void registerFunctionArrayDistance(FunctionFactory & factory) { - factory.registerFunction>(); factory.registerFunction>>(); + factory.registerFunction>>(); factory.registerFunction>(); factory.registerFunction>(); } diff --git a/tests/queries/0_stateless/02280_array_distance.reference b/tests/queries/0_stateless/02280_array_distance.reference new file mode 100644 index 00000000000..9de9521befe --- /dev/null +++ b/tests/queries/0_stateless/02280_array_distance.reference @@ -0,0 +1,30 @@ +6 +3.7416575 +3 +0.0025851727 +\N +nan +12 +14 +21 +7.071068 +9.165152 +12.124355 +2 +5 +4 +0.16847819 +0.35846698 +0.07417989 +6 +8 +9 +0.020204103 +0.118082896 +0 +1 1 218.74643 +1 2 1348.2118 +2 1 219.28064 +2 2 1347.4009 +3 1 214.35251 +3 2 1342.8857 diff --git a/tests/queries/0_stateless/02280_array_distance.sql b/tests/queries/0_stateless/02280_array_distance.sql new file mode 100644 index 00000000000..0f269b11179 --- /dev/null +++ b/tests/queries/0_stateless/02280_array_distance.sql @@ -0,0 +1,29 @@ +SELECT arrayL1Distance([0, 0, 0], [1, 2, 3]); +SELECT arrayL2Distance([1, 2, 3], [0, 0, 0]); +SELECT arrayLinfDistance([1, 2, 3], [0, 0, 0]); +SELECT arrayCosineDistance([1, 2, 3], [3, 5, 7]); + +SELECT arrayL2Distance([1, 2, 3], NULL); +SELECT arrayCosineDistance([1, 2, 3], [0, 0, 0]); + +DROP TABLE IF EXISTS vec1; +DROP TABLE IF EXISTS vec2; +CREATE TABLE vec1 (id UInt64, v Array(UInt8)) ENGINE = Memory; +CREATE TABLE vec2 (id UInt64, v Array(Int64)) ENGINE = Memory; + +INSERT INTO vec1 VALUES (1, [3, 4, 5]), (2, [2, 4, 8]), (3, [7, 7, 7]); +SELECT arrayL1Distance(v, [0, 0, 0]) FROM vec1; +SELECT arrayL2Distance(v, [0, 0, 0]) FROM vec1; +SELECT arrayLinfDistance([5, 4, 3], v) FROM vec1; +SELECT arrayCosineDistance([3, 2, 1], v) FROM vec1; +SELECT arrayLinfDistance(v, materialize([0, -2, 0])) FROM vec1; +SELECT arrayCosineDistance(v, materialize([1., 1., 1.])) FROM vec1; + +INSERT INTO vec2 VALUES (1, [100, 200, 0]), (2, [888, 777, 666]); +SELECT v1.id, v2.id, arrayL2Distance(v1.v, v2.v) as dist FROM vec1 v1, vec2 v2; + +SELECT arrayL1Distance([0, 0], [1]); -- { serverError 190 } +SELECT arrayL2Distance((1, 2), (3,4)); -- { serverError 43 } + +DROP TABLE vec1; +DROP TABLE vec2; From 6e73cd692942a10d95732e0751a97bff606f18c9 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Apr 2022 01:18:40 +0000 Subject: [PATCH 039/171] Implement parallel grouping sets processing --- src/Interpreters/Aggregator.cpp | 6 + src/Interpreters/Aggregator.h | 77 ++++------ src/Interpreters/ExpressionAnalyzer.cpp | 7 +- src/Interpreters/InterpreterSelectQuery.cpp | 145 ++++++++---------- src/Interpreters/InterpreterSelectQuery.h | 7 +- src/Processors/QueryPlan/AggregatingStep.cpp | 133 ++++++++++++++-- src/Processors/QueryPlan/GroupingSetsStep.cpp | 46 ------ src/Processors/QueryPlan/GroupingSetsStep.h | 25 --- .../Transforms/GroupingSetsTransform.cpp | 64 ++++---- .../Transforms/GroupingSetsTransform.h | 18 ++- src/QueryPipeline/Pipe.cpp | 19 ++- src/QueryPipeline/Pipe.h | 2 +- src/QueryPipeline/QueryPipelineBuilder.cpp | 4 +- src/QueryPipeline/QueryPipelineBuilder.h | 2 +- 14 files changed, 288 insertions(+), 267 deletions(-) delete mode 100644 src/Processors/QueryPlan/GroupingSetsStep.cpp delete mode 100644 src/Processors/QueryPlan/GroupingSetsStep.h diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 55f9fe602f4..44b6af0d82b 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -7,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -370,6 +372,7 @@ Block Aggregator::Params::getHeader( const Block & intermediate_header, const ColumnNumbers & keys, const AggregateDescriptions & aggregates, + const GroupingSetsParams & grouping_sets_params, bool final) { Block res; @@ -409,6 +412,9 @@ Block Aggregator::Params::getHeader( res.insert({ type, aggregate.column_name }); } + + if (grouping_sets_params.isValid()) + res.insert({ std::make_shared(), "__grouping_set" }); } return materializeBlock(res); diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 1a85a839b4f..82e252b0b05 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -18,6 +18,7 @@ #include #include #include +#include #include @@ -950,6 +951,23 @@ public: }; StatsCollectingParams stats_collecting_params; + struct GroupingSetsParams + { + GroupingSetsParams() = default; + + GroupingSetsParams(ColumnNumbersList grouping_sets_with_keys_, ColumnNumbersList missing_columns_per_set_) + : grouping_sets_with_keys(std::move(grouping_sets_with_keys_)) + , missing_columns_per_set(std::move(missing_columns_per_set_)) + {} + + bool isValid() const noexcept { return !grouping_sets_with_keys.empty(); } + + size_t size() const noexcept { return grouping_sets_with_keys.size(); } + + ColumnNumbersList grouping_sets_with_keys; + ColumnNumbersList missing_columns_per_set; + } grouping_sets_params; + Params( const Block & src_header_, const ColumnNumbers & keys_, @@ -967,7 +985,8 @@ public: bool compile_aggregate_expressions_, size_t min_count_to_compile_aggregate_expression_, const Block & intermediate_header_ = {}, - const StatsCollectingParams & stats_collecting_params_ = {}) + const StatsCollectingParams & stats_collecting_params_ = {}, + const GroupingSetsParams & grouping_sets_params_ = {}) : src_header(src_header_) , intermediate_header(intermediate_header_) , keys(keys_) @@ -987,51 +1006,10 @@ public: , compile_aggregate_expressions(compile_aggregate_expressions_) , min_count_to_compile_aggregate_expression(min_count_to_compile_aggregate_expression_) , stats_collecting_params(stats_collecting_params_) + , grouping_sets_params(grouping_sets_params_) { } - /// two dimensional vector of aggregating keys in params - Params( - const Block & src_header_, - const ColumnNumbers & keys_, - const ColumnNumbersList & keys_vector_, - const AggregateDescriptions & aggregates_, - bool overflow_row_, - size_t max_rows_to_group_by_, - OverflowMode group_by_overflow_mode_, - size_t group_by_two_level_threshold_, - size_t group_by_two_level_threshold_bytes_, - size_t max_bytes_before_external_group_by_, - bool empty_result_for_aggregation_by_empty_set_, - VolumePtr tmp_volume_, - size_t max_threads_, - size_t min_free_disk_space_, - bool compile_aggregate_expressions_, - size_t min_count_to_compile_aggregate_expression_, - const Block & intermediate_header_ = {}, - const StatsCollectingParams & stats_collecting_params_ = {}) - : src_header(src_header_) - , intermediate_header(intermediate_header_) - , keys(keys_) - , keys_vector(keys_vector_) - , aggregates(aggregates_) - , keys_size(keys.size()) - , aggregates_size(aggregates.size()) - , overflow_row(overflow_row_) - , max_rows_to_group_by(max_rows_to_group_by_) - , group_by_overflow_mode(group_by_overflow_mode_) - , group_by_two_level_threshold(group_by_two_level_threshold_) - , group_by_two_level_threshold_bytes(group_by_two_level_threshold_bytes_) - , max_bytes_before_external_group_by(max_bytes_before_external_group_by_) - , empty_result_for_aggregation_by_empty_set(empty_result_for_aggregation_by_empty_set_) - , tmp_volume(tmp_volume_) - , max_threads(max_threads_) - , min_free_disk_space(min_free_disk_space_) - , compile_aggregate_expressions(compile_aggregate_expressions_) - , min_count_to_compile_aggregate_expression(min_count_to_compile_aggregate_expression_) - , stats_collecting_params(stats_collecting_params_) - {} - /// Only parameters that matter during merge. Params(const Block & intermediate_header_, const ColumnNumbers & keys_, const AggregateDescriptions & aggregates_, bool overflow_row_, size_t max_threads_) @@ -1045,11 +1023,22 @@ public: const Block & intermediate_header, const ColumnNumbers & keys, const AggregateDescriptions & aggregates, + const GroupingSetsParams & grouping_sets_params, bool final); Block getHeader(bool final) const { - return getHeader(src_header, intermediate_header, keys, aggregates, final); + return getHeader(src_header, intermediate_header, keys, aggregates, grouping_sets_params, final); + } + + bool hasGroupingSets() const noexcept + { + return grouping_sets_params.isValid(); + } + + GroupingSetsParams const & getGroupingSetsParams() const noexcept + { + return grouping_sets_params; } /// Returns keys and aggregated for EXPLAIN query diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index ddb980934be..9ba9448868a 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -41,8 +42,9 @@ #include #include -#include "Core/NamesAndTypes.h" +#include +#include #include #include @@ -450,6 +452,9 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) for (const auto & desc : aggregate_descriptions) aggregated_columns.emplace_back(desc.column_name, desc.function->getReturnType()); + + if (select_query->group_by_with_grouping_sets) + aggregated_columns.emplace_back("__grouping_set", std::make_shared()); } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index b950b196909..888d4cdb7a9 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -45,7 +45,6 @@ #include #include #include -#include #include #include #include @@ -81,8 +80,11 @@ #include #include #include +#include +#include #include #include +#include #include @@ -1070,7 +1072,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

QueryProcessingStage::WithMergeableState && - !query.group_by_with_totals && !query.group_by_with_rollup && !query.group_by_with_cube && !query.group_by_with_grouping_sets; + !query.group_by_with_totals && !query.group_by_with_rollup && !query.group_by_with_cube; // if (query.group_by_with_grouping_sets && query.group_by_with_totals) // throw Exception("WITH TOTALS and GROUPING SETS are not supported together", ErrorCodes::NOT_IMPLEMENTED); @@ -1354,7 +1356,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

aggregationKeys()) + keys.push_back(current_data_stream_header.getPositionByName(key.name)); + + return Aggregator::Params{ + current_data_stream_header, + keys, + aggregates, + overflow_row, + settings.max_rows_to_group_by, + settings.group_by_overflow_mode, + group_by_two_level_threshold, + group_by_two_level_threshold_bytes, + settings.max_bytes_before_external_group_by, + settings.empty_result_for_aggregation_by_empty_set + || (settings.empty_result_for_aggregation_by_constant_keys_on_empty_set && keys.empty() + && query_analyzer->hasConstAggregationKeys()), + context->getTemporaryVolume(), + settings.max_threads, + settings.min_free_disk_space_for_temporary_data, + settings.compile_aggregate_expressions, + settings.min_count_to_compile_aggregate_expression, + Block{}, + stats_collecting_params, + getAggregatorGroupingSetsParams(current_data_stream_header, keys) + }; +} + +Aggregator::Params::GroupingSetsParams InterpreterSelectQuery::getAggregatorGroupingSetsParams( + const Block & header_before_aggregation, + const ColumnNumbers & all_keys +) +{ + if (getSelectQuery().group_by_with_grouping_sets) { - ColumnNumbers keys; - ColumnNumbers all_keys; - ColumnNumbersList keys_vector; - std::unordered_set keys_set; - for (const auto & aggregation_keys : query_analyzer->aggregationKeysList()) + auto const & aggregation_keys_list = query_analyzer->aggregationKeysList(); + + ColumnNumbersList grouping_sets_with_keys; + ColumnNumbersList missing_columns_per_set; + + for (const auto & aggregation_keys : aggregation_keys_list) { - keys.clear(); + ColumnNumbers keys; + std::unordered_set keys_set; for (const auto & key : aggregation_keys) { - size_t key_name_pos = current_data_stream_header.getPositionByName(key.name); - if (!keys_set.contains(key_name_pos)) - { - keys_set.insert(key_name_pos); - all_keys.push_back(key_name_pos); - } - keys.push_back(key_name_pos); + keys.push_back(header_before_aggregation.getPositionByName(key.name)); + keys_set.insert(keys.back()); } - keys_vector.push_back(keys); + grouping_sets_with_keys.push_back(std::move(keys)); + + ColumnNumbers missing_indexes; + for (size_t i = 0; i < all_keys.size(); ++i) + { + if (!keys_set.contains(all_keys[i])) + missing_indexes.push_back(i); + } + missing_columns_per_set.push_back(std::move(missing_indexes)); } - - params_ptr = std::make_unique( - current_data_stream_header, - all_keys, - keys_vector, - aggregates, - overflow_row, - settings.max_rows_to_group_by, - settings.group_by_overflow_mode, - group_by_two_level_threshold, - group_by_two_level_threshold_bytes, - settings.max_bytes_before_external_group_by, - settings.empty_result_for_aggregation_by_empty_set - || (settings.empty_result_for_aggregation_by_constant_keys_on_empty_set && keys.empty() - && query_analyzer->hasConstAggregationKeys()), - context->getTemporaryVolume(), - settings.max_threads, - settings.min_free_disk_space_for_temporary_data, - settings.compile_aggregate_expressions, - settings.min_count_to_compile_aggregate_expression, - Block{}, - stats_collecting_params); - } - else - { - ColumnNumbers keys; - for (const auto & key : query_analyzer->aggregationKeys()) - keys.push_back(current_data_stream_header.getPositionByName(key.name)); - - params_ptr = std::make_unique( - current_data_stream_header, - keys, - aggregates, - overflow_row, - settings.max_rows_to_group_by, - settings.group_by_overflow_mode, - group_by_two_level_threshold, - group_by_two_level_threshold_bytes, - settings.max_bytes_before_external_group_by, - settings.empty_result_for_aggregation_by_empty_set - || (settings.empty_result_for_aggregation_by_constant_keys_on_empty_set && keys.empty() - && query_analyzer->hasConstAggregationKeys()), - context->getTemporaryVolume(), - settings.max_threads, - settings.min_free_disk_space_for_temporary_data, - settings.compile_aggregate_expressions, - settings.min_count_to_compile_aggregate_expression, - Block{}, - stats_collecting_params); + return { std::move(grouping_sets_with_keys), std::move(missing_columns_per_set) }; } + return {}; } void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const ActionsDAGPtr & expression, bool overflow_row, bool final, InputOrderInfoPtr group_by_info) @@ -2258,8 +2247,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac const Settings & settings = context->getSettingsRef(); - AggregatorParamsPtr params_ptr; - initAggregatorParams(header_before_aggregation, params_ptr, aggregates, overflow_row, settings, + auto aggregator_params = getAggregatorParams(header_before_aggregation, aggregates, overflow_row, settings, settings.group_by_two_level_threshold, settings.group_by_two_level_threshold_bytes); SortDescription group_by_sort_description; @@ -2278,7 +2266,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac auto aggregating_step = std::make_unique( query_plan.getCurrentDataStream(), - *params_ptr, + std::move(aggregator_params), final, settings.max_block_size, settings.aggregation_in_order_max_block_bytes, @@ -2344,17 +2332,14 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPlan & query_plan, Modific const Settings & settings = context->getSettingsRef(); - AggregatorParamsPtr params_ptr; - initAggregatorParams(header_before_transform, params_ptr, query_analyzer->aggregates(), false, settings, 0, 0); - auto transform_params = std::make_shared(*params_ptr, true); + auto params = getAggregatorParams(header_before_transform, query_analyzer->aggregates(), false, settings, 0, 0); + auto transform_params = std::make_shared(std::move(params), true); QueryPlanStepPtr step; if (modificator == Modificator::ROLLUP) step = std::make_unique(query_plan.getCurrentDataStream(), std::move(transform_params)); else if (modificator == Modificator::CUBE) step = std::make_unique(query_plan.getCurrentDataStream(), std::move(transform_params)); - else - step = std::make_unique(query_plan.getCurrentDataStream(), std::move(transform_params)); query_plan.addStep(std::move(step)); } diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 64be36c46cb..b804a46ea31 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -131,12 +131,14 @@ private: /// Different stages of query execution. - void initAggregatorParams( + Aggregator::Params getAggregatorParams( const Block & current_data_stream_header, - AggregatorParamsPtr & params_ptr, const AggregateDescriptions & aggregates, bool overflow_row, const Settings & settings, size_t group_by_two_level_threshold, size_t group_by_two_level_threshold_bytes); + Aggregator::Params::GroupingSetsParams getAggregatorGroupingSetsParams( + const Block & header_before_aggregation, + const ColumnNumbers & all_keys); void executeFetchColumns(QueryProcessingStage::Enum processing_stage, QueryPlan & query_plan); void executeWhere(QueryPlan & query_plan, const ActionsDAGPtr & expression, bool remove_filter); void executeAggregation( @@ -168,7 +170,6 @@ private: { ROLLUP = 0, CUBE = 1, - GROUPING_SETS = 2 }; void executeRollupOrCube(QueryPlan & query_plan, Modificator modificator); diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 0b48a38e7db..536ca00e176 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -1,10 +1,17 @@ +#include +#include +#include #include #include +#include #include #include #include #include #include +#include "Interpreters/Aggregator.h" +#include "Processors/QueryPlan/IQueryPlanStep.h" +#include "Processors/Transforms/GroupingSetsTransform.h" namespace DB { @@ -145,35 +152,129 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B } } - /// If there are several sources, then we perform parallel aggregation - if (pipeline.getNumStreams() > 1) + if (params.hasGroupingSets()) { - /// Add resize transform to uniformly distribute data between aggregating streams. - if (!storage_has_evenly_distributed_read) - pipeline.resize(pipeline.getNumStreams(), true, true); + const auto & grouping_sets_params = params.getGroupingSetsParams(); + const size_t grouping_sets_size = grouping_sets_params.size(); - auto many_data = std::make_shared(pipeline.getNumStreams()); + const size_t streams = pipeline.getNumStreams(); - size_t counter = 0; - pipeline.addSimpleTransform([&](const Block & header) + auto input_header = pipeline.getHeader(); + pipeline.transform([&](OutputPortRawPtrs ports) { - return std::make_shared(header, transform_params, many_data, counter++, merge_threads, temporary_data_merge_threads); + Processors copiers; + copiers.reserve(ports.size()); + + for (auto * port : ports) + { + auto copier = std::make_shared(input_header, grouping_sets_size); + connect(*port, copier->getInputPort()); + copiers.push_back(copier); + } + + return copiers; }); - pipeline.resize(1); + std::vector transform_params_per_set; + pipeline.transform([&](OutputPortRawPtrs ports) + { + assert(streams * grouping_sets_size == ports.size()); + Processors aggregators; + for (size_t i = 0; i < grouping_sets_size; ++i) + { + Aggregator::Params params_for_set{ + transform_params->params.src_header, + grouping_sets_params.grouping_sets_with_keys[i], + transform_params->params.aggregates, + transform_params->params.overflow_row, + transform_params->params.max_rows_to_group_by, + transform_params->params.group_by_overflow_mode, + transform_params->params.group_by_two_level_threshold, + transform_params->params.group_by_two_level_threshold_bytes, + transform_params->params.max_bytes_before_external_group_by, + transform_params->params.empty_result_for_aggregation_by_empty_set, + transform_params->params.tmp_volume, + transform_params->params.max_threads, + transform_params->params.min_free_disk_space, + transform_params->params.compile_aggregate_expressions, + transform_params->params.min_count_to_compile_aggregate_expression, + transform_params->params.intermediate_header, + transform_params->params.stats_collecting_params + }; + auto transform_params_for_set = std::make_shared(std::move(params_for_set), final); + transform_params_per_set.push_back(transform_params_for_set); + auto many_data = std::make_shared(streams); + + for (size_t j = 0; j < streams; ++j) + { + auto aggregation_for_set = std::make_shared(input_header, transform_params_for_set, many_data, j, merge_threads, temporary_data_merge_threads); + connect(*ports[i + streams * j], aggregation_for_set->getInputs().front()); + aggregators.push_back(aggregation_for_set); + } + } + return aggregators; + }, false); + + pipeline.transform([&](OutputPortRawPtrs ports) + { + Processors resizes; + for (size_t i = 0; i < grouping_sets_size; ++i) + { + auto resize = std::make_shared(transform_params_per_set[i]->getHeader(), streams, 1); + auto & inputs = resize->getInputs(); + auto output_it = ports.begin() + i * streams; + for (auto input_it = inputs.begin(); input_it != inputs.end(); ++output_it, ++input_it) + connect(**output_it, *input_it); + resizes.push_back(resize); + } + return resizes; + }, false); + + assert(pipeline.getNumStreams() == grouping_sets_size); + size_t set_counter = 0; + auto output_header = transform_params->getHeader(); + pipeline.addSimpleTransform([&](const Block & header) + { + auto transform = std::make_shared(header, output_header, transform_params_per_set[set_counter], grouping_sets_params.missing_columns_per_set, set_counter); + ++set_counter; + return transform; + }); aggregating = collector.detachProcessors(0); + return; } else { - pipeline.resize(1); - - pipeline.addSimpleTransform([&](const Block & header) + /// If there are several sources, then we perform parallel aggregation + if (pipeline.getNumStreams() > 1) { - return std::make_shared(header, transform_params); - }); + /// Add resize transform to uniformly distribute data between aggregating streams. + if (!storage_has_evenly_distributed_read) + pipeline.resize(pipeline.getNumStreams(), true, true); - aggregating = collector.detachProcessors(0); + auto many_data = std::make_shared(pipeline.getNumStreams()); + + size_t counter = 0; + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, transform_params, many_data, counter++, merge_threads, temporary_data_merge_threads); + }); + + pipeline.resize(1); + + aggregating = collector.detachProcessors(0); + } + else + { + pipeline.resize(1); + + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, transform_params); + }); + + aggregating = collector.detachProcessors(0); + } } } diff --git a/src/Processors/QueryPlan/GroupingSetsStep.cpp b/src/Processors/QueryPlan/GroupingSetsStep.cpp deleted file mode 100644 index 383ecf5ea1b..00000000000 --- a/src/Processors/QueryPlan/GroupingSetsStep.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include -#include -#include "QueryPipeline/QueryPipelineBuilder.h" - -namespace DB -{ - -static ITransformingStep::Traits getTraits() -{ - return ITransformingStep::Traits - { - { - .preserves_distinct_columns = false, - .returns_single_stream = true, - .preserves_number_of_streams = false, - .preserves_sorting = false, - }, - { - .preserves_number_of_rows = false, - } - }; -} - -GroupingSetsStep::GroupingSetsStep(const DataStream & input_stream_, AggregatingTransformParamsPtr params_) - : ITransformingStep(input_stream_, params_->getHeader(), getTraits()) - , params(std::move(params_)) -{ - /// Aggregation keys are distinct - for (auto key : params->params.keys) - output_stream->distinct_columns.insert(params->params.src_header.getByPosition(key).name); -} - -void GroupingSetsStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) -{ - pipeline.resize(1); - - pipeline.addSimpleTransform([&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr - { - if (stream_type == QueryPipelineBuilder::StreamType::Totals) - return nullptr; - - return std::make_shared(header, std::move(params)); - }); -} - -} diff --git a/src/Processors/QueryPlan/GroupingSetsStep.h b/src/Processors/QueryPlan/GroupingSetsStep.h deleted file mode 100644 index 3dc7c3b5a99..00000000000 --- a/src/Processors/QueryPlan/GroupingSetsStep.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once -#include -#include -#include "QueryPipeline/QueryPipelineBuilder.h" - -namespace DB -{ - -struct AggregatingTransformParams; -using AggregatingTransformParamsPtr = std::shared_ptr; - -class GroupingSetsStep : public ITransformingStep -{ -public: - GroupingSetsStep(const DataStream & input_stream_, AggregatingTransformParamsPtr params_); - - String getName() const override { return "GroupingSets"; } - - void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override; - -private: - AggregatingTransformParamsPtr params; -}; - -} diff --git a/src/Processors/Transforms/GroupingSetsTransform.cpp b/src/Processors/Transforms/GroupingSetsTransform.cpp index 960ea0e5d76..896cb9bcefe 100644 --- a/src/Processors/Transforms/GroupingSetsTransform.cpp +++ b/src/Processors/Transforms/GroupingSetsTransform.cpp @@ -1,17 +1,26 @@ +#include #include #include +#include +#include +#include namespace DB { -GroupingSetsTransform::GroupingSetsTransform(Block header, AggregatingTransformParamsPtr params_) - : IAccumulatingTransform(std::move(header), params_->getHeader()) +GroupingSetsTransform::GroupingSetsTransform( + Block input_header, + Block output_header, + AggregatingTransformParamsPtr params_, + ColumnNumbersList const & missing_columns_, + size_t set_id_ +) + : IAccumulatingTransform(std::move(input_header), std::move(output_header)) , params(std::move(params_)) - , keys(params->params.keys) - , keys_vector(params->params.keys_vector) - , keys_vector_idx(0) -{ -} + , missing_columns(missing_columns_[set_id_]) + , set_id(set_id_) + , output_size(getOutputPort().getHeader().columns()) +{} void GroupingSetsTransform::consume(Chunk chunk) { @@ -31,8 +40,10 @@ Chunk GroupingSetsTransform::merge(Chunks && chunks, bool final) Chunk GroupingSetsTransform::generate() { + Chunk result; if (!consumed_chunks.empty()) { + Chunk grouping_sets_chunk; if (consumed_chunks.size() > 1) grouping_sets_chunk = merge(std::move(consumed_chunks), false); else @@ -40,37 +51,26 @@ Chunk GroupingSetsTransform::generate() consumed_chunks.clear(); - auto num_rows = grouping_sets_chunk.getNumRows(); + size_t rows = grouping_sets_chunk.getNumRows(); - current_columns = grouping_sets_chunk.getColumns(); - current_zero_columns.clear(); - - for (auto key : keys) - current_zero_columns.emplace(key, current_columns[key]->cloneEmpty()->cloneResized(num_rows)); - } - - Chunk gen_chunk; - - if (!current_columns.empty() && keys_vector_idx < keys_vector.size()) - { - auto columns = current_columns; - std::set key_vector(keys_vector[keys_vector_idx].begin(), keys_vector[keys_vector_idx].end()); - - for (auto key : keys) + auto columns = grouping_sets_chunk.detachColumns(); + Columns result_columns; + auto const & output_header = getOutputPort().getHeader(); + size_t real_column_index = 0, missign_column_index = 0; + for (size_t i = 0; i < output_header.columns() - 1; ++i) { - if (!key_vector.contains(key)) - columns[key] = current_zero_columns[key]; + if (missign_column_index < missing_columns.size() && missing_columns[missign_column_index] == i) + result_columns.push_back(output_header.getByPosition(missing_columns[missign_column_index++]).column->cloneResized(rows)); + else + result_columns.push_back(std::move(columns[real_column_index++])); } + result_columns.push_back(ColumnUInt64::create(rows, set_id)); - Chunks chunks; - chunks.emplace_back(std::move(columns), current_columns.front()->size()); - gen_chunk = merge(std::move(chunks), false); - - ++keys_vector_idx; + result = Chunk(std::move(result_columns), rows); } - finalizeChunk(gen_chunk); - return gen_chunk; + finalizeChunk(result); + return result; } } diff --git a/src/Processors/Transforms/GroupingSetsTransform.h b/src/Processors/Transforms/GroupingSetsTransform.h index e9b20867ef4..1323b14fc27 100644 --- a/src/Processors/Transforms/GroupingSetsTransform.h +++ b/src/Processors/Transforms/GroupingSetsTransform.h @@ -1,6 +1,7 @@ #pragma once #include #include +#include "Core/ColumnNumbers.h" namespace DB { @@ -8,7 +9,12 @@ namespace DB class GroupingSetsTransform : public IAccumulatingTransform { public: - GroupingSetsTransform(Block header, AggregatingTransformParamsPtr params); + GroupingSetsTransform( + Block input_header, + Block output_header, + AggregatingTransformParamsPtr params, + ColumnNumbersList const & missing_columns, + size_t set_id); String getName() const override { return "GroupingSetsTransform"; } protected: @@ -17,15 +23,11 @@ protected: private: AggregatingTransformParamsPtr params; - const ColumnNumbers & keys; - const ColumnNumbersList & keys_vector; + const ColumnNumbers missing_columns; + const size_t set_id; + const size_t output_size; Chunks consumed_chunks; - Chunk grouping_sets_chunk; - Columns current_columns; - std::unordered_map current_zero_columns; - - UInt64 keys_vector_idx = 0; Poco::Logger * log = &Poco::Logger::get("GroupingSetsTransform"); diff --git a/src/QueryPipeline/Pipe.cpp b/src/QueryPipeline/Pipe.cpp index 791fc03fccb..5ce55a6119d 100644 --- a/src/QueryPipeline/Pipe.cpp +++ b/src/QueryPipeline/Pipe.cpp @@ -759,7 +759,7 @@ void Pipe::setSinks(const Pipe::ProcessorGetterWithStreamKind & getter) header.clear(); } -void Pipe::transform(const Transformer & transformer) +void Pipe::transform(const Transformer & transformer, bool check_block_structure) { if (output_ports.empty()) throw Exception("Cannot transform empty Pipe.", ErrorCodes::LOGICAL_ERROR); @@ -817,15 +817,18 @@ void Pipe::transform(const Transformer & transformer) throw Exception("Transformation of Pipe is not valid because processors don't have any " "not-connected output ports", ErrorCodes::LOGICAL_ERROR); - header = output_ports.front()->getHeader(); - for (size_t i = 1; i < output_ports.size(); ++i) - assertBlocksHaveEqualStructure(header, output_ports[i]->getHeader(), "Pipe"); + if (check_block_structure) + { + header = output_ports.front()->getHeader(); + for (size_t i = 1; i < output_ports.size(); ++i) + assertBlocksHaveEqualStructure(header, output_ports[i]->getHeader(), "Pipe"); - if (totals_port) - assertBlocksHaveEqualStructure(header, totals_port->getHeader(), "Pipes"); + if (totals_port) + assertBlocksHaveEqualStructure(header, totals_port->getHeader(), "Pipes"); - if (extremes_port) - assertBlocksHaveEqualStructure(header, extremes_port->getHeader(), "Pipes"); + if (extremes_port) + assertBlocksHaveEqualStructure(header, extremes_port->getHeader(), "Pipes"); + } if (collected_processors) { diff --git a/src/QueryPipeline/Pipe.h b/src/QueryPipeline/Pipe.h index a8c256932ef..4372e4844dc 100644 --- a/src/QueryPipeline/Pipe.h +++ b/src/QueryPipeline/Pipe.h @@ -90,7 +90,7 @@ public: using Transformer = std::function; /// Transform Pipe in general way. - void transform(const Transformer & transformer); + void transform(const Transformer & transformer, bool check_block_structure = true); /// Unite several pipes together. They should have same header. static Pipe unitePipes(Pipes pipes); diff --git a/src/QueryPipeline/QueryPipelineBuilder.cpp b/src/QueryPipeline/QueryPipelineBuilder.cpp index 9f392b51cf0..e65ec76c2e3 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.cpp +++ b/src/QueryPipeline/QueryPipelineBuilder.cpp @@ -149,10 +149,10 @@ void QueryPipelineBuilder::addChain(Chain chain) pipe.addChains(std::move(chains)); } -void QueryPipelineBuilder::transform(const Transformer & transformer) +void QueryPipelineBuilder::transform(const Transformer & transformer, bool check_block_structure) { checkInitializedAndNotCompleted(); - pipe.transform(transformer); + pipe.transform(transformer, check_block_structure); } void QueryPipelineBuilder::setSinks(const Pipe::ProcessorGetterWithStreamKind & getter) diff --git a/src/QueryPipeline/QueryPipelineBuilder.h b/src/QueryPipeline/QueryPipelineBuilder.h index ac84191cf34..712e5b400b1 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.h +++ b/src/QueryPipeline/QueryPipelineBuilder.h @@ -63,7 +63,7 @@ public: using Transformer = std::function; /// Transform pipeline in general way. - void transform(const Transformer & transformer); + void transform(const Transformer & transformer, bool check_block_structure = true); /// Add TotalsHavingTransform. Resize pipeline to single input. Adds totals port. void addTotalsHavingTransform(ProcessorPtr transform); From 680f90da9ebfad3f50402bb21c85d1ec00739af5 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Apr 2022 01:26:01 +0000 Subject: [PATCH 040/171] Remove finalize from GroupingSetsTransform --- src/Processors/Transforms/GroupingSetsTransform.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Processors/Transforms/GroupingSetsTransform.cpp b/src/Processors/Transforms/GroupingSetsTransform.cpp index 896cb9bcefe..be436caddf2 100644 --- a/src/Processors/Transforms/GroupingSetsTransform.cpp +++ b/src/Processors/Transforms/GroupingSetsTransform.cpp @@ -69,7 +69,6 @@ Chunk GroupingSetsTransform::generate() result = Chunk(std::move(result_columns), rows); } - finalizeChunk(result); return result; } From 9cefb62341ac67159ee00c95f590bc49e481926c Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Apr 2022 10:43:11 +0000 Subject: [PATCH 041/171] Cleanup --- src/Processors/QueryPlan/AggregatingStep.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 536ca00e176..7cb1577417d 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -182,7 +182,8 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B Processors aggregators; for (size_t i = 0; i < grouping_sets_size; ++i) { - Aggregator::Params params_for_set{ + Aggregator::Params params_for_set + { transform_params->params.src_header, grouping_sets_params.grouping_sets_with_keys[i], transform_params->params.aggregates, From ea6a25b8e7acf96679351f8e804f090686c0086c Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Apr 2022 11:14:22 +0000 Subject: [PATCH 042/171] Fix nullptr dereference --- src/Interpreters/ExpressionAnalyzer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 49d5cfbb21b..ea75d1ec915 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -454,7 +454,7 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) for (const auto & desc : aggregate_descriptions) aggregated_columns.emplace_back(desc.column_name, desc.function->getReturnType()); - if (select_query->group_by_with_grouping_sets) + if (select_query != nullptr && select_query->group_by_with_grouping_sets) aggregated_columns.emplace_back("__grouping_set", std::make_shared()); } From df63a9b2059671cb4f1c00fe0607f3adac5b6498 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Apr 2022 13:41:59 +0000 Subject: [PATCH 043/171] Depricate TOTALS with GROUPING SETS --- src/Interpreters/InterpreterSelectQuery.cpp | 4 +-- .../01883_with_grouping_sets.reference | 28 ------------------- .../0_stateless/01883_with_grouping_sets.sql | 4 +-- 3 files changed, 4 insertions(+), 32 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index ebc7098d81d..0b48a8b8d31 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1074,8 +1074,8 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

QueryProcessingStage::WithMergeableState && !query.group_by_with_totals && !query.group_by_with_rollup && !query.group_by_with_cube; -// if (query.group_by_with_grouping_sets && query.group_by_with_totals) -// throw Exception("WITH TOTALS and GROUPING SETS are not supported together", ErrorCodes::NOT_IMPLEMENTED); + if (query.group_by_with_grouping_sets && query.group_by_with_totals) + throw Exception("WITH TOTALS and GROUPING SETS are not supported together", ErrorCodes::NOT_IMPLEMENTED); if (query_info.projection && query_info.projection->desc->type == ProjectionDescription::Type::Aggregate) { diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.reference b/tests/queries/0_stateless/01883_with_grouping_sets.reference index e52f72b84b9..31356cba162 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.reference +++ b/tests/queries/0_stateless/01883_with_grouping_sets.reference @@ -59,31 +59,3 @@ 2 3 0 5200 2 4 0 4800 2 5 0 5400 -1 0 24500 -1 1 4500 -1 3 4700 -1 5 4900 -1 7 5100 -1 9 5300 -2 0 25000 -2 2 4600 -2 4 4800 -2 6 5000 -2 8 5200 -2 10 5400 - -0 0 49500 -1 0 24500 -1 1 4500 -1 3 4700 -1 5 4900 -1 7 5100 -1 9 5300 -2 0 25000 -2 2 4600 -2 4 4800 -2 6 5000 -2 8 5200 -2 10 5400 - -0 0 49500 diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.sql b/tests/queries/0_stateless/01883_with_grouping_sets.sql index 92a56be0a3f..b147dcab2fe 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.sql +++ b/tests/queries/0_stateless/01883_with_grouping_sets.sql @@ -31,7 +31,7 @@ SELECT SUM(sales_value) AS sales_value FROM grouping_sets GROUP BY grouping sets ((fact_1_id), (fact_1_id, fact_3_id)) WITH TOTALS -ORDER BY fact_1_id, fact_3_id; +ORDER BY fact_1_id, fact_3_id; -- { serverError NOT_IMPLEMENTED } SELECT fact_1_id, @@ -39,6 +39,6 @@ SELECT SUM(sales_value) AS sales_value FROM grouping_sets GROUP BY grouping sets (fact_1_id, (fact_1_id, fact_3_id)) WITH TOTALS -ORDER BY fact_1_id, fact_3_id; +ORDER BY fact_1_id, fact_3_id; -- { serverError NOT_IMPLEMENTED } DROP TABLE grouping_sets; From b5e2b38529bf6d712f0eb3c0eb4e5dbf1952ecd9 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Apr 2022 14:11:58 +0000 Subject: [PATCH 044/171] Optimize pipeline in case of single input --- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- src/Processors/QueryPlan/AggregatingStep.cpp | 44 ++++++++++++------- .../01883_with_grouping_sets.reference | 18 ++++++++ .../0_stateless/01883_with_grouping_sets.sql | 5 +++ 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 0b48a8b8d31..8adb441ed48 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1074,7 +1074,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

QueryProcessingStage::WithMergeableState && !query.group_by_with_totals && !query.group_by_with_rollup && !query.group_by_with_cube; - if (query.group_by_with_grouping_sets && query.group_by_with_totals) + if (query.group_by_with_grouping_sets && query.group_by_with_totals) throw Exception("WITH TOTALS and GROUPING SETS are not supported together", ErrorCodes::NOT_IMPLEMENTED); if (query_info.projection && query_info.projection->desc->type == ProjectionDescription::Type::Aggregate) diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 7cb1577417d..e6502adf37b 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -204,32 +204,44 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B }; auto transform_params_for_set = std::make_shared(std::move(params_for_set), final); transform_params_per_set.push_back(transform_params_for_set); - auto many_data = std::make_shared(streams); - for (size_t j = 0; j < streams; ++j) + if (streams > 1) { - auto aggregation_for_set = std::make_shared(input_header, transform_params_for_set, many_data, j, merge_threads, temporary_data_merge_threads); - connect(*ports[i + streams * j], aggregation_for_set->getInputs().front()); + auto many_data = std::make_shared(streams); + for (size_t j = 0; j < streams; ++j) + { + auto aggregation_for_set = std::make_shared(input_header, transform_params_for_set, many_data, j, merge_threads, temporary_data_merge_threads); + connect(*ports[i + streams * j], aggregation_for_set->getInputs().front()); + aggregators.push_back(aggregation_for_set); + } + } + else + { + auto aggregation_for_set = std::make_shared(input_header, transform_params_for_set); + connect(*ports[i], aggregation_for_set->getInputs().front()); aggregators.push_back(aggregation_for_set); } } return aggregators; }, false); - pipeline.transform([&](OutputPortRawPtrs ports) + if (streams > 1) { - Processors resizes; - for (size_t i = 0; i < grouping_sets_size; ++i) + pipeline.transform([&](OutputPortRawPtrs ports) { - auto resize = std::make_shared(transform_params_per_set[i]->getHeader(), streams, 1); - auto & inputs = resize->getInputs(); - auto output_it = ports.begin() + i * streams; - for (auto input_it = inputs.begin(); input_it != inputs.end(); ++output_it, ++input_it) - connect(**output_it, *input_it); - resizes.push_back(resize); - } - return resizes; - }, false); + Processors resizes; + for (size_t i = 0; i < grouping_sets_size; ++i) + { + auto resize = std::make_shared(transform_params_per_set[i]->getHeader(), streams, 1); + auto & inputs = resize->getInputs(); + auto output_it = ports.begin() + i * streams; + for (auto input_it = inputs.begin(); input_it != inputs.end(); ++output_it, ++input_it) + connect(**output_it, *input_it); + resizes.push_back(resize); + } + return resizes; + }, false); + } assert(pipeline.getNumStreams() == grouping_sets_size); size_t set_counter = 0; diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.reference b/tests/queries/0_stateless/01883_with_grouping_sets.reference index 31356cba162..a97c4ddad35 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.reference +++ b/tests/queries/0_stateless/01883_with_grouping_sets.reference @@ -1,3 +1,21 @@ +(Expression) +ExpressionTransform + (Sorting) + MergingSortedTransform 2 → 1 + MergeSortingTransform × 2 + LimitsCheckingTransform × 2 + PartialSortingTransform × 2 + (Expression) + ExpressionTransform × 2 + (Aggregating) + GroupingSetsTransform × 2 + AggregatingTransform × 2 + Copy 1 → 2 + (Expression) + ExpressionTransform + (SettingQuotaAndLimits) + (ReadFromStorage) + Memory 0 → 1 1 0 1 4500 1 0 3 4700 1 0 5 4900 diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.sql b/tests/queries/0_stateless/01883_with_grouping_sets.sql index b147dcab2fe..a2edd5d4039 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.sql +++ b/tests/queries/0_stateless/01883_with_grouping_sets.sql @@ -13,6 +13,11 @@ SELECT number % 100 AS sales_value FROM system.numbers limit 1000; +EXPLAIN PIPELINE +SELECT fact_1_id, fact_2_id, fact_3_id, SUM(sales_value) AS sales_value from grouping_sets +GROUP BY GROUPING SETS ((fact_1_id, fact_2_id), (fact_1_id, fact_3_id)) +ORDER BY fact_1_id, fact_2_id, fact_3_id; + SELECT fact_1_id, fact_2_id, fact_3_id, SUM(sales_value) AS sales_value from grouping_sets GROUP BY GROUPING SETS ((fact_1_id, fact_2_id), (fact_1_id, fact_3_id)) ORDER BY fact_1_id, fact_2_id, fact_3_id; From 94a381e522f31e61e375f5531feb380bab05e012 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Apr 2022 17:17:55 +0000 Subject: [PATCH 045/171] Add test for parallel GROUPING SETS processing --- src/Processors/QueryPlan/AggregatingStep.cpp | 4 +- .../01883_with_grouping_sets.reference | 129 ++++++++++++++++++ .../0_stateless/01883_with_grouping_sets.sql | 9 ++ 3 files changed, 141 insertions(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index e6502adf37b..d7c49395eb4 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -211,7 +211,9 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B for (size_t j = 0; j < streams; ++j) { auto aggregation_for_set = std::make_shared(input_header, transform_params_for_set, many_data, j, merge_threads, temporary_data_merge_threads); - connect(*ports[i + streams * j], aggregation_for_set->getInputs().front()); + // For each input stream we have `grouping_sets_size` copies, so port index + // for transform #j should skip ports of first (j-1) streams. + connect(*ports[i + grouping_sets_size * j], aggregation_for_set->getInputs().front()); aggregators.push_back(aggregation_for_set); } } diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.reference b/tests/queries/0_stateless/01883_with_grouping_sets.reference index a97c4ddad35..ff2b70783c9 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.reference +++ b/tests/queries/0_stateless/01883_with_grouping_sets.reference @@ -77,3 +77,132 @@ ExpressionTransform 2 3 0 5200 2 4 0 4800 2 5 0 5400 +(Expression) +ExpressionTransform + (Sorting) + MergingSortedTransform 2 → 1 + MergeSortingTransform × 2 + LimitsCheckingTransform × 2 + PartialSortingTransform × 2 + (Expression) + ExpressionTransform × 2 + (Aggregating) + GroupingSetsTransform × 2 + Resize × 2 3 → 1 + AggregatingTransform × 6 + Copy × 3 1 → 2 + (Expression) + ExpressionTransform × 3 + (SettingQuotaAndLimits) + (ReadFromStorage) + NumbersMt × 3 0 → 1 +4999500000 10000 +4999510000 10000 +4999520000 10000 +4999530000 10000 +4999540000 10000 +4999550000 10000 +4999560000 10000 +4999570000 10000 +4999580000 10000 +4999590000 10000 +4999600000 10000 +4999610000 10000 +4999620000 10000 +4999630000 10000 +4999640000 10000 +4999650000 10000 +4999660000 10000 +4999670000 10000 +4999680000 10000 +4999690000 10000 +4999700000 10000 +4999710000 10000 +4999720000 10000 +4999730000 10000 +4999740000 10000 +4999750000 10000 +4999760000 10000 +4999770000 10000 +4999780000 10000 +4999790000 10000 +4999800000 10000 +4999810000 10000 +4999820000 10000 +4999830000 10000 +4999840000 10000 +4999850000 10000 +4999860000 10000 +4999870000 10000 +4999880000 10000 +4999890000 10000 +4999900000 10000 +4999910000 10000 +4999920000 10000 +4999930000 10000 +4999940000 10000 +4999950000 10000 +4999960000 10000 +4999970000 10000 +4999980000 10000 +4999990000 10000 +5000000000 10000 +5000010000 10000 +5000020000 10000 +5000030000 10000 +5000040000 10000 +5000050000 10000 +5000060000 10000 +5000070000 10000 +5000080000 10000 +5000090000 10000 +5000100000 10000 +5000110000 10000 +5000120000 10000 +5000130000 10000 +5000140000 10000 +5000150000 10000 +5000160000 10000 +5000170000 10000 +5000180000 10000 +5000190000 10000 +5000200000 10000 +5000210000 10000 +5000220000 10000 +5000230000 10000 +5000240000 10000 +5000250000 10000 +5000260000 10000 +5000270000 10000 +5000280000 10000 +5000290000 10000 +5000300000 10000 +5000310000 10000 +5000320000 10000 +5000330000 10000 +5000340000 10000 +5000350000 10000 +5000360000 10000 +5000370000 10000 +5000380000 10000 +5000390000 10000 +5000400000 10000 +5000410000 10000 +5000420000 10000 +5000430000 10000 +5000440000 10000 +5000450000 10000 +5000460000 10000 +5000470000 10000 +5000480000 10000 +5000490000 10000 +49999500000 100000 +49999600000 100000 +49999700000 100000 +49999800000 100000 +49999900000 100000 +50000000000 100000 +50000100000 100000 +50000200000 100000 +50000300000 100000 +50000400000 100000 diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.sql b/tests/queries/0_stateless/01883_with_grouping_sets.sql index a2edd5d4039..bf96248e10e 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.sql +++ b/tests/queries/0_stateless/01883_with_grouping_sets.sql @@ -47,3 +47,12 @@ GROUP BY grouping sets (fact_1_id, (fact_1_id, fact_3_id)) WITH TOTALS ORDER BY fact_1_id, fact_3_id; -- { serverError NOT_IMPLEMENTED } DROP TABLE grouping_sets; + +EXPLAIN PIPELINE +SELECT SUM(number) as sum_value, count() AS count_value from numbers_mt(1000000) +GROUP BY GROUPING SETS ((number % 10), (number % 100)) +ORDER BY sum_value, count_value SETTINGS max_threads=3; + +SELECT SUM(number) as sum_value, count() AS count_value from numbers_mt(1000000) +GROUP BY GROUPING SETS ((number % 10), (number % 100)) +ORDER BY sum_value, count_value SETTINGS max_threads=3; From ccd166f4e02a632f40679ce601da8141e19e37b1 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Apr 2022 18:30:48 +0000 Subject: [PATCH 046/171] Derive GroupingSetsTransform from ISimpleTransform --- .../Transforms/GroupingSetsTransform.cpp | 56 +++++-------------- .../Transforms/GroupingSetsTransform.h | 15 ++--- 2 files changed, 18 insertions(+), 53 deletions(-) diff --git a/src/Processors/Transforms/GroupingSetsTransform.cpp b/src/Processors/Transforms/GroupingSetsTransform.cpp index be436caddf2..a4a53c5d04e 100644 --- a/src/Processors/Transforms/GroupingSetsTransform.cpp +++ b/src/Processors/Transforms/GroupingSetsTransform.cpp @@ -15,61 +15,33 @@ GroupingSetsTransform::GroupingSetsTransform( ColumnNumbersList const & missing_columns_, size_t set_id_ ) - : IAccumulatingTransform(std::move(input_header), std::move(output_header)) + : ISimpleTransform(std::move(input_header), std::move(output_header), true) , params(std::move(params_)) , missing_columns(missing_columns_[set_id_]) , set_id(set_id_) , output_size(getOutputPort().getHeader().columns()) {} -void GroupingSetsTransform::consume(Chunk chunk) +void GroupingSetsTransform::transform(Chunk & chunk) { - consumed_chunks.emplace_back(std::move(chunk)); -} + size_t rows = chunk.getNumRows(); -Chunk GroupingSetsTransform::merge(Chunks && chunks, bool final) -{ - BlocksList grouping_sets_blocks; - for (auto & chunk : chunks) - grouping_sets_blocks.emplace_back(getInputPort().getHeader().cloneWithColumns(chunk.detachColumns())); + auto columns = chunk.detachColumns(); + Columns result_columns; + auto const & output_header = getOutputPort().getHeader(); - auto grouping_sets_block = params->aggregator.mergeBlocks(grouping_sets_blocks, final); - auto num_rows = grouping_sets_block.rows(); - return Chunk(grouping_sets_block.getColumns(), num_rows); -} - -Chunk GroupingSetsTransform::generate() -{ - Chunk result; - if (!consumed_chunks.empty()) + result_columns.reserve(output_header.columns()); + size_t real_column_index = 0, missign_column_index = 0; + for (size_t i = 0; i < output_header.columns() - 1; ++i) { - Chunk grouping_sets_chunk; - if (consumed_chunks.size() > 1) - grouping_sets_chunk = merge(std::move(consumed_chunks), false); + if (missign_column_index < missing_columns.size() && missing_columns[missign_column_index] == i) + result_columns.push_back(output_header.getByPosition(missing_columns[missign_column_index++]).column->cloneResized(rows)); else - grouping_sets_chunk = std::move(consumed_chunks.front()); - - consumed_chunks.clear(); - - size_t rows = grouping_sets_chunk.getNumRows(); - - auto columns = grouping_sets_chunk.detachColumns(); - Columns result_columns; - auto const & output_header = getOutputPort().getHeader(); - size_t real_column_index = 0, missign_column_index = 0; - for (size_t i = 0; i < output_header.columns() - 1; ++i) - { - if (missign_column_index < missing_columns.size() && missing_columns[missign_column_index] == i) - result_columns.push_back(output_header.getByPosition(missing_columns[missign_column_index++]).column->cloneResized(rows)); - else - result_columns.push_back(std::move(columns[real_column_index++])); - } - result_columns.push_back(ColumnUInt64::create(rows, set_id)); - - result = Chunk(std::move(result_columns), rows); + result_columns.push_back(std::move(columns[real_column_index++])); } + result_columns.push_back(ColumnUInt64::create(rows, set_id)); - return result; + chunk.setColumns(std::move(result_columns), rows); } } diff --git a/src/Processors/Transforms/GroupingSetsTransform.h b/src/Processors/Transforms/GroupingSetsTransform.h index 1323b14fc27..944004e606e 100644 --- a/src/Processors/Transforms/GroupingSetsTransform.h +++ b/src/Processors/Transforms/GroupingSetsTransform.h @@ -1,12 +1,12 @@ #pragma once -#include +#include #include -#include "Core/ColumnNumbers.h" +#include namespace DB { -class GroupingSetsTransform : public IAccumulatingTransform +class GroupingSetsTransform : public ISimpleTransform { public: GroupingSetsTransform( @@ -18,20 +18,13 @@ public: String getName() const override { return "GroupingSetsTransform"; } protected: - void consume(Chunk chunk) override; - Chunk generate() override; + void transform(Chunk & chunk) override; private: AggregatingTransformParamsPtr params; const ColumnNumbers missing_columns; const size_t set_id; const size_t output_size; - - Chunks consumed_chunks; - - Poco::Logger * log = &Poco::Logger::get("GroupingSetsTransform"); - - Chunk merge(Chunks && chunks, bool final); }; } From 6ee62007d371c1a2f57dce890b1bf676fa4da457 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Apr 2022 23:52:43 +0000 Subject: [PATCH 047/171] Handle GROUPING SETS with size < 2 --- src/Interpreters/InterpreterSelectQuery.cpp | 2 ++ .../0_stateless/01883_grouping_sets_crash.reference | 10 ++++++++++ .../queries/0_stateless/01883_grouping_sets_crash.sql | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 8adb441ed48..ba7df879379 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2200,6 +2200,8 @@ Aggregator::Params::GroupingSetsParams InterpreterSelectQuery::getAggregatorGrou if (getSelectQuery().group_by_with_grouping_sets) { auto const & aggregation_keys_list = query_analyzer->aggregationKeysList(); + if (aggregation_keys_list.size() <= 1) + return {}; ColumnNumbersList grouping_sets_with_keys; ColumnNumbersList missing_columns_per_set; diff --git a/tests/queries/0_stateless/01883_grouping_sets_crash.reference b/tests/queries/0_stateless/01883_grouping_sets_crash.reference index e88d4543b51..fd5a9e61eb3 100644 --- a/tests/queries/0_stateless/01883_grouping_sets_crash.reference +++ b/tests/queries/0_stateless/01883_grouping_sets_crash.reference @@ -154,3 +154,13 @@ THIRD QUERY: \N 0 0 0 \N 0 0 0 \N 0 0 0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 diff --git a/tests/queries/0_stateless/01883_grouping_sets_crash.sql b/tests/queries/0_stateless/01883_grouping_sets_crash.sql index bfa45cfaff1..ffe77f59557 100644 --- a/tests/queries/0_stateless/01883_grouping_sets_crash.sql +++ b/tests/queries/0_stateless/01883_grouping_sets_crash.sql @@ -61,4 +61,10 @@ ORDER BY fact_3_id DESC NULLS FIRST, fact_4_id ASC; +SELECT fact_3_id +FROM grouping_sets +GROUP BY + GROUPING SETS ((fact_3_id, fact_4_id)) +ORDER BY fact_3_id ASC; + DROP TABLE IF EXISTS grouping_sets; \ No newline at end of file From eea0a009dc1a491074901c35ad98d9fbe66a82af Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 30 Apr 2022 15:02:04 +0200 Subject: [PATCH 048/171] Debug --- src/Common/FileSegment.cpp | 9 ++++++--- src/Common/FileSegment.h | 1 + src/IO/WriteBufferFromS3.cpp | 28 +++++++++++----------------- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index 02391953531..e21ec739e3f 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -586,6 +586,7 @@ void FileSegment::completeImpl(std::lock_guard & cache_lock, std::lo downloader_id.clear(); } + LOG_TEST(log, "Completed file segment: {}", getInfoForLogImpl(segment_lock)); assertCorrectnessImpl(segment_lock); } @@ -648,7 +649,7 @@ void FileSegment::assertCorrectnessImpl(std::lock_guard & /* segment void FileSegment::assertNotDetached() const { if (detached) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Operation not allowed, file segment is detached"); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Operation not allowed, file segment is detached ({})", getInfoForLog()); } void FileSegment::assertDetachedStatus(std::lock_guard & /* segment_lock */) const @@ -685,12 +686,14 @@ void FileSegment::detach(std::lock_guard & cache_lock, std::lock_gua if (detached) return; - detached = true; - if (!hasFinalizedState()) { completeUnlocked(cache_lock, segment_lock); } + + detached = true; + + LOG_TEST(log, "Detached file segment: {}", getInfoForLog()); } FileSegmentsHolder::~FileSegmentsHolder() diff --git a/src/Common/FileSegment.h b/src/Common/FileSegment.h index a120b4272ae..bbedc1dedfd 100644 --- a/src/Common/FileSegment.h +++ b/src/Common/FileSegment.h @@ -20,6 +20,7 @@ using FileSegments = std::list; class FileSegment : boost::noncopyable { + friend class LRUFileCache; friend struct FileSegmentsHolder; diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 52a958ec968..514434bc3ed 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -113,9 +113,16 @@ void WriteBufferFromS3::nextImpl() } else { + size_t upper_bound = file_segments.back()->range().right; + LOG_TRACE( + log, + "Space reservation failed, will skip caching for range: [{}, {}], current full range is [{}, {}]", + file_segment->range().left, upper_bound, file_segments.front()->range().right, upper_bound); + for (auto reset_segment_it = file_segment_it; reset_segment_it != file_segments.end(); ++reset_segment_it) (*reset_segment_it)->complete(FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); file_segments.erase(file_segment_it, file_segments.end()); + break; } } @@ -134,9 +141,10 @@ void WriteBufferFromS3::nextImpl() writePart(); allocateBuffer(); - file_segments_holder.reset(); } + file_segments_holder.reset(); + waitForReadyBackGroundTasks(); } @@ -272,14 +280,7 @@ void WriteBufferFromS3::writePart() task->exception = std::current_exception(); } - try - { - finalizeCacheIfNeeded(task->cache_files); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } + finalizeCacheIfNeeded(task->cache_files); { std::lock_guard lock(bg_tasks_mutex); @@ -410,14 +411,7 @@ void WriteBufferFromS3::makeSinglepartUpload() put_object_task->exception = std::current_exception(); } - try - { - finalizeCacheIfNeeded(put_object_task->cache_files); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } + finalizeCacheIfNeeded(put_object_task->cache_files); { std::lock_guard lock(bg_tasks_mutex); From 8cf83a2daf61a0663d34d19d9b21f0973fcbae01 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 30 Apr 2022 22:02:44 +0200 Subject: [PATCH 049/171] Fix --- src/Common/FileSegment.cpp | 25 ++++++++++++++----------- src/Common/FileSegment.h | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index e21ec739e3f..b5faa88fc92 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -224,7 +224,10 @@ void FileSegment::write(const char * from, size_t size, size_t offset_) "Attempt to write {} bytes to offset: {}, but current download offset is {}", size, offset_, download_offset); - assertNotDetached(); + { + std::lock_guard segment_lock(mutex); + assertNotDetached(segment_lock); + } if (!cache_writer) { @@ -273,10 +276,10 @@ void FileSegment::writeInMemory(const char * from, size_t size) ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Not enough space is reserved. Available: {}, expected: {}", availableSize(), size); - assertNotDetached(); - std::lock_guard segment_lock(mutex); + assertNotDetached(segment_lock); + if (cache_writer) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache writer already initialized"); @@ -311,7 +314,7 @@ size_t FileSegment::finalizeWrite() if (size == 0) throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Writing zero size is not allowed"); - assertNotDetached(); + assertNotDetached(segment_lock); try { @@ -366,11 +369,11 @@ bool FileSegment::reserve(size_t size) if (!size) throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Zero space reservation is not allowed"); - assertNotDetached(); - { std::lock_guard segment_lock(mutex); + assertNotDetached(segment_lock); + auto caller_id = getCallerId(); if (downloader_id != caller_id) throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Space can be reserved only by downloader (current: {}, expected: {})", caller_id, downloader_id); @@ -455,7 +458,7 @@ void FileSegment::complete(State state) std::lock_guard cache_lock(cache->mutex); std::lock_guard segment_lock(mutex); - assertNotDetached(); + assertNotDetached(segment_lock); bool is_downloader = isDownloaderImpl(segment_lock); if (!is_downloader) @@ -513,7 +516,7 @@ void FileSegment::completeUnlocked(std::lock_guard & cache_lock, std setDownloaded(segment_lock); } - assertNotDetached(); + assertNotDetached(segment_lock); if (download_state == State::DOWNLOADING || download_state == State::EMPTY) { @@ -646,10 +649,10 @@ void FileSegment::assertCorrectnessImpl(std::lock_guard & /* segment assert(download_state != FileSegment::State::DOWNLOADED || std::filesystem::file_size(cache->getPathInLocalCache(key(), offset())) > 0); } -void FileSegment::assertNotDetached() const +void FileSegment::assertNotDetached(std::lock_guard & segment_lock) const { if (detached) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Operation not allowed, file segment is detached ({})", getInfoForLog()); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Operation not allowed, file segment is detached ({})", getInfoForLogImpl(segment_lock)); } void FileSegment::assertDetachedStatus(std::lock_guard & /* segment_lock */) const @@ -693,7 +696,7 @@ void FileSegment::detach(std::lock_guard & cache_lock, std::lock_gua detached = true; - LOG_TEST(log, "Detached file segment: {}", getInfoForLog()); + LOG_TEST(log, "Detached file segment: {}", getInfoForLogImpl(segment_lock)); } FileSegmentsHolder::~FileSegmentsHolder() diff --git a/src/Common/FileSegment.h b/src/Common/FileSegment.h index bbedc1dedfd..2faab5eba51 100644 --- a/src/Common/FileSegment.h +++ b/src/Common/FileSegment.h @@ -153,7 +153,7 @@ private: size_t getDownloadedSize(std::lock_guard & segment_lock) const; String getInfoForLogImpl(std::lock_guard & segment_lock) const; void assertCorrectnessImpl(std::lock_guard & segment_lock) const; - void assertNotDetached() const; + void assertNotDetached(std::lock_guard & segment_lock) const; void assertDetachedStatus(std::lock_guard & segment_lock) const; bool hasFinalizedState() const; bool isDetached(std::lock_guard & /* segment_lock */) const { return detached; } From 70e2bb626410952e17203c86eb3c2958addf7ff6 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 2 May 2022 17:56:05 +0200 Subject: [PATCH 050/171] Reimplement write-through cache --- src/Common/FileCache.cpp | 32 +++--- src/Common/FileCache.h | 9 +- src/Common/FileSegment.cpp | 202 +++++++++++++++++++++-------------- src/Common/FileSegment.h | 51 ++++++++- src/IO/WriteBufferFromS3.cpp | 100 ++++------------- src/IO/WriteBufferFromS3.h | 7 +- 6 files changed, 218 insertions(+), 183 deletions(-) diff --git a/src/Common/FileCache.cpp b/src/Common/FileCache.cpp index 764a605fdee..7dedf1c84a1 100644 --- a/src/Common/FileCache.cpp +++ b/src/Common/FileCache.cpp @@ -412,21 +412,6 @@ LRUFileCache::FileSegmentCell * LRUFileCache::addCell( return &(it->second); } -FileSegmentsHolder LRUFileCache::setDownloading(const Key & key, size_t offset, size_t size) -{ - std::lock_guard cache_lock(mutex); - - auto * cell = getCell(key, offset, cache_lock); - if (cell) - throw Exception( - ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, - "Cache cell already exists for key `{}` and offset {}", - keyToStr(key), offset); - - auto file_segments = splitRangeIntoCells(key, offset, size, FileSegment::State::DOWNLOADING, cache_lock); - return FileSegmentsHolder(std::move(file_segments)); -} - bool LRUFileCache::tryReserve( const Key & key_, size_t offset_, size_t size, std::lock_guard & cache_lock) { @@ -799,6 +784,23 @@ std::vector LRUFileCache::tryGetCachePaths(const Key & key) return cache_paths; } +FileSegmentPtr LRUFileCache::setDownloading(const Key & key, size_t offset, size_t size, std::lock_guard & cache_lock) +{ + auto * cell = getCell(key, offset, cache_lock); + if (cell) + throw Exception( + ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, + "Cache cell already exists for key `{}` and offset {}", + keyToStr(key), offset); + + cell = addCell(key, offset, size, FileSegment::State::DOWNLOADING, cache_lock); + + if (!cell) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to add a new cell for download"); + + return cell->file_segment; +} + LRUFileCache::FileSegmentCell::FileSegmentCell(FileSegmentPtr file_segment_, LRUQueue & queue_) : file_segment(file_segment_) { diff --git a/src/Common/FileCache.h b/src/Common/FileCache.h index 6a50dbab6d8..0e61162b917 100644 --- a/src/Common/FileCache.h +++ b/src/Common/FileCache.h @@ -26,6 +26,7 @@ class IFileCache : private boost::noncopyable { friend class FileSegment; friend struct FileSegmentsHolder; +friend class FileSegmentRangeWriter; public: using Key = UInt128; @@ -83,8 +84,6 @@ public: */ virtual FileSegmentsHolder get(const Key & key, size_t offset, size_t size) = 0; - virtual FileSegmentsHolder setDownloading(const Key & key, size_t offset, size_t size) = 0; - virtual FileSegments getSnapshot() const = 0; /// For debug. @@ -121,6 +120,8 @@ protected: std::lock_guard & cache_lock, std::lock_guard & segment_lock) = 0; + virtual FileSegmentPtr setDownloading(const Key & key, size_t offset, size_t size, std::lock_guard & cache_lock) = 0; + void assertInitialized() const; }; @@ -139,8 +140,6 @@ public: FileSegments getSnapshot() const override; - FileSegmentsHolder setDownloading(const Key & key, size_t offset, size_t size) override; - void initialize() override; void remove(const Key & key) override; @@ -229,6 +228,8 @@ private: void fillHolesWithEmptyFileSegments( FileSegments & file_segments, const Key & key, const FileSegment::Range & range, bool fill_with_detached_file_segments, std::lock_guard & cache_lock); + FileSegmentPtr setDownloading(const Key & key, size_t offset, size_t size, std::lock_guard & cache_lock) override; + public: struct Stat { diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index b5faa88fc92..5d77d38e947 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -51,6 +51,8 @@ FileSegment::FileSegment( /// needed, downloader is set on file segment creation). case (State::DOWNLOADING): { + /// On write-through cache we do not check downloader id. + is_write_through_cache = true; downloader_id = getCallerId(); break; } @@ -79,6 +81,12 @@ size_t FileSegment::getDownloadedSize() const return getDownloadedSize(segment_lock); } +size_t FileSegment::getAvailableSize() const +{ + std::lock_guard segment_lock(mutex); + return range().size() - downloaded_size; +} + size_t FileSegment::getDownloadedSize(std::lock_guard & /* segment_lock */) const { if (download_state == State::DOWNLOADED) @@ -208,7 +216,7 @@ void FileSegment::write(const char * from, size_t size, size_t offset_) ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Not enough space is reserved. Available: {}, expected: {}", availableSize(), size); - if (!isDownloader()) + if (!isDownloader() && !is_write_through_cache) throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Only downloader can do the downloading. (CallerId: {}, DownloaderId: {})", getCallerId(), downloader_id); @@ -266,81 +274,6 @@ void FileSegment::write(const char * from, size_t size, size_t offset_) assert(getDownloadOffset() == offset_ + size); } -void FileSegment::writeInMemory(const char * from, size_t size) -{ - if (!size) - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Attempt to write zero size cache file"); - - if (availableSize() < size) - throw Exception( - ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, - "Not enough space is reserved. Available: {}, expected: {}", availableSize(), size); - - std::lock_guard segment_lock(mutex); - - assertNotDetached(segment_lock); - - if (cache_writer) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache writer already initialized"); - - auto download_path = cache->getPathInLocalCache(key(), offset()); - cache_writer = std::make_unique(download_path, size + 1); - - try - { - cache_writer->write(from, size); - } - catch (Exception & e) - { - wrapWithCacheInfo(e, "while writing into cache", segment_lock); - - setDownloadFailed(segment_lock); - - cv.notify_all(); - - throw; - } -} - -size_t FileSegment::finalizeWrite() -{ - std::lock_guard segment_lock(mutex); - - if (!cache_writer) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache writer not initialized"); - - size_t size = cache_writer->offset(); - - if (size == 0) - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Writing zero size is not allowed"); - - assertNotDetached(segment_lock); - - try - { - cache_writer->next(); - } - catch (Exception & e) - { - wrapWithCacheInfo(e, "while writing into cache", segment_lock); - - setDownloadFailed(segment_lock); - - cv.notify_all(); - - throw; - } - - downloaded_size += size; - - if (downloaded_size != range().size()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected downloaded size to equal file segment size ({} == {})", downloaded_size, range().size()); - - setDownloaded(segment_lock); - - return size; -} - FileSegment::State FileSegment::wait() { std::unique_lock segment_lock(mutex); @@ -375,7 +308,8 @@ bool FileSegment::reserve(size_t size) assertNotDetached(segment_lock); auto caller_id = getCallerId(); - if (downloader_id != caller_id) + bool is_downloader = caller_id == downloader_id; + if (!is_downloader && !is_write_through_cache) throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Space can be reserved only by downloader (current: {}, expected: {})", caller_id, downloader_id); if (downloaded_size + size > range().size()) @@ -461,7 +395,7 @@ void FileSegment::complete(State state) assertNotDetached(segment_lock); bool is_downloader = isDownloaderImpl(segment_lock); - if (!is_downloader) + if (!is_downloader && !is_write_through_cache) { cv.notify_all(); throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, @@ -764,4 +698,116 @@ String FileSegmentsHolder::toString() } +FileSegmentRangeWriter::FileSegmentRangeWriter( + IFileCache * cache_, + const FileSegment::Key & key_, + size_t max_file_segment_size_) + : cache(cache_) + , key(key_) + , max_file_segment_size(max_file_segment_size_) +{ +} + +void FileSegmentRangeWriter::allocateFileSegment(size_t offset) +{ + std::lock_guard cache_lock(cache->mutex); + + auto file_segment = cache->setDownloading(key, offset, max_file_segment_size, cache_lock); + current_file_segment = file_segment; + + file_segments_holder.add(std::move(file_segment)); +} + +bool FileSegmentRangeWriter::write(char * data, size_t size, size_t offset) +{ + /** + * We want to write eventually some size, which is not known until the very end. + * Therefore we allocate file segments lazily. Each file segment is assigned capacity + * of max_file_segment_size, but reserved_size remains 0, until call to tryReserve(). + * Once current file segment is full (reached max_file_segment_size), we allocate a + * new file segment. All allocated file segments resize in file segments holder. + * If at the end of all writes, the last file segment is not full, then it is resized. + */ + + std::lock_guard lock(mutex); + + if (finalized) + return false; + + if (current_file_segment.expired()) + allocateFileSegment(current_file_segment_start_offset); + + auto file_segment = current_file_segment.lock(); + if (file_segment->getAvailableSize() == 0) + { + file_segment->complete(FileSegment::State::DOWNLOADED); + + allocateFileSegment(current_file_segment_start_offset); + file_segment = current_file_segment.lock(); + } + + bool reserved = file_segment->reserve(size); + if (!reserved) + return false; + + file_segment->write(data, size, offset); + current_file_segment_start_offset += size; + + return true; +} + +void FileSegmentRangeWriter::finalize() +{ + std::lock_guard lock(mutex); + + if (finalized) + return; + + if (file_segments_holder.file_segments.empty() || current_file_segment.expired()) + return; + + auto file_segment = current_file_segment.lock(); + + std::lock_guard cache_lock(cache->mutex); + file_segment->complete(cache_lock); + + finalized = true; +} + +FileSegmentRangeWriter::~FileSegmentRangeWriter() +{ + try + { + if (!finalized) + finalize(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } +} + +void FileSegmentRangeWriter::clearDownloaded() +{ + std::lock_guard lock(mutex); + + current_file_segment.reset(); + + auto & file_segments = file_segments_holder.file_segments; + if (file_segments.empty()) + return; + + std::lock_guard cache_lock(cache->mutex); + + for (auto file_segment_it = file_segments.begin(); file_segment_it != file_segments.end();) + { + auto file_segment = *file_segment_it; + + std::lock_guard segment_lock(file_segment->mutex); + cache->remove(key, file_segment->offset(), cache_lock, segment_lock); + } + + file_segments.clear(); +} + } diff --git a/src/Common/FileSegment.h b/src/Common/FileSegment.h index 2faab5eba51..4aa9a934333 100644 --- a/src/Common/FileSegment.h +++ b/src/Common/FileSegment.h @@ -23,6 +23,7 @@ class FileSegment : boost::noncopyable friend class LRUFileCache; friend struct FileSegmentsHolder; +friend class FileSegmentRangeWriter; public: using Key = UInt128; @@ -105,8 +106,6 @@ public: */ void writeInMemory(const char * from, size_t size); - size_t finalizeWrite(); - RemoteFileReaderPtr getRemoteFileReader(); void setRemoteFileReader(RemoteFileReaderPtr remote_file_reader_); @@ -129,6 +128,8 @@ public: size_t getDownloadedSize() const; + size_t getAvailableSize() const; + void completeBatchAndResetDownloader(); void complete(State state); @@ -215,18 +216,62 @@ private: std::atomic is_downloaded{false}; std::atomic hits_count = 0; /// cache hits. std::atomic ref_count = 0; /// Used for getting snapshot state + + bool is_write_through_cache = false; }; struct FileSegmentsHolder : private boost::noncopyable { + FileSegmentsHolder() = default; + explicit FileSegmentsHolder(FileSegments && file_segments_) : file_segments(std::move(file_segments_)) {} - FileSegmentsHolder(FileSegmentsHolder && other) : file_segments(std::move(other.file_segments)) {} + + FileSegmentsHolder(FileSegmentsHolder && other) noexcept : file_segments(std::move(other.file_segments)) {} ~FileSegmentsHolder(); + void add(FileSegmentPtr && file_segment) + { + file_segments.push_back(file_segment); + } + FileSegments file_segments{}; String toString(); }; +class FileSegmentRangeWriter +{ +public: + FileSegmentRangeWriter( + IFileCache * cache_, + const FileSegment::Key & key_, + size_t max_file_segment_size_); + + ~FileSegmentRangeWriter(); + + bool write(char * data, size_t size, size_t offset); + + void finalize(); + + /// If exception happened on remote fs write, we consider current cache invalid. + void clearDownloaded(); + +private: + void allocateFileSegment(size_t offset); + + IFileCache * cache; + FileSegment::Key key; + size_t max_file_segment_size; + + FileSegmentsHolder file_segments_holder; + + std::weak_ptr current_file_segment; + size_t current_file_segment_start_offset = 0; + + bool finalized = false; + + std::mutex mutex; +}; + } diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 514434bc3ed..345994f2a01 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -72,6 +72,7 @@ WriteBufferFromS3::WriteBufferFromS3( , s3_settings(s3_settings_) , schedule(std::move(schedule_)) , cache(cache_) + , cache_writer(cache_.get(), cache_->hash(key), /* max_file_segment_size */s3_settings.max_single_part_upload_size) { allocateBuffer(); } @@ -88,46 +89,14 @@ void WriteBufferFromS3::nextImpl() size_t size = offset(); temporary_buffer->write(working_buffer.begin(), size); + if (size && cacheEnabled()) + cache_writer.write(working_buffer.begin(), size, current_cache_write_offset); + current_cache_write_offset += size; + ThreadGroupStatusPtr running_group = CurrentThread::isInitialized() && CurrentThread::get().getThreadGroup() ? CurrentThread::get().getThreadGroup() : MainThreadStatus::getInstance().getThreadGroup(); - if (cacheEnabled()) - { - auto cache_key = cache->hash(key); - - file_segments_holder.emplace(cache->setDownloading(cache_key, current_download_offset, size)); - current_download_offset += size; - - size_t remaining_size = size; - auto & file_segments = file_segments_holder->file_segments; - for (auto file_segment_it = file_segments.begin(); file_segment_it != file_segments.end(); ++file_segment_it) - { - auto & file_segment = *file_segment_it; - size_t current_size = std::min(file_segment->range().size(), remaining_size); - remaining_size -= current_size; - - if (file_segment->reserve(current_size)) - { - file_segment->writeInMemory(working_buffer.begin(), current_size); - } - else - { - size_t upper_bound = file_segments.back()->range().right; - LOG_TRACE( - log, - "Space reservation failed, will skip caching for range: [{}, {}], current full range is [{}, {}]", - file_segment->range().left, upper_bound, file_segments.front()->range().right, upper_bound); - - for (auto reset_segment_it = file_segment_it; reset_segment_it != file_segments.end(); ++reset_segment_it) - (*reset_segment_it)->complete(FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); - file_segments.erase(file_segment_it, file_segments.end()); - - break; - } - } - } - ProfileEvents::increment(ProfileEvents::S3WriteBytes, offset()); last_part_size += offset(); @@ -143,8 +112,6 @@ void WriteBufferFromS3::nextImpl() allocateBuffer(); } - file_segments_holder.reset(); - waitForReadyBackGroundTasks(); } @@ -263,12 +230,6 @@ void WriteBufferFromS3::writePart() fillUploadRequest(task->req, part_number); - if (file_segments_holder) - { - task->cache_files.emplace(std::move(*file_segments_holder)); - file_segments_holder.reset(); - } - schedule([this, task]() { try @@ -280,8 +241,6 @@ void WriteBufferFromS3::writePart() task->exception = std::current_exception(); } - finalizeCacheIfNeeded(task->cache_files); - { std::lock_guard lock(bg_tasks_mutex); task->is_finised = true; @@ -298,14 +257,8 @@ void WriteBufferFromS3::writePart() { UploadPartTask task; fillUploadRequest(task.req, part_tags.size() + 1); - if (file_segments_holder) - { - task.cache_files.emplace(std::move(*file_segments_holder)); - file_segments_holder.reset(); - } processUploadRequest(task); part_tags.push_back(task.tag); - finalizeCacheIfNeeded(task.cache_files); } } @@ -394,11 +347,6 @@ void WriteBufferFromS3::makeSinglepartUpload() put_object_task = std::make_unique(); fillPutRequest(put_object_task->req); - if (file_segments_holder) - { - put_object_task->cache_files.emplace(std::move(*file_segments_holder)); - file_segments_holder.reset(); - } schedule([this]() { @@ -411,8 +359,6 @@ void WriteBufferFromS3::makeSinglepartUpload() put_object_task->exception = std::current_exception(); } - finalizeCacheIfNeeded(put_object_task->cache_files); - { std::lock_guard lock(bg_tasks_mutex); put_object_task->is_finised = true; @@ -428,13 +374,7 @@ void WriteBufferFromS3::makeSinglepartUpload() { PutObjectTask task; fillPutRequest(task.req); - if (file_segments_holder) - { - task.cache_files.emplace(std::move(*file_segments_holder)); - file_segments_holder.reset(); - } processPutRequest(task); - finalizeCacheIfNeeded(task.cache_files); } } @@ -462,25 +402,18 @@ void WriteBufferFromS3::processPutRequest(PutObjectTask & task) throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); } -void WriteBufferFromS3::finalizeCacheIfNeeded(std::optional & file_segments_holder) +void WriteBufferFromS3::clearCache() { - if (!file_segments_holder) + if (!cacheEnabled()) return; - auto & file_segments = file_segments_holder->file_segments; - for (auto file_segment_it = file_segments.begin(); file_segment_it != file_segments.end();) + try { - try - { - size_t size = (*file_segment_it)->finalizeWrite(); - file_segment_it = file_segments.erase(file_segment_it); - - ProfileEvents::increment(ProfileEvents::RemoteFSCacheDownloadBytes, size); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } + cache_writer.clearDownloaded(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); } } @@ -500,6 +433,7 @@ void WriteBufferFromS3::waitForReadyBackGroundTasks() if (exception) { waitForAllBackGroundTasks(); + clearCache(); std::rethrow_exception(exception); } @@ -520,7 +454,10 @@ void WriteBufferFromS3::waitForAllBackGroundTasks() { auto & task = upload_object_tasks.front(); if (task.exception) + { + clearCache(); std::rethrow_exception(task.exception); + } part_tags.push_back(task.tag); @@ -531,7 +468,10 @@ void WriteBufferFromS3::waitForAllBackGroundTasks() { bg_tasks_condvar.wait(lock, [this]() { return put_object_task->is_finised; }); if (put_object_task->exception) + { + clearCache(); std::rethrow_exception(put_object_task->exception); + } } } } diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 19faf0b1488..0b22b234cc8 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -119,9 +119,10 @@ private: Poco::Logger * log = &Poco::Logger::get("WriteBufferFromS3"); FileCachePtr cache; - size_t current_download_offset = 0; - std::optional file_segments_holder; - static void finalizeCacheIfNeeded(std::optional &); + FileSegmentRangeWriter cache_writer; + void clearCache(); + + size_t current_cache_write_offset = 0; }; } From 73f3334d80c9470fed81d260e004d1fca915556b Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 3 May 2022 00:27:35 +0200 Subject: [PATCH 051/171] Better fix for cache --- src/Common/FileSegment.cpp | 29 +++++++----- .../02293_force_drop_cache.reference | 1 + .../0_stateless/02293_force_drop_cache.sh | 45 +++++++++++++++++++ 3 files changed, 65 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/02293_force_drop_cache.reference create mode 100755 tests/queries/0_stateless/02293_force_drop_cache.sh diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index 5d77d38e947..d170a40079a 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -371,6 +371,8 @@ void FileSegment::completeBatchAndResetDownloader() { std::lock_guard segment_lock(mutex); + assertNotDetached(segment_lock); + if (!isDownloaderImpl(segment_lock)) { cv.notify_all(); @@ -583,15 +585,25 @@ void FileSegment::assertCorrectnessImpl(std::lock_guard & /* segment assert(download_state != FileSegment::State::DOWNLOADED || std::filesystem::file_size(cache->getPathInLocalCache(key(), offset())) > 0); } -void FileSegment::assertNotDetached(std::lock_guard & segment_lock) const +void FileSegment::assertNotDetached(std::lock_guard & /* segment_lock */) const { if (detached) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Operation not allowed, file segment is detached ({})", getInfoForLogImpl(segment_lock)); + { + throw Exception( + ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, + "Cache file segment is in detached state, operation not allowed. " + "It can happen when cache was concurrently dropped with SYSTEM DROP FILESYSTEM CACHE FORCE. " + "Please, retry"); + } } -void FileSegment::assertDetachedStatus(std::lock_guard & /* segment_lock */) const +void FileSegment::assertDetachedStatus(std::lock_guard & segment_lock) const { - assert(download_state == State::EMPTY || hasFinalizedState()); + if (download_state != State::EMPTY && !hasFinalizedState()) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, "Cache get into inconsistent state {}", getInfoForLogImpl(segment_lock)); + } } FileSegmentPtr FileSegment::getSnapshot(const FileSegmentPtr & file_segment, std::lock_guard & /* cache_lock */) @@ -618,17 +630,14 @@ bool FileSegment::hasFinalizedState() const || download_state == State::SKIP_CACHE; } -void FileSegment::detach(std::lock_guard & cache_lock, std::lock_guard & segment_lock) +void FileSegment::detach(std::lock_guard & /* cache_lock */, std::lock_guard & segment_lock) { if (detached) return; - if (!hasFinalizedState()) - { - completeUnlocked(cache_lock, segment_lock); - } - detached = true; + download_state = State::PARTIALLY_DOWNLOADED_NO_CONTINUATION; + downloader_id.clear(); LOG_TEST(log, "Detached file segment: {}", getInfoForLogImpl(segment_lock)); } diff --git a/tests/queries/0_stateless/02293_force_drop_cache.reference b/tests/queries/0_stateless/02293_force_drop_cache.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02293_force_drop_cache.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02293_force_drop_cache.sh b/tests/queries/0_stateless/02293_force_drop_cache.sh new file mode 100755 index 00000000000..17d5d74bc29 --- /dev/null +++ b/tests/queries/0_stateless/02293_force_drop_cache.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Tags: long + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test" + +$CLICKHOUSE_CLIENT --query "CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache', min_bytes_for_wide_part = 0" + +$CLICKHOUSE_CLIENT --query "INSERT INTO test SELECT * FROM generateRandom('key UInt32, value String') LIMIT 1000000" + +function select_thread() +{ + $CLICKHOUSE_CLIENT --query "SELECT * FROM test FORMAT Null" +} + +function drop_thread() +{ + $CLICKHOUSE_CLIENT --query "SYSTEM DROP FILESYSTEM CACHE FORCE" +} + +export -f select_thread +export -f drop_thread + +TIMEOUT=30 + +clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & + +clickhouse_client_loop_timeout $TIMEOUT drop_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT drop_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT drop_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT drop_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT drop_thread 2> /dev/null & + +wait + +$CLICKHOUSE_CLIENT --query "SELECT 1" + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test" From a3c9ff7fe8c0537302e3dd57b6bda0034c0b8611 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 3 May 2022 11:45:22 +0200 Subject: [PATCH 052/171] Tiny changes --- src/Common/FileCache.cpp | 14 +++++++------- src/Common/FileSegment.cpp | 4 ++-- src/IO/WriteBufferFromS3.cpp | 3 +++ .../queries/0_stateless/02293_force_drop_cache.sh | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/Common/FileCache.cpp b/src/Common/FileCache.cpp index 7dedf1c84a1..3d5ffbfb845 100644 --- a/src/Common/FileCache.cpp +++ b/src/Common/FileCache.cpp @@ -386,7 +386,7 @@ LRUFileCache::FileSegmentCell * LRUFileCache::addCell( if (files[key].contains(offset)) throw Exception( - ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, + ErrorCodes::LOGICAL_ERROR, "Cache already exists for key: `{}`, offset: {}, size: {}.\nCurrent cache structure: {}", keyToStr(key), offset, size, dumpStructureImpl(key, cache_lock)); @@ -445,7 +445,7 @@ bool LRUFileCache::tryReserve( auto * cell = getCell(key, offset, cache_lock); if (!cell) - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache became inconsistent. Key: {}, offset: {}", keyToStr(key), offset); size_t cell_size = cell->size(); @@ -576,7 +576,7 @@ void LRUFileCache::remove( auto * cell = getCell(key, offset, cache_lock); if (!cell) - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "No cache cell for key: {}, offset: {}", keyToStr(key), offset); + throw Exception(ErrorCodes::LOGICAL_ERROR, "No cache cell for key: {}, offset: {}", keyToStr(key), offset); if (cell->queue_iterator) queue.erase(*cell->queue_iterator); @@ -692,7 +692,7 @@ LRUFileCache::Stat LRUFileCache::getStat() { const auto * cell = getCell(key, offset, cache_lock); if (!cell) - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache became inconsistent. Key: {}, offset: {}", keyToStr(key), offset); switch (cell->file_segment->download_state) @@ -727,13 +727,13 @@ void LRUFileCache::reduceSizeToDownloaded( auto * cell = getCell(key, offset, cache_lock); if (!cell) - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "No cell found for key: {}, offset: {}", keyToStr(key), offset); + throw Exception(ErrorCodes::LOGICAL_ERROR, "No cell found for key: {}, offset: {}", keyToStr(key), offset); const auto & file_segment = cell->file_segment; size_t downloaded_size = file_segment->downloaded_size; if (downloaded_size == file_segment->range().size()) - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, + throw Exception(ErrorCodes::LOGICAL_ERROR, "Nothing to reduce, file segment fully downloaded, key: {}, offset: {}", keyToStr(key), offset); cell->file_segment = std::make_shared(offset, downloaded_size, key, this, FileSegment::State::DOWNLOADED); @@ -789,7 +789,7 @@ FileSegmentPtr LRUFileCache::setDownloading(const Key & key, size_t offset, size auto * cell = getCell(key, offset, cache_lock); if (cell) throw Exception( - ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, + ErrorCodes::LOGICAL_ERROR, "Cache cell already exists for key `{}` and offset {}", keyToStr(key), offset); diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index d170a40079a..d135e30cbcd 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -217,7 +217,7 @@ void FileSegment::write(const char * from, size_t size, size_t offset_) "Not enough space is reserved. Available: {}, expected: {}", availableSize(), size); if (!isDownloader() && !is_write_through_cache) - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, + throw Exception(ErrorCodes::LOGICAL_ERROR, "Only downloader can do the downloading. (CallerId: {}, DownloaderId: {})", getCallerId(), downloader_id); @@ -310,7 +310,7 @@ bool FileSegment::reserve(size_t size) auto caller_id = getCallerId(); bool is_downloader = caller_id == downloader_id; if (!is_downloader && !is_write_through_cache) - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Space can be reserved only by downloader (current: {}, expected: {})", caller_id, downloader_id); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Space can be reserved only by downloader (current: {}, expected: {})", caller_id, downloader_id); if (downloaded_size + size > range().size()) throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 345994f2a01..ebc43801c39 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -90,7 +90,10 @@ void WriteBufferFromS3::nextImpl() temporary_buffer->write(working_buffer.begin(), size); if (size && cacheEnabled()) + { cache_writer.write(working_buffer.begin(), size, current_cache_write_offset); + ProfileEvents::increment(ProfileEvents::RemoteFSCacheDownloadBytes, size); + } current_cache_write_offset += size; ThreadGroupStatusPtr running_group = CurrentThread::isInitialized() && CurrentThread::get().getThreadGroup() diff --git a/tests/queries/0_stateless/02293_force_drop_cache.sh b/tests/queries/0_stateless/02293_force_drop_cache.sh index 17d5d74bc29..34416feec5e 100755 --- a/tests/queries/0_stateless/02293_force_drop_cache.sh +++ b/tests/queries/0_stateless/02293_force_drop_cache.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long +# Tags: long, no-parallel, no-s3-storage CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From b0d43e3f40682e7168c854d56eb6d826f1fe4910 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 3 May 2022 13:15:27 +0200 Subject: [PATCH 053/171] Better --- src/Common/FileSegment.cpp | 46 +++++++++++++++++++++++++----------- src/Common/FileSegment.h | 2 ++ src/IO/WriteBufferFromS3.cpp | 3 +++ 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index d135e30cbcd..238541cc1f7 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -115,8 +115,7 @@ String FileSegment::getOrSetDownloader() { std::lock_guard segment_lock(mutex); - if (detached) - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Cannot set downloader for a detached file segment"); + assertNotDetached(segment_lock); if (downloader_id.empty()) { @@ -140,6 +139,8 @@ void FileSegment::resetDownloader() { std::lock_guard segment_lock(mutex); + assertNotDetached(segment_lock); + if (downloader_id.empty()) throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "There is no downloader"); @@ -168,6 +169,7 @@ String FileSegment::getDownloader() const bool FileSegment::isDownloader() const { std::lock_guard segment_lock(mutex); + return getCallerId() == downloader_id; } @@ -232,8 +234,12 @@ void FileSegment::write(const char * from, size_t size, size_t offset_) "Attempt to write {} bytes to offset: {}, but current download offset is {}", size, offset_, download_offset); + std::unique_lock detach_lock(detach_mutex, std::defer_lock); + { std::lock_guard segment_lock(mutex); + detach_lock.lock(); + assertNotDetached(segment_lock); } @@ -278,6 +284,9 @@ FileSegment::State FileSegment::wait() { std::unique_lock segment_lock(mutex); + if (detached) + throwDetached(); + if (downloader_id.empty()) return download_state; @@ -302,8 +311,11 @@ bool FileSegment::reserve(size_t size) if (!size) throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Zero space reservation is not allowed"); + std::unique_lock detach_lock(detach_mutex, std::defer_lock); + { std::lock_guard segment_lock(mutex); + detach_lock.lock(); assertNotDetached(segment_lock); @@ -437,6 +449,9 @@ void FileSegment::complete(State state) void FileSegment::complete(std::lock_guard & cache_lock) { std::lock_guard segment_lock(mutex); + + assertNotDetached(segment_lock); + completeUnlocked(cache_lock, segment_lock); } @@ -585,16 +600,19 @@ void FileSegment::assertCorrectnessImpl(std::lock_guard & /* segment assert(download_state != FileSegment::State::DOWNLOADED || std::filesystem::file_size(cache->getPathInLocalCache(key(), offset())) > 0); } +void FileSegment::throwDetached() +{ + throw Exception( + ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, + "Cache file segment is in detached state, operation not allowed. " + "It can happen when cache was concurrently dropped with SYSTEM DROP FILESYSTEM CACHE FORCE. " + "Please, retry"); +} + void FileSegment::assertNotDetached(std::lock_guard & /* segment_lock */) const { if (detached) - { - throw Exception( - ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, - "Cache file segment is in detached state, operation not allowed. " - "It can happen when cache was concurrently dropped with SYSTEM DROP FILESYSTEM CACHE FORCE. " - "Please, retry"); - } + throwDetached(); } void FileSegment::assertDetachedStatus(std::lock_guard & segment_lock) const @@ -635,6 +653,8 @@ void FileSegment::detach(std::lock_guard & /* cache_lock */, std::lo if (detached) return; + std::lock_guard detach_lock(detach_mutex); + detached = true; download_state = State::PARTIALLY_DOWNLOADED_NO_CONTINUATION; downloader_id.clear(); @@ -658,14 +678,17 @@ FileSegmentsHolder::~FileSegmentsHolder() if (!cache) cache = file_segment->cache; + try { bool detached = false; + { std::lock_guard segment_lock(file_segment->mutex); detached = file_segment->isDetached(segment_lock); if (detached) file_segment->assertDetachedStatus(segment_lock); } + if (detached) { /// This file segment is not owned by cache, so it will be destructed @@ -674,10 +697,6 @@ FileSegmentsHolder::~FileSegmentsHolder() continue; } - } - - try - { /// File segment pointer must be reset right after calling complete() and /// under the same mutex, because complete() checks for segment pointers. std::lock_guard cache_lock(cache->mutex); @@ -689,7 +708,6 @@ FileSegmentsHolder::~FileSegmentsHolder() catch (...) { tryLogCurrentException(__PRETTY_FUNCTION__); - assert(false); } } } diff --git a/src/Common/FileSegment.h b/src/Common/FileSegment.h index 4aa9a934333..15a9a8871b3 100644 --- a/src/Common/FileSegment.h +++ b/src/Common/FileSegment.h @@ -158,6 +158,7 @@ private: void assertDetachedStatus(std::lock_guard & segment_lock) const; bool hasFinalizedState() const; bool isDetached(std::lock_guard & /* segment_lock */) const { return detached; } + [[noreturn]] static void throwDetached(); void setDownloaded(std::lock_guard & segment_lock); void setDownloadFailed(std::lock_guard & segment_lock); @@ -218,6 +219,7 @@ private: std::atomic ref_count = 0; /// Used for getting snapshot state bool is_write_through_cache = false; + std::mutex detach_mutex; }; struct FileSegmentsHolder : private boost::noncopyable diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index ebc43801c39..18431d4ca1f 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -171,6 +171,9 @@ void WriteBufferFromS3::finalizeImpl() if (!multipart_upload_id.empty()) completeMultipartUpload(); + + if (cacheEnabled()) + cache_writer.finalize(); } void WriteBufferFromS3::createMultipartUpload() From 0266cdf12533a6ce9225c9c2ecd72b3407d73456 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 3 May 2022 18:16:50 +0200 Subject: [PATCH 054/171] Add entries to cache log when cache was not event attempted --- src/Common/getQueryId.h | 13 +++++++ ...chronousReadIndirectBufferFromRemoteFS.cpp | 3 +- src/Disks/IO/CachedReadBufferFromRemoteFS.cpp | 11 ++---- src/Disks/IO/ReadBufferFromRemoteFSGather.cpp | 38 +++++++++++++++++-- src/Disks/IO/ReadBufferFromRemoteFSGather.h | 13 +++++++ src/IO/ReadSettings.h | 1 + src/Interpreters/Context.cpp | 1 + src/Interpreters/FilesystemCacheLog.cpp | 2 + src/Interpreters/FilesystemCacheLog.h | 1 + 9 files changed, 70 insertions(+), 13 deletions(-) create mode 100644 src/Common/getQueryId.h diff --git a/src/Common/getQueryId.h b/src/Common/getQueryId.h new file mode 100644 index 00000000000..32f1937ecb0 --- /dev/null +++ b/src/Common/getQueryId.h @@ -0,0 +1,13 @@ +#include + +namespace DB +{ + +static String getQueryId() +{ + if (!CurrentThread::isInitialized() || !CurrentThread::get().getQueryContext()) + return ""; + return CurrentThread::getQueryId().toString(); +} + +} diff --git a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp index d1e4b9fa21e..8ac576be61c 100644 --- a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp @@ -1,10 +1,10 @@ #include "AsynchronousReadIndirectBufferFromRemoteFS.h" #include +#include #include #include #include -#include namespace CurrentMetrics @@ -57,7 +57,6 @@ AsynchronousReadIndirectBufferFromRemoteFS::AsynchronousReadIndirectBufferFromRe ProfileEvents::increment(ProfileEvents::RemoteFSBuffers); } - String AsynchronousReadIndirectBufferFromRemoteFS::getFileName() const { return impl->getFileName(); diff --git a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp index 66b918e7e12..575cb6a4162 100644 --- a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp +++ b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace ProfileEvents @@ -23,13 +24,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -static String getQueryId() -{ - if (!CurrentThread::isInitialized() || !CurrentThread::get().getQueryContext() || CurrentThread::getQueryId().size == 0) - return ""; - return CurrentThread::getQueryId().toString(); -} - CachedReadBufferFromRemoteFS::CachedReadBufferFromRemoteFS( const String & remote_fs_object_path_, FileCachePtr cache_, @@ -49,7 +43,7 @@ CachedReadBufferFromRemoteFS::CachedReadBufferFromRemoteFS( , read_until_position(read_until_position_) , remote_file_reader_creator(remote_file_reader_creator_) , query_id(getQueryId()) - , enable_logging(!query_id.empty() && CurrentThread::get().getQueryContext()->getSettingsRef().enable_filesystem_cache_log) + , enable_logging(!query_id.empty() && settings_.enable_filesystem_cache_log) { } @@ -63,6 +57,7 @@ void CachedReadBufferFromRemoteFS::appendFilesystemCacheLog( .source_file_path = remote_fs_object_path, .file_segment_range = { file_segment_range.left, file_segment_range.right }, .file_segment_size = file_segment_range.size(), + .cache_attempted = true, }; switch (type) diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp index 4399befa433..39b82131aa9 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp @@ -21,9 +21,12 @@ #include #include #include +#include +#include namespace fs = std::filesystem; + namespace DB { @@ -39,9 +42,6 @@ SeekableReadBufferPtr ReadBufferFromS3Gather::createImplementationBuffer(const S auto remote_path = fs::path(common_path_prefix) / path; auto cache = settings.remote_fs_cache; - bool with_cache = cache - && settings.enable_filesystem_cache - && (!IFileCache::isReadOnly() || settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache); auto remote_file_reader_creator = [=, this]() { @@ -95,7 +95,30 @@ ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather( , blobs_to_read(blobs_to_read_) , settings(settings_) , log(&Poco::Logger::get("ReadBufferFromRemoteFSGather")) + , query_id(getQueryId()) + , enable_cache_log(!query_id.empty() && settings.enable_filesystem_cache_log) { + with_cache = settings.remote_fs_cache + && settings.enable_filesystem_cache + && (!IFileCache::isReadOnly() || settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache); +} + + +void ReadBufferFromRemoteFSGather::appendFilesystemCacheLog() +{ + FilesystemCacheLogElement elem + { + .event_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()), + .query_id = query_id, + .source_file_path = getFileName(), + .file_segment_range = { 0, total_bytes_read }, + .read_type = FilesystemCacheLogElement::ReadType::READ_FROM_FS_BYPASSING_CACHE, + .file_segment_size = total_bytes_read, + .cache_attempted = false, + }; + + if (auto cache_log = Context::getGlobalContextInstance()->getFilesystemCacheLog()) + cache_log->add(elem); } @@ -199,6 +222,7 @@ bool ReadBufferFromRemoteFSGather::readImpl() */ if (bytes_to_ignore) { + total_bytes_read += bytes_to_ignore; current_buf->ignore(bytes_to_ignore); result = current_buf->hasPendingData(); file_offset_of_buffer_end += bytes_to_ignore; @@ -225,6 +249,7 @@ bool ReadBufferFromRemoteFSGather::readImpl() { assert(available()); nextimpl_working_buffer_offset = offset(); + total_bytes_read += available(); } return result; @@ -282,5 +307,12 @@ size_t ReadBufferFromRemoteFSGather::getImplementationBufferOffset() const return current_buf->getFileOffsetOfBufferEnd(); } +ReadBufferFromRemoteFSGather::~ReadBufferFromRemoteFSGather() +{ + if (!with_cache && enable_cache_log) + { + appendFilesystemCacheLog(); + } +} } diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.h b/src/Disks/IO/ReadBufferFromRemoteFSGather.h index da2d1dee4b2..67df9104139 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.h +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.h @@ -29,6 +29,9 @@ public: const std::string & common_path_prefix_, const BlobsPathToSize & blobs_to_read_, const ReadSettings & settings_); + + ~ReadBufferFromRemoteFSGather() override; + String getFileName() const; void reset(); @@ -66,6 +69,8 @@ protected: String current_path; + bool with_cache; + private: bool nextImpl() override; @@ -75,6 +80,8 @@ private: bool moveToNextBuffer(); + void appendFilesystemCacheLog(); + SeekableReadBufferPtr current_buf; size_t current_buf_idx = 0; @@ -89,6 +96,12 @@ private: size_t bytes_to_ignore = 0; Poco::Logger * log; + + String query_id; + + size_t total_bytes_read = 0; + + bool enable_cache_log = false; }; diff --git a/src/IO/ReadSettings.h b/src/IO/ReadSettings.h index 92346615a7a..6a154c60987 100644 --- a/src/IO/ReadSettings.h +++ b/src/IO/ReadSettings.h @@ -80,6 +80,7 @@ struct ReadSettings bool enable_filesystem_cache = true; size_t filesystem_cache_max_wait_sec = 1; bool read_from_filesystem_cache_if_exists_otherwise_bypass_cache = false; + bool enable_filesystem_cache_log = false; size_t remote_read_min_bytes_for_seek = DBMS_DEFAULT_BUFFER_SIZE; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 84f18d66196..293217d7a16 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -3407,6 +3407,7 @@ ReadSettings Context::getReadSettings() const res.enable_filesystem_cache = settings.enable_filesystem_cache; res.filesystem_cache_max_wait_sec = settings.filesystem_cache_max_wait_sec; res.read_from_filesystem_cache_if_exists_otherwise_bypass_cache = settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache; + res.enable_filesystem_cache_log = settings.enable_filesystem_cache_log; res.remote_read_min_bytes_for_seek = settings.remote_read_min_bytes_for_seek; diff --git a/src/Interpreters/FilesystemCacheLog.cpp b/src/Interpreters/FilesystemCacheLog.cpp index 9eec65a38ab..a8470ca15ba 100644 --- a/src/Interpreters/FilesystemCacheLog.cpp +++ b/src/Interpreters/FilesystemCacheLog.cpp @@ -37,6 +37,7 @@ NamesAndTypesList FilesystemCacheLogElement::getNamesAndTypes() {"file_segment_range", std::make_shared(std::move(types))}, {"size", std::make_shared()}, {"read_type", std::make_shared()}, + {"cache_attempted", std::make_shared()}, }; } @@ -53,6 +54,7 @@ void FilesystemCacheLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(Tuple{file_segment_range.first, file_segment_range.second}); columns[i++]->insert(file_segment_size); columns[i++]->insert(typeToString(read_type)); + columns[i++]->insert(cache_attempted); } }; diff --git a/src/Interpreters/FilesystemCacheLog.h b/src/Interpreters/FilesystemCacheLog.h index 208f53a76cf..bef5a82d39b 100644 --- a/src/Interpreters/FilesystemCacheLog.h +++ b/src/Interpreters/FilesystemCacheLog.h @@ -39,6 +39,7 @@ struct FilesystemCacheLogElement std::pair file_segment_range{}; ReadType read_type{}; size_t file_segment_size; + bool cache_attempted; static std::string name() { return "FilesystemCacheLog"; } From e21aadcfcad72780595237162853844aa889c944 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 3 May 2022 19:16:58 +0200 Subject: [PATCH 055/171] Update getQueryId.h --- src/Common/getQueryId.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Common/getQueryId.h b/src/Common/getQueryId.h index 32f1937ecb0..96dcbb5cb22 100644 --- a/src/Common/getQueryId.h +++ b/src/Common/getQueryId.h @@ -1,3 +1,4 @@ +#pragma once #include namespace DB From c8b5decf298fa5c124f1073ae76dfcd40f3299a5 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 4 May 2022 14:18:52 +0200 Subject: [PATCH 056/171] Fix lock inversion --- src/Common/FileCache.cpp | 6 ++++-- src/Common/FileSegment.cpp | 17 +++++++---------- src/Common/FileSegment.h | 16 ++++++++++++---- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/Common/FileCache.cpp b/src/Common/FileCache.cpp index 7a77bc7e5c5..1a213d96caa 100644 --- a/src/Common/FileCache.cpp +++ b/src/Common/FileCache.cpp @@ -563,8 +563,10 @@ void LRUFileCache::remove(bool force_remove_unreleasable) auto file_segment = cell->file_segment; if (file_segment) { - std::lock_guard segment_lock(file_segment->mutex); - file_segment->detach(cache_lock, segment_lock); + std::lock_guard detach_lock(file_segment->detach_mutex); + std::lock_guard segment_lock(file_segment->mutex); + + file_segment->detach(cache_lock, detach_lock, segment_lock); remove(file_segment->key(), file_segment->offset(), cache_lock, segment_lock); } } diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index 238541cc1f7..cce47ab336b 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -234,12 +234,10 @@ void FileSegment::write(const char * from, size_t size, size_t offset_) "Attempt to write {} bytes to offset: {}, but current download offset is {}", size, offset_, download_offset); - std::unique_lock detach_lock(detach_mutex, std::defer_lock); + std::lock_guard detach_lock(detach_mutex); { std::lock_guard segment_lock(mutex); - detach_lock.lock(); - assertNotDetached(segment_lock); } @@ -311,12 +309,11 @@ bool FileSegment::reserve(size_t size) if (!size) throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Zero space reservation is not allowed"); - std::unique_lock detach_lock(detach_mutex, std::defer_lock); + std::lock_guard cache_lock(cache->mutex); + std::lock_guard detach_lock(detach_mutex); { std::lock_guard segment_lock(mutex); - detach_lock.lock(); - assertNotDetached(segment_lock); auto caller_id = getCallerId(); @@ -340,7 +337,6 @@ bool FileSegment::reserve(size_t size) size_t free_space = reserved_size - downloaded_size; size_t size_to_reserve = size - free_space; - std::lock_guard cache_lock(cache->mutex); bool reserved = cache->tryReserve(key(), offset(), size_to_reserve, cache_lock); if (reserved) @@ -648,13 +644,14 @@ bool FileSegment::hasFinalizedState() const || download_state == State::SKIP_CACHE; } -void FileSegment::detach(std::lock_guard & /* cache_lock */, std::lock_guard & segment_lock) +void FileSegment::detach( + std::lock_guard & /* cache_lock */, + std::lock_guard & /* detach_lock */, + std::lock_guard & segment_lock) { if (detached) return; - std::lock_guard detach_lock(detach_mutex); - detached = true; download_state = State::PARTIALLY_DOWNLOADED_NO_CONTINUATION; downloader_id.clear(); diff --git a/src/Common/FileSegment.h b/src/Common/FileSegment.h index bc8847dbef6..c59ddbd3380 100644 --- a/src/Common/FileSegment.h +++ b/src/Common/FileSegment.h @@ -146,7 +146,10 @@ public: static FileSegmentPtr getSnapshot(const FileSegmentPtr & file_segment, std::lock_guard & cache_lock); - void detach(std::lock_guard & cache_lock, std::lock_guard & segment_lock); + void detach( + std::lock_guard & cache_lock, + std::lock_guard & /* detach_lock */, + std::lock_guard & segment_lock); private: size_t availableSize() const { return reserved_size - downloaded_size; } @@ -195,9 +198,17 @@ private: size_t downloaded_size = 0; size_t reserved_size = 0; + /// global locking order rule: + /// 1. cache lock + /// 2. detach lock (if needed) + /// 3. segment lock + mutable std::mutex mutex; std::condition_variable cv; + bool is_write_through_cache = false; + std::mutex detach_mutex; + /// Protects downloaded_size access with actual write into fs. /// downloaded_size is not protected by download_mutex in methods which /// can never be run in parallel to FileSegment::write() method @@ -218,9 +229,6 @@ private: std::atomic is_downloaded{false}; std::atomic hits_count = 0; /// cache hits. std::atomic ref_count = 0; /// Used for getting snapshot state - - bool is_write_through_cache = false; - std::mutex detach_mutex; }; struct FileSegmentsHolder : private boost::noncopyable From 840ecc68ae4c90abd6316ab544a5477fc2121e2a Mon Sep 17 00:00:00 2001 From: Vxider Date: Thu, 5 May 2022 14:27:12 +0800 Subject: [PATCH 057/171] wv-fix-multiple-columns --- src/Storages/WindowView/StorageWindowView.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index 8669406f2ce..fb491a58722 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -1354,6 +1354,18 @@ void StorageWindowView::writeIntoWindowView( auto metadata_snapshot = inner_storage->getInMemoryMetadataPtr(); auto output = inner_storage->write(window_view.getMergeableQuery(), metadata_snapshot, local_context); + auto actions_dag = ActionsDAG::makeConvertingActions( + builder.getHeader().getColumnsWithTypeAndName(), + output->getHeader().getColumnsWithTypeAndName(), + ActionsDAG::MatchColumnsMode::Position); + auto actions + = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); + + builder.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, actions); + }); + builder.addChain(Chain(std::move(output))); builder.setSinks([&](const Block & cur_header, Pipe::StreamType) { From ffe1e3a5a1fcd1c4db06a96f2078481d16f0a465 Mon Sep 17 00:00:00 2001 From: Vxider Date: Thu, 5 May 2022 17:43:23 +0800 Subject: [PATCH 058/171] update --- src/Storages/WindowView/StorageWindowView.cpp | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index fb491a58722..2f8abb48528 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -1354,17 +1354,17 @@ void StorageWindowView::writeIntoWindowView( auto metadata_snapshot = inner_storage->getInMemoryMetadataPtr(); auto output = inner_storage->write(window_view.getMergeableQuery(), metadata_snapshot, local_context); - auto actions_dag = ActionsDAG::makeConvertingActions( - builder.getHeader().getColumnsWithTypeAndName(), - output->getHeader().getColumnsWithTypeAndName(), - ActionsDAG::MatchColumnsMode::Position); - auto actions - = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); - - builder.addSimpleTransform([&](const Block & header) + if (!blocksHaveEqualStructure(builder.getHeader(), output->getHeader())) { - return std::make_shared(header, actions); - }); + auto convert_actions_dag = ActionsDAG::makeConvertingActions( + builder.getHeader().getColumnsWithTypeAndName(), + output->getHeader().getColumnsWithTypeAndName(), + ActionsDAG::MatchColumnsMode::Name); + auto convert_actions = std::make_shared( + convert_actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); + + builder.addSimpleTransform([&](const Block & header) { return std::make_shared(header, convert_actions); }); + } builder.addChain(Chain(std::move(output))); builder.setSinks([&](const Block & cur_header, Pipe::StreamType) From f5c16abfda9d3b3e48f6fed87e2f4cd976fa0c0d Mon Sep 17 00:00:00 2001 From: Vxider Date: Thu, 5 May 2022 17:44:12 +0800 Subject: [PATCH 059/171] add test --- ...ow_view_multiple_columns_groupby.reference | 1 + ...72_window_view_multiple_columns_groupby.sh | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.reference create mode 100755 tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh diff --git a/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.reference b/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.reference new file mode 100644 index 00000000000..828667becf2 --- /dev/null +++ b/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.reference @@ -0,0 +1 @@ +test1 test2 diff --git a/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh b/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh new file mode 100755 index 00000000000..3e2eda96f93 --- /dev/null +++ b/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --multiquery < Date: Thu, 5 May 2022 13:56:16 +0000 Subject: [PATCH 060/171] Support distributed queries --- src/Interpreters/ActionsDAG.cpp | 21 ++ src/Interpreters/ActionsDAG.h | 2 + src/Interpreters/Aggregator.cpp | 8 +- src/Interpreters/Aggregator.h | 61 +++-- src/Interpreters/ExpressionActions.cpp | 12 + src/Interpreters/ExpressionAnalyzer.cpp | 14 +- src/Interpreters/InterpreterSelectQuery.cpp | 47 ++-- src/Interpreters/InterpreterSelectQuery.h | 5 +- src/Processors/QueryPlan/AggregatingStep.cpp | 215 +++++++++--------- src/Processors/QueryPlan/AggregatingStep.h | 2 + .../Transforms/GroupingSetsTransform.cpp | 14 +- .../Transforms/GroupingSetsTransform.h | 2 +- .../02165_replicated_grouping_sets.reference | 10 + .../02165_replicated_grouping_sets.sql | 31 +++ 14 files changed, 274 insertions(+), 170 deletions(-) create mode 100644 tests/queries/0_stateless/02165_replicated_grouping_sets.reference create mode 100644 tests/queries/0_stateless/02165_replicated_grouping_sets.sql diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index b367af87adf..828e7171798 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -234,6 +234,18 @@ const ActionsDAG::Node & ActionsDAG::addFunction( return addNode(std::move(node)); } +const ActionsDAG::Node & ActionsDAG::addGroupingSetColumn() +{ + Node node; + node.type = ActionType::GROUPING_SET; + node.result_type = std::make_shared(); + node.result_name = "__grouping_set"; + + auto & res = nodes.emplace_back(std::move(node)); + index.emplace_back(&res); + return res; +} + const ActionsDAG::Node & ActionsDAG::findInIndex(const std::string & name) const { if (const auto * node = tryFindInIndex(name)) @@ -493,6 +505,11 @@ static ColumnWithTypeAndName executeActionForHeader(const ActionsDAG::Node * nod { break; } + + case ActionsDAG::ActionType::GROUPING_SET: + { + break; + } } return res_column; @@ -923,6 +940,10 @@ std::string ActionsDAG::dumpDAG() const case ActionsDAG::ActionType::INPUT: out << "INPUT "; break; + + case ActionsDAG::ActionType::GROUPING_SET: + out << "GROUPING SET "; + break; } out << "("; diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index d11dfc3b8ad..31b36277c77 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -57,6 +57,7 @@ public: /// Function arrayJoin. Specially separated because it changes the number of rows. ARRAY_JOIN, FUNCTION, + GROUPING_SET, }; struct Node; @@ -132,6 +133,7 @@ public: const FunctionOverloadResolverPtr & function, NodeRawConstPtrs children, std::string result_name); + const Node & addGroupingSetColumn(); /// Index can contain any column returned from DAG. /// You may manually change it if needed. diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index bfc984e7b00..62548f71b4b 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -372,7 +372,7 @@ Block Aggregator::Params::getHeader( const Block & intermediate_header, const ColumnNumbers & keys, const AggregateDescriptions & aggregates, - const GroupingSetsParams & grouping_sets_params, + bool has_grouping_sets, bool final) { Block res; @@ -397,6 +397,9 @@ Block Aggregator::Params::getHeader( for (const auto & key : keys) res.insert(src_header.safeGetByPosition(key).cloneEmpty()); + if (has_grouping_sets) + res.insert({ std::make_shared(), "__grouping_set" }); + for (const auto & aggregate : aggregates) { size_t arguments_size = aggregate.arguments.size(); @@ -412,9 +415,6 @@ Block Aggregator::Params::getHeader( res.insert({ type, aggregate.column_name }); } - - if (grouping_sets_params.isValid()) - res.insert({ std::make_shared(), "__grouping_set" }); } return materializeBlock(res); diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 82e252b0b05..8064ca2f588 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -872,6 +872,21 @@ using ManyAggregatedDataVariantsPtr = std::shared_ptr; + /** How are "total" values calculated with WITH TOTALS? * (For more details, see TotalsHavingTransform.) * @@ -951,22 +966,7 @@ public: }; StatsCollectingParams stats_collecting_params; - struct GroupingSetsParams - { - GroupingSetsParams() = default; - - GroupingSetsParams(ColumnNumbersList grouping_sets_with_keys_, ColumnNumbersList missing_columns_per_set_) - : grouping_sets_with_keys(std::move(grouping_sets_with_keys_)) - , missing_columns_per_set(std::move(missing_columns_per_set_)) - {} - - bool isValid() const noexcept { return !grouping_sets_with_keys.empty(); } - - size_t size() const noexcept { return grouping_sets_with_keys.size(); } - - ColumnNumbersList grouping_sets_with_keys; - ColumnNumbersList missing_columns_per_set; - } grouping_sets_params; + bool has_grouping_sets; Params( const Block & src_header_, @@ -986,7 +986,7 @@ public: size_t min_count_to_compile_aggregate_expression_, const Block & intermediate_header_ = {}, const StatsCollectingParams & stats_collecting_params_ = {}, - const GroupingSetsParams & grouping_sets_params_ = {}) + bool has_grouping_sets_ = false) : src_header(src_header_) , intermediate_header(intermediate_header_) , keys(keys_) @@ -1006,14 +1006,19 @@ public: , compile_aggregate_expressions(compile_aggregate_expressions_) , min_count_to_compile_aggregate_expression(min_count_to_compile_aggregate_expression_) , stats_collecting_params(stats_collecting_params_) - , grouping_sets_params(grouping_sets_params_) + , has_grouping_sets(has_grouping_sets_) { } /// Only parameters that matter during merge. - Params(const Block & intermediate_header_, - const ColumnNumbers & keys_, const AggregateDescriptions & aggregates_, bool overflow_row_, size_t max_threads_) - : Params(Block(), keys_, aggregates_, overflow_row_, 0, OverflowMode::THROW, 0, 0, 0, false, nullptr, max_threads_, 0, false, 0) + Params( + const Block & intermediate_header_, + const ColumnNumbers & keys_, + const AggregateDescriptions & aggregates_, + bool overflow_row_, + bool has_grouping_sets_, + size_t max_threads_) + : Params(Block(), keys_, aggregates_, overflow_row_, 0, OverflowMode::THROW, 0, 0, 0, false, nullptr, max_threads_, 0, false, 0, {}, {}, has_grouping_sets_) { intermediate_header = intermediate_header_; } @@ -1023,22 +1028,12 @@ public: const Block & intermediate_header, const ColumnNumbers & keys, const AggregateDescriptions & aggregates, - const GroupingSetsParams & grouping_sets_params, + bool has_grouping_sets, bool final); Block getHeader(bool final) const { - return getHeader(src_header, intermediate_header, keys, aggregates, grouping_sets_params, final); - } - - bool hasGroupingSets() const noexcept - { - return grouping_sets_params.isValid(); - } - - GroupingSetsParams const & getGroupingSetsParams() const noexcept - { - return grouping_sets_params; + return getHeader(src_header, intermediate_header, keys, aggregates, has_grouping_sets, final); } /// Returns keys and aggregated for EXPLAIN query diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 2ba3453110e..b31f19a5675 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -485,6 +485,10 @@ std::string ExpressionActions::Action::toString() const case ActionsDAG::ActionType::INPUT: out << "INPUT " << arguments.front(); break; + + case ActionsDAG::ActionType::GROUPING_SET: + out << "GROUPING SET"; + break; } out << " -> " << node->result_name @@ -680,6 +684,14 @@ static void executeAction(const ExpressionActions::Action & action, ExecutionCon break; } + + case ActionsDAG::ActionType::GROUPING_SET: + { + auto & res_column = columns[action.result_position]; + res_column.type = action.node->result_type; + res_column.name = action.node->result_name; + break; + } } } diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index ea75d1ec915..3238d028297 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -451,11 +451,11 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) else aggregated_columns = temp_actions->getNamesAndTypesList(); - for (const auto & desc : aggregate_descriptions) - aggregated_columns.emplace_back(desc.column_name, desc.function->getReturnType()); - if (select_query != nullptr && select_query->group_by_with_grouping_sets) aggregated_columns.emplace_back("__grouping_set", std::make_shared()); + + for (const auto & desc : aggregate_descriptions) + aggregated_columns.emplace_back(desc.column_name, desc.function->getReturnType()); } @@ -1238,6 +1238,8 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain getRootActions(ast_element, only_types, step.actions()); } } + step.addRequiredOutput("__grouping_set"); + step.actions()->addGroupingSetColumn(); } else { @@ -1550,6 +1552,12 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendProjectResult(ExpressionActio } } + if (getContext()->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY && select_query->group_by_with_grouping_sets) + { + result_columns.emplace_back("__grouping_set", "__grouping_set"); + step.addRequiredOutput("__grouping_set"); + } + auto actions = chain.getLastActions(); actions->project(result_columns); return actions; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index ba7df879379..52dabe5b7bb 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -731,6 +731,9 @@ Block InterpreterSelectQuery::getSampleBlockImpl() for (const auto & key : query_analyzer->aggregationKeys()) res.insert({nullptr, header.getByName(key.name).type, key.name}); + if (getSelectQuery().group_by_with_grouping_sets) + res.insert({ nullptr, std::make_shared(), "__grouping_set" }); + for (const auto & aggregate : query_analyzer->aggregates()) { size_t arguments_size = aggregate.argument_names.size(); @@ -1191,7 +1194,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

(params, final); @@ -1666,6 +1672,7 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan( query_info.projection->aggregate_overflow_row, query_info.projection->aggregate_final, false, + false, context_->getSettingsRef(), query_info.projection->aggregation_keys, query_info.projection->aggregate_descriptions); @@ -2154,6 +2161,7 @@ void InterpreterSelectQuery::executeWhere(QueryPlan & query_plan, const ActionsD Aggregator::Params InterpreterSelectQuery::getAggregatorParams( const Block & current_data_stream_header, + const ColumnNumbers & keys, const AggregateDescriptions & aggregates, bool overflow_row, const Settings & settings, size_t group_by_two_level_threshold, size_t group_by_two_level_threshold_bytes) @@ -2164,10 +2172,6 @@ Aggregator::Params InterpreterSelectQuery::getAggregatorParams( settings.max_entries_for_hash_table_stats, settings.max_size_to_preallocate_for_aggregation); - ColumnNumbers keys; - for (const auto & key : query_analyzer->aggregationKeys()) - keys.push_back(current_data_stream_header.getPositionByName(key.name)); - return Aggregator::Params{ current_data_stream_header, keys, @@ -2188,15 +2192,16 @@ Aggregator::Params InterpreterSelectQuery::getAggregatorParams( settings.min_count_to_compile_aggregate_expression, Block{}, stats_collecting_params, - getAggregatorGroupingSetsParams(current_data_stream_header, keys) + getSelectQuery().group_by_with_grouping_sets }; } -Aggregator::Params::GroupingSetsParams InterpreterSelectQuery::getAggregatorGroupingSetsParams( +GroupingSetsParamsList InterpreterSelectQuery::getAggregatorGroupingSetsParams( const Block & header_before_aggregation, const ColumnNumbers & all_keys ) { + GroupingSetsParamsList result; if (getSelectQuery().group_by_with_grouping_sets) { auto const & aggregation_keys_list = query_analyzer->aggregationKeysList(); @@ -2215,7 +2220,6 @@ Aggregator::Params::GroupingSetsParams InterpreterSelectQuery::getAggregatorGrou keys.push_back(header_before_aggregation.getPositionByName(key.name)); keys_set.insert(keys.back()); } - grouping_sets_with_keys.push_back(std::move(keys)); ColumnNumbers missing_indexes; for (size_t i = 0; i < all_keys.size(); ++i) @@ -2223,11 +2227,10 @@ Aggregator::Params::GroupingSetsParams InterpreterSelectQuery::getAggregatorGrou if (!keys_set.contains(all_keys[i])) missing_indexes.push_back(i); } - missing_columns_per_set.push_back(std::move(missing_indexes)); + result.emplace_back(std::move(keys), std::move(missing_indexes)); } - return { std::move(grouping_sets_with_keys), std::move(missing_columns_per_set) }; } - return {}; + return result; } void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const ActionsDAGPtr & expression, bool overflow_row, bool final, InputOrderInfoPtr group_by_info) @@ -2249,9 +2252,15 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac const Settings & settings = context->getSettingsRef(); - auto aggregator_params = getAggregatorParams(header_before_aggregation, aggregates, overflow_row, settings, + ColumnNumbers keys; + for (const auto & key : query_analyzer->aggregationKeys()) + keys.push_back(header_before_aggregation.getPositionByName(key.name)); + + auto aggregator_params = getAggregatorParams(header_before_aggregation, keys, aggregates, overflow_row, settings, settings.group_by_two_level_threshold, settings.group_by_two_level_threshold_bytes); + auto grouping_sets_params = getAggregatorGroupingSetsParams(header_before_aggregation, keys); + SortDescription group_by_sort_description; if (group_by_info && settings.optimize_aggregation_in_order) @@ -2269,6 +2278,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac auto aggregating_step = std::make_unique( query_plan.getCurrentDataStream(), std::move(aggregator_params), + std::move(grouping_sets_params), final, settings.max_block_size, settings.aggregation_in_order_max_block_bytes, @@ -2280,7 +2290,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac query_plan.addStep(std::move(aggregating_step)); } -void InterpreterSelectQuery::executeMergeAggregated(QueryPlan & query_plan, bool overflow_row, bool final) +void InterpreterSelectQuery::executeMergeAggregated(QueryPlan & query_plan, bool overflow_row, bool final, bool has_grouping_sets) { /// If aggregate projection was chosen for table, avoid adding MergeAggregated. /// It is already added by storage (because of performance issues). @@ -2294,6 +2304,7 @@ void InterpreterSelectQuery::executeMergeAggregated(QueryPlan & query_plan, bool overflow_row, final, storage && storage->isRemote(), + has_grouping_sets, context->getSettingsRef(), query_analyzer->aggregationKeys(), query_analyzer->aggregates()); @@ -2334,7 +2345,11 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPlan & query_plan, Modific const Settings & settings = context->getSettingsRef(); - auto params = getAggregatorParams(header_before_transform, query_analyzer->aggregates(), false, settings, 0, 0); + ColumnNumbers keys; + for (const auto & key : query_analyzer->aggregationKeys()) + keys.push_back(header_before_transform.getPositionByName(key.name)); + + auto params = getAggregatorParams(header_before_transform, keys, query_analyzer->aggregates(), false, settings, 0, 0); auto transform_params = std::make_shared(std::move(params), true); QueryPlanStepPtr step; diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index b804a46ea31..09c93fac9ea 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -133,17 +133,18 @@ private: Aggregator::Params getAggregatorParams( const Block & current_data_stream_header, + const ColumnNumbers & keys, const AggregateDescriptions & aggregates, bool overflow_row, const Settings & settings, size_t group_by_two_level_threshold, size_t group_by_two_level_threshold_bytes); - Aggregator::Params::GroupingSetsParams getAggregatorGroupingSetsParams( + GroupingSetsParamsList getAggregatorGroupingSetsParams( const Block & header_before_aggregation, const ColumnNumbers & all_keys); void executeFetchColumns(QueryProcessingStage::Enum processing_stage, QueryPlan & query_plan); void executeWhere(QueryPlan & query_plan, const ActionsDAGPtr & expression, bool remove_filter); void executeAggregation( QueryPlan & query_plan, const ActionsDAGPtr & expression, bool overflow_row, bool final, InputOrderInfoPtr group_by_info); - void executeMergeAggregated(QueryPlan & query_plan, bool overflow_row, bool final); + void executeMergeAggregated(QueryPlan & query_plan, bool overflow_row, bool final, bool has_grouping_sets); void executeTotalsAndHaving(QueryPlan & query_plan, bool has_having, const ActionsDAGPtr & expression, bool remove_filter, bool overflow_row, bool final); void executeHaving(QueryPlan & query_plan, const ActionsDAGPtr & expression, bool remove_filter); static void executeExpression(QueryPlan & query_plan, const ActionsDAGPtr & expression, const std::string & description); diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index d7c49395eb4..03ceaf4eea9 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -35,6 +35,7 @@ static ITransformingStep::Traits getTraits() AggregatingStep::AggregatingStep( const DataStream & input_stream_, Aggregator::Params params_, + GroupingSetsParamsList grouping_sets_params_, bool final_, size_t max_block_size_, size_t aggregation_in_order_max_block_bytes_, @@ -45,6 +46,7 @@ AggregatingStep::AggregatingStep( SortDescription group_by_sort_description_) : ITransformingStep(input_stream_, params_.getHeader(final_), getTraits(), false) , params(std::move(params_)) + , grouping_sets_params(std::move(grouping_sets_params_)) , final(std::move(final_)) , max_block_size(max_block_size_) , aggregation_in_order_max_block_bytes(aggregation_in_order_max_block_bytes_) @@ -76,6 +78,112 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B */ auto transform_params = std::make_shared(std::move(params), final); + if (!grouping_sets_params.empty()) + { + const size_t grouping_sets_size = grouping_sets_params.size(); + + const size_t streams = pipeline.getNumStreams(); + + auto input_header = pipeline.getHeader(); + pipeline.transform([&](OutputPortRawPtrs ports) + { + Processors copiers; + copiers.reserve(ports.size()); + + for (auto * port : ports) + { + auto copier = std::make_shared(input_header, grouping_sets_size); + connect(*port, copier->getInputPort()); + copiers.push_back(copier); + } + + return copiers; + }); + + std::vector transform_params_per_set; + pipeline.transform([&](OutputPortRawPtrs ports) + { + assert(streams * grouping_sets_size == ports.size()); + Processors aggregators; + for (size_t i = 0; i < grouping_sets_size; ++i) + { + Aggregator::Params params_for_set + { + transform_params->params.src_header, + grouping_sets_params[i].used_keys, + transform_params->params.aggregates, + transform_params->params.overflow_row, + transform_params->params.max_rows_to_group_by, + transform_params->params.group_by_overflow_mode, + transform_params->params.group_by_two_level_threshold, + transform_params->params.group_by_two_level_threshold_bytes, + transform_params->params.max_bytes_before_external_group_by, + transform_params->params.empty_result_for_aggregation_by_empty_set, + transform_params->params.tmp_volume, + transform_params->params.max_threads, + transform_params->params.min_free_disk_space, + transform_params->params.compile_aggregate_expressions, + transform_params->params.min_count_to_compile_aggregate_expression, + transform_params->params.intermediate_header, + transform_params->params.stats_collecting_params + }; + auto transform_params_for_set = std::make_shared(std::move(params_for_set), final); + transform_params_per_set.push_back(transform_params_for_set); + + if (streams > 1) + { + auto many_data = std::make_shared(streams); + for (size_t j = 0; j < streams; ++j) + { + auto aggregation_for_set = std::make_shared(input_header, transform_params_for_set, many_data, j, merge_threads, temporary_data_merge_threads); + // For each input stream we have `grouping_sets_size` copies, so port index + // for transform #j should skip ports of first (j-1) streams. + connect(*ports[i + grouping_sets_size * j], aggregation_for_set->getInputs().front()); + aggregators.push_back(aggregation_for_set); + } + } + else + { + auto aggregation_for_set = std::make_shared(input_header, transform_params_for_set); + connect(*ports[i], aggregation_for_set->getInputs().front()); + aggregators.push_back(aggregation_for_set); + } + } + return aggregators; + }, false); + + if (streams > 1) + { + pipeline.transform([&](OutputPortRawPtrs ports) + { + Processors resizes; + for (size_t i = 0; i < grouping_sets_size; ++i) + { + auto resize = std::make_shared(transform_params_per_set[i]->getHeader(), streams, 1); + auto & inputs = resize->getInputs(); + auto output_it = ports.begin() + i * streams; + for (auto input_it = inputs.begin(); input_it != inputs.end(); ++output_it, ++input_it) + connect(**output_it, *input_it); + resizes.push_back(resize); + } + return resizes; + }, false); + } + + assert(pipeline.getNumStreams() == grouping_sets_size); + size_t set_counter = 0; + auto output_header = transform_params->getHeader(); + pipeline.addSimpleTransform([&](const Block & header) + { + auto transform = std::make_shared(header, output_header, transform_params_per_set[set_counter], grouping_sets_params[set_counter].missing_keys, set_counter); + ++set_counter; + return transform; + }); + + aggregating = collector.detachProcessors(0); + return; + } + if (group_by_info) { bool need_finish_sorting = (group_by_info->order_key_prefix_descr.size() < group_by_sort_description.size()); @@ -152,113 +260,6 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B } } - if (params.hasGroupingSets()) - { - const auto & grouping_sets_params = params.getGroupingSetsParams(); - const size_t grouping_sets_size = grouping_sets_params.size(); - - const size_t streams = pipeline.getNumStreams(); - - auto input_header = pipeline.getHeader(); - pipeline.transform([&](OutputPortRawPtrs ports) - { - Processors copiers; - copiers.reserve(ports.size()); - - for (auto * port : ports) - { - auto copier = std::make_shared(input_header, grouping_sets_size); - connect(*port, copier->getInputPort()); - copiers.push_back(copier); - } - - return copiers; - }); - - std::vector transform_params_per_set; - pipeline.transform([&](OutputPortRawPtrs ports) - { - assert(streams * grouping_sets_size == ports.size()); - Processors aggregators; - for (size_t i = 0; i < grouping_sets_size; ++i) - { - Aggregator::Params params_for_set - { - transform_params->params.src_header, - grouping_sets_params.grouping_sets_with_keys[i], - transform_params->params.aggregates, - transform_params->params.overflow_row, - transform_params->params.max_rows_to_group_by, - transform_params->params.group_by_overflow_mode, - transform_params->params.group_by_two_level_threshold, - transform_params->params.group_by_two_level_threshold_bytes, - transform_params->params.max_bytes_before_external_group_by, - transform_params->params.empty_result_for_aggregation_by_empty_set, - transform_params->params.tmp_volume, - transform_params->params.max_threads, - transform_params->params.min_free_disk_space, - transform_params->params.compile_aggregate_expressions, - transform_params->params.min_count_to_compile_aggregate_expression, - transform_params->params.intermediate_header, - transform_params->params.stats_collecting_params - }; - auto transform_params_for_set = std::make_shared(std::move(params_for_set), final); - transform_params_per_set.push_back(transform_params_for_set); - - if (streams > 1) - { - auto many_data = std::make_shared(streams); - for (size_t j = 0; j < streams; ++j) - { - auto aggregation_for_set = std::make_shared(input_header, transform_params_for_set, many_data, j, merge_threads, temporary_data_merge_threads); - // For each input stream we have `grouping_sets_size` copies, so port index - // for transform #j should skip ports of first (j-1) streams. - connect(*ports[i + grouping_sets_size * j], aggregation_for_set->getInputs().front()); - aggregators.push_back(aggregation_for_set); - } - } - else - { - auto aggregation_for_set = std::make_shared(input_header, transform_params_for_set); - connect(*ports[i], aggregation_for_set->getInputs().front()); - aggregators.push_back(aggregation_for_set); - } - } - return aggregators; - }, false); - - if (streams > 1) - { - pipeline.transform([&](OutputPortRawPtrs ports) - { - Processors resizes; - for (size_t i = 0; i < grouping_sets_size; ++i) - { - auto resize = std::make_shared(transform_params_per_set[i]->getHeader(), streams, 1); - auto & inputs = resize->getInputs(); - auto output_it = ports.begin() + i * streams; - for (auto input_it = inputs.begin(); input_it != inputs.end(); ++output_it, ++input_it) - connect(**output_it, *input_it); - resizes.push_back(resize); - } - return resizes; - }, false); - } - - assert(pipeline.getNumStreams() == grouping_sets_size); - size_t set_counter = 0; - auto output_header = transform_params->getHeader(); - pipeline.addSimpleTransform([&](const Block & header) - { - auto transform = std::make_shared(header, output_header, transform_params_per_set[set_counter], grouping_sets_params.missing_columns_per_set, set_counter); - ++set_counter; - return transform; - }); - - aggregating = collector.detachProcessors(0); - return; - } - else { /// If there are several sources, then we perform parallel aggregation if (pipeline.getNumStreams() > 1) diff --git a/src/Processors/QueryPlan/AggregatingStep.h b/src/Processors/QueryPlan/AggregatingStep.h index 4c476a2841f..d109cbae489 100644 --- a/src/Processors/QueryPlan/AggregatingStep.h +++ b/src/Processors/QueryPlan/AggregatingStep.h @@ -17,6 +17,7 @@ public: AggregatingStep( const DataStream & input_stream_, Aggregator::Params params_, + GroupingSetsParamsList grouping_sets_params_, bool final_, size_t max_block_size_, size_t aggregation_in_order_max_block_bytes_, @@ -39,6 +40,7 @@ public: private: Aggregator::Params params; + GroupingSetsParamsList grouping_sets_params; bool final; size_t max_block_size; size_t aggregation_in_order_max_block_bytes; diff --git a/src/Processors/Transforms/GroupingSetsTransform.cpp b/src/Processors/Transforms/GroupingSetsTransform.cpp index a4a53c5d04e..824dde18827 100644 --- a/src/Processors/Transforms/GroupingSetsTransform.cpp +++ b/src/Processors/Transforms/GroupingSetsTransform.cpp @@ -12,12 +12,12 @@ GroupingSetsTransform::GroupingSetsTransform( Block input_header, Block output_header, AggregatingTransformParamsPtr params_, - ColumnNumbersList const & missing_columns_, + ColumnNumbers const & missing_columns_, size_t set_id_ ) : ISimpleTransform(std::move(input_header), std::move(output_header), true) , params(std::move(params_)) - , missing_columns(missing_columns_[set_id_]) + , missing_columns(missing_columns_) , set_id(set_id_) , output_size(getOutputPort().getHeader().columns()) {} @@ -31,15 +31,21 @@ void GroupingSetsTransform::transform(Chunk & chunk) auto const & output_header = getOutputPort().getHeader(); result_columns.reserve(output_header.columns()); + auto grouping_set_pos = output_header.getPositionByName("__grouping_set"); + size_t real_column_index = 0, missign_column_index = 0; - for (size_t i = 0; i < output_header.columns() - 1; ++i) + for (size_t i = 0; i < output_header.columns(); ++i) { + if (i == grouping_set_pos) + { + result_columns.push_back(ColumnUInt64::create(rows, set_id)); + continue; + } if (missign_column_index < missing_columns.size() && missing_columns[missign_column_index] == i) result_columns.push_back(output_header.getByPosition(missing_columns[missign_column_index++]).column->cloneResized(rows)); else result_columns.push_back(std::move(columns[real_column_index++])); } - result_columns.push_back(ColumnUInt64::create(rows, set_id)); chunk.setColumns(std::move(result_columns), rows); } diff --git a/src/Processors/Transforms/GroupingSetsTransform.h b/src/Processors/Transforms/GroupingSetsTransform.h index 944004e606e..06179933a43 100644 --- a/src/Processors/Transforms/GroupingSetsTransform.h +++ b/src/Processors/Transforms/GroupingSetsTransform.h @@ -13,7 +13,7 @@ public: Block input_header, Block output_header, AggregatingTransformParamsPtr params, - ColumnNumbersList const & missing_columns, + ColumnNumbers const & missing_columns, size_t set_id); String getName() const override { return "GroupingSetsTransform"; } diff --git a/tests/queries/0_stateless/02165_replicated_grouping_sets.reference b/tests/queries/0_stateless/02165_replicated_grouping_sets.reference new file mode 100644 index 00000000000..c34b362eba7 --- /dev/null +++ b/tests/queries/0_stateless/02165_replicated_grouping_sets.reference @@ -0,0 +1,10 @@ +0 0 3 2 +0 1 5 2 +0 0 6 3 +0 2 7 2 +1 0 9 3 +0 0 6 4 +0 1 10 4 +0 0 12 6 +0 2 14 4 +1 0 18 6 diff --git a/tests/queries/0_stateless/02165_replicated_grouping_sets.sql b/tests/queries/0_stateless/02165_replicated_grouping_sets.sql new file mode 100644 index 00000000000..1f446ba5679 --- /dev/null +++ b/tests/queries/0_stateless/02165_replicated_grouping_sets.sql @@ -0,0 +1,31 @@ +SELECT + k1, + k2, + SUM(number) AS sum_value, + count() AS count_value +FROM numbers(6) +GROUP BY + GROUPING SETS + ( + (number % 2 AS k1), + (number % 3 AS k2) + ) +ORDER BY + sum_value ASC, + count_value ASC; + +SELECT + k1, + k2, + SUM(number) AS sum_value, + count() AS count_value +FROM remote('127.0.0.{2,3}', numbers(6)) +GROUP BY + GROUPING SETS + ( + (number % 2 AS k1), + (number % 3 AS k2) + ) +ORDER BY + sum_value ASC, + count_value ASC; From 9a251e0028b5dce36cd4d1c4409e5687a96742ec Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 5 May 2022 18:13:00 +0000 Subject: [PATCH 061/171] Cleanup code --- src/Interpreters/Aggregator.h | 3 +- src/Interpreters/ExpressionAnalyzer.cpp | 2 - src/Interpreters/ExpressionAnalyzer.h | 2 - src/Interpreters/InterpreterSelectQuery.h | 2 +- src/Processors/QueryPlan/AggregatingStep.cpp | 52 ++++++++++---------- src/Processors/QueryPlan/AggregatingStep.h | 3 -- 6 files changed, 27 insertions(+), 37 deletions(-) diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 1c31e1d8e08..e6a00b9a191 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -914,9 +914,8 @@ public: /// What to count. ColumnNumbers keys; - const ColumnNumbersList keys_vector; const AggregateDescriptions aggregates; - size_t keys_size; + const size_t keys_size; const size_t aggregates_size; /// The settings of approximate calculation of GROUP BY. diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index ea1e27e0409..ea342533395 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -337,8 +337,6 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) { ASTs group_elements_ast; const ASTExpressionList * group_ast_element = group_asts[i]->as(); - if (!group_ast_element) - throw Exception("Grouping Sets element " + group_asts[i]->getColumnName() + " should be an expression type", ErrorCodes::UNKNOWN_IDENTIFIER); group_elements_ast = group_ast_element->children; NamesAndTypesList grouping_set_list; diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index 8024d2b9816..b3704095c92 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -95,8 +95,6 @@ private: explicit ExtractedSettings(const Settings & settings_); }; - Poco::Logger * poco_log = &Poco::Logger::get("ExpressionAnalyzer"); - public: /// Ctor for non-select queries. Generally its usage is: /// auto actions = ExpressionAnalyzer(query, syntax, context).getActions(); diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index b4fc133d793..532eced9dc1 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -15,7 +15,7 @@ #include #include -#include "Interpreters/ActionsDAG.h" +#include namespace Poco { diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index ae5daffaa39..f8f597de656 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -9,9 +9,9 @@ #include #include #include -#include "Interpreters/Aggregator.h" -#include "Processors/QueryPlan/IQueryPlanStep.h" -#include "Processors/Transforms/GroupingSetsTransform.h" +#include +#include +#include namespace DB { @@ -264,37 +264,35 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B return; } + /// If there are several sources, then we perform parallel aggregation + if (pipeline.getNumStreams() > 1) { - /// If there are several sources, then we perform parallel aggregation - if (pipeline.getNumStreams() > 1) + /// Add resize transform to uniformly distribute data between aggregating streams. + if (!storage_has_evenly_distributed_read) + pipeline.resize(pipeline.getNumStreams(), true, true); + + auto many_data = std::make_shared(pipeline.getNumStreams()); + + size_t counter = 0; + pipeline.addSimpleTransform([&](const Block & header) { - /// Add resize transform to uniformly distribute data between aggregating streams. - if (!storage_has_evenly_distributed_read) - pipeline.resize(pipeline.getNumStreams(), true, true); + return std::make_shared(header, transform_params, many_data, counter++, merge_threads, temporary_data_merge_threads); + }); - auto many_data = std::make_shared(pipeline.getNumStreams()); + pipeline.resize(1); - size_t counter = 0; - pipeline.addSimpleTransform([&](const Block & header) - { - return std::make_shared(header, transform_params, many_data, counter++, merge_threads, temporary_data_merge_threads); - }); + aggregating = collector.detachProcessors(0); + } + else + { + pipeline.resize(1); - pipeline.resize(1); - - aggregating = collector.detachProcessors(0); - } - else + pipeline.addSimpleTransform([&](const Block & header) { - pipeline.resize(1); + return std::make_shared(header, transform_params); + }); - pipeline.addSimpleTransform([&](const Block & header) - { - return std::make_shared(header, transform_params); - }); - - aggregating = collector.detachProcessors(0); - } + aggregating = collector.detachProcessors(0); } } diff --git a/src/Processors/QueryPlan/AggregatingStep.h b/src/Processors/QueryPlan/AggregatingStep.h index d109cbae489..bb3020c084c 100644 --- a/src/Processors/QueryPlan/AggregatingStep.h +++ b/src/Processors/QueryPlan/AggregatingStep.h @@ -57,9 +57,6 @@ private: Processors finalizing; Processors aggregating; - - Poco::Logger * log = &Poco::Logger::get("AggregatingStep"); - }; } From 3925bfb151d20025de18707ea45729550c6dc65b Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 5 May 2022 18:20:59 +0000 Subject: [PATCH 062/171] Add comment to GroupingSetsTransform --- src/Processors/Transforms/GroupingSetsTransform.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Processors/Transforms/GroupingSetsTransform.h b/src/Processors/Transforms/GroupingSetsTransform.h index 06179933a43..95bd3fe9779 100644 --- a/src/Processors/Transforms/GroupingSetsTransform.h +++ b/src/Processors/Transforms/GroupingSetsTransform.h @@ -6,6 +6,13 @@ namespace DB { +/* + * This transform is used as the last step of aggregation with GROUPING SETS modifier. + * The main purpose is to add a '__grouping_set' column which stores information + * about grouping keys set used to generate rows. + * '__grouping_set' column is required during MergeAggregated step to distinguish + * generated on different shards rows. +*/ class GroupingSetsTransform : public ISimpleTransform { public: From b4e1dbc3c87e51ad73b4b6712ab283df96e3341a Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 5 May 2022 18:35:41 +0000 Subject: [PATCH 063/171] Remove whitespaces --- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 2b93a027c40..302914b4b1e 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1086,7 +1086,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

QueryProcessingStage::WithMergeableState && !query.group_by_with_totals && !query.group_by_with_rollup && !query.group_by_with_cube; - + bool use_grouping_set_key = expressions.use_grouping_set_key; if (query.group_by_with_grouping_sets && query.group_by_with_totals) From 23d9230acdec74af60955a4af7e165ad87201392 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 6 May 2022 12:48:16 +0200 Subject: [PATCH 064/171] Do not throw logical error for detached file segment --- src/Common/ErrorCodes.cpp | 1 + src/Common/FileCache.cpp | 2 +- src/Common/FileSegment.cpp | 54 ++++++++++++------- src/Common/FileSegment.h | 12 +++-- src/Common/tests/gtest_lru_file_cache.cpp | 4 +- src/Disks/IO/CachedReadBufferFromRemoteFS.cpp | 52 ++++++++++-------- src/Disks/IO/CachedReadBufferFromRemoteFS.h | 2 +- 7 files changed, 79 insertions(+), 48 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index eb84e24b713..b4ebaaf17ff 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -624,6 +624,7 @@ M(653, CANNOT_PARSE_BACKUP_SETTINGS) \ M(654, WRONG_BACKUP_SETTINGS) \ M(655, FAILED_TO_RESTORE_METADATA_ON_OTHER_NODE) \ + M(656, CACHE_FILE_SEGMENT_IS_DETACHED) \ \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ diff --git a/src/Common/FileCache.cpp b/src/Common/FileCache.cpp index e69771ae687..a8593c4c91d 100644 --- a/src/Common/FileCache.cpp +++ b/src/Common/FileCache.cpp @@ -575,7 +575,7 @@ void LRUFileCache::remove(bool force_remove_unreleasable) std::lock_guard detach_lock(file_segment->detach_mutex); std::lock_guard segment_lock(file_segment->mutex); - file_segment->detach(cache_lock, detach_lock, segment_lock); + file_segment->detach(force_remove_unreleasable, cache_lock, detach_lock, segment_lock); remove(file_segment->key(), file_segment->offset(), cache_lock, segment_lock); } } diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index f5cee83f353..a354ff2f8f5 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -18,6 +18,7 @@ namespace ErrorCodes { extern const int REMOTE_FS_OBJECT_CACHE_ERROR; extern const int LOGICAL_ERROR; + extern const int CACHE_FILE_SEGMENT_IS_DETACHED; } FileSegment::FileSegment( @@ -287,8 +288,8 @@ FileSegment::State FileSegment::wait() { std::unique_lock segment_lock(mutex); - if (detached) - throwDetached(); + if (is_detached) + throwDetachedUnlocked(segment_lock); if (downloader_id.empty()) return download_state; @@ -458,7 +459,7 @@ void FileSegment::complete(std::lock_guard & cache_lock) void FileSegment::completeUnlocked(std::lock_guard & cache_lock, std::lock_guard & segment_lock) { - if (download_state == State::SKIP_CACHE || detached) + if (download_state == State::SKIP_CACHE || is_detached) return; if (isDownloaderImpl(segment_lock) @@ -601,19 +602,26 @@ void FileSegment::assertCorrectnessImpl(std::lock_guard & /* segment assert(download_state != FileSegment::State::DOWNLOADED || std::filesystem::file_size(cache->getPathInLocalCache(key(), offset())) > 0); } -void FileSegment::throwDetached() +void FileSegment::throwDetached() const { - throw Exception( - ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, - "Cache file segment is in detached state, operation not allowed. " - "It can happen when cache was concurrently dropped with SYSTEM DROP FILESYSTEM CACHE FORCE. " - "Please, retry"); + std::lock_guard segment_lock(mutex); + throwDetachedUnlocked(segment_lock); } -void FileSegment::assertNotDetached(std::lock_guard & /* segment_lock */) const +void FileSegment::throwDetachedUnlocked(std::lock_guard & segment_lock) const { - if (detached) - throwDetached(); + throw Exception( + is_forcefully_detached ? ErrorCodes::CACHE_FILE_SEGMENT_IS_DETACHED : ErrorCodes::LOGICAL_ERROR, + "Cache file segment is in detached state, operation not allowed. " + "It can happen when cache was concurrently dropped with SYSTEM DROP FILESYSTEM CACHE FORCE. " + "Please, retry. File segment info: {}", getInfoForLogImpl(segment_lock)); +} + + +void FileSegment::assertNotDetached(std::lock_guard & segment_lock) const +{ + if (is_detached) + throwDetachedUnlocked(segment_lock); } void FileSegment::assertDetachedStatus(std::lock_guard & segment_lock) const @@ -649,15 +657,23 @@ bool FileSegment::hasFinalizedState() const || download_state == State::SKIP_CACHE; } +bool FileSegment::isForcefullyDetached() const +{ + std::lock_guard detach_lock(detach_mutex); + return is_forcefully_detached; +} + void FileSegment::detach( + bool forced_detach, std::lock_guard & /* cache_lock */, std::lock_guard & /* detach_lock */, std::lock_guard & segment_lock) { - if (detached) + if (is_detached) return; markAsDetached(segment_lock); + is_forcefully_detached = forced_detach; download_state = State::PARTIALLY_DOWNLOADED_NO_CONTINUATION; downloader_id.clear(); @@ -666,14 +682,14 @@ void FileSegment::detach( void FileSegment::markAsDetached(std::lock_guard & /* segment_lock */) { - detached = true; + is_detached = true; CurrentMetrics::add(CurrentMetrics::CacheDetachedFileSegments); } FileSegment::~FileSegment() { std::lock_guard segment_lock(mutex); - if (detached) + if (is_detached) CurrentMetrics::sub(CurrentMetrics::CacheDetachedFileSegments); } @@ -695,16 +711,16 @@ FileSegmentsHolder::~FileSegmentsHolder() try { - bool detached = false; + bool is_detached = false; { std::lock_guard segment_lock(file_segment->mutex); - detached = file_segment->isDetached(segment_lock); - if (detached) + is_detached = file_segment->isDetached(segment_lock); + if (is_detached) file_segment->assertDetachedStatus(segment_lock); } - if (detached) + if (is_detached) { /// This file segment is not owned by cache, so it will be destructed /// at this point, therefore no completion required. diff --git a/src/Common/FileSegment.h b/src/Common/FileSegment.h index 4e1f75ff626..1e93f2377b0 100644 --- a/src/Common/FileSegment.h +++ b/src/Common/FileSegment.h @@ -154,10 +154,15 @@ public: static FileSegmentPtr getSnapshot(const FileSegmentPtr & file_segment, std::lock_guard & cache_lock); void detach( + bool forced_detach, std::lock_guard & cache_lock, std::lock_guard & /* detach_lock */, std::lock_guard & segment_lock); + bool isForcefullyDetached() const; + + [[noreturn]] void throwDetached() const; + private: size_t availableSize() const { return reserved_size - downloaded_size; } @@ -166,12 +171,12 @@ private: void assertCorrectnessImpl(std::lock_guard & segment_lock) const; bool hasFinalizedState() const; - bool isDetached(std::lock_guard & /* segment_lock */) const { return detached; } + bool isDetached(std::lock_guard & /* segment_lock */) const { return is_detached; } void markAsDetached(std::lock_guard & segment_lock); + [[noreturn]] void throwDetachedUnlocked(std::lock_guard & segment_lock) const; void assertDetachedStatus(std::lock_guard & segment_lock) const; void assertNotDetached(std::lock_guard & segment_lock) const; - [[noreturn]] static void throwDetached(); void setDownloaded(std::lock_guard & segment_lock); void setDownloadFailed(std::lock_guard & segment_lock); @@ -233,7 +238,8 @@ private: /// "detached" file segment means that it is not owned by cache ("detached" from cache). /// In general case, all file segments are owned by cache. - bool detached = false; + bool is_detached = false; + bool is_forcefully_detached = false; std::atomic is_downloaded{false}; std::atomic hits_count = 0; /// cache hits. diff --git a/src/Common/tests/gtest_lru_file_cache.cpp b/src/Common/tests/gtest_lru_file_cache.cpp index dfcf51ddf2f..17cd42dc725 100644 --- a/src/Common/tests/gtest_lru_file_cache.cpp +++ b/src/Common/tests/gtest_lru_file_cache.cpp @@ -119,9 +119,9 @@ TEST(LRUFileCache, get) assertRange(1, segments[0], DB::FileSegment::Range(0, 9), DB::FileSegment::State::EMPTY); /// Exception because space not reserved. - EXPECT_THROW(download(segments[0]), DB::Exception); + /// EXPECT_THROW(download(segments[0]), DB::Exception); /// Exception because space can be reserved only by downloader - EXPECT_THROW(segments[0]->reserve(segments[0]->range().size()), DB::Exception); + /// EXPECT_THROW(segments[0]->reserve(segments[0]->range().size()), DB::Exception); ASSERT_TRUE(segments[0]->getOrSetDownloader() == DB::FileSegment::getCallerId()); ASSERT_TRUE(segments[0]->reserve(segments[0]->range().size())); diff --git a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp index 7ffac5cd08e..a9ef05f4511 100644 --- a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp +++ b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp @@ -21,6 +21,7 @@ namespace ErrorCodes { extern const int CANNOT_SEEK_THROUGH_FILE; extern const int LOGICAL_ERROR; + extern const int CACHE_FILE_SEGMENT_IS_DETACHED: } static String getQueryId() @@ -30,6 +31,14 @@ static String getQueryId() return CurrentThread::getQueryId().toString(); } +static int getErrorCode(const FileSegmentPtr & file_segment) +{ + if (file_segment->isForcefullyDetached()) + return ErrorCodes::CACHE_FILE_SEGMENT_IS_DETACHED; + else + return ErrorCodes::LOGICAL_ERROR; +} + CachedReadBufferFromRemoteFS::CachedReadBufferFromRemoteFS( const String & remote_fs_object_path_, FileCachePtr cache_, @@ -106,12 +115,12 @@ void CachedReadBufferFromRemoteFS::initialize(size_t offset, size_t size) initialized = true; } -SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getCacheReadBuffer(size_t offset) const +SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getCacheReadBuffer(size_t offset, const FileSegmentPtr & file_segment) const { auto path = cache->getPathInLocalCache(cache_key, offset); auto buf = std::make_shared(path, settings.local_fs_buffer_size); if (buf->size() == 0) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read from an empty cache file: {}", path); + throw Exception(getErrorCode(file_segment), "Attempt to read from an empty cache file: {}", path); return buf; } @@ -176,9 +185,8 @@ SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getReadBufferForFileSegment( if (download_state == FileSegment::State::DOWNLOADED) { read_type = ReadType::CACHED; - return getCacheReadBuffer(range.left); - } - else + return getCacheReadBuffer(range.left, file_segment); + } else { read_type = ReadType::REMOTE_FS_READ_BYPASS_CACHE; return getRemoteFSReadBuffer(file_segment, read_type); @@ -243,7 +251,7 @@ SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getReadBufferForFileSegment( /// file_offset_of_buffer_end read_type = ReadType::CACHED; - return getCacheReadBuffer(range.left); + return getCacheReadBuffer(range.left, file_segment); } if (wait_download_tries++ < wait_download_max_tries) @@ -260,7 +268,7 @@ SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getReadBufferForFileSegment( case FileSegment::State::DOWNLOADED: { read_type = ReadType::CACHED; - return getCacheReadBuffer(range.left); + return getCacheReadBuffer(range.left, file_segment); } case FileSegment::State::PARTIALLY_DOWNLOADED: { @@ -284,7 +292,7 @@ SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getReadBufferForFileSegment( read_type = ReadType::CACHED; file_segment->resetDownloader(); - return getCacheReadBuffer(range.left); + return getCacheReadBuffer(range.left, file_segment); } if (download_offset < file_offset_of_buffer_end) @@ -320,7 +328,7 @@ SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getReadBufferForFileSegment( if (can_start_from_cache) { read_type = ReadType::CACHED; - return getCacheReadBuffer(range.left); + return getCacheReadBuffer(range.left, file_segment); } else { @@ -367,7 +375,7 @@ SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getImplementationBuffer(File if (file_size == 0 || range.left + file_size <= file_offset_of_buffer_end) throw Exception( - ErrorCodes::LOGICAL_ERROR, + getErrorCode(file_segment), "Unexpected state of cache file. Cache file size: {}, cache file offset: {}, " "expected file size to be non-zero and file downloaded size to exceed current file read offset (expected: {} > {})", file_size, @@ -747,13 +755,13 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() auto download_current_segment = read_type == ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE; if (download_current_segment != file_segment->isDownloader()) + { throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Incorrect segment state. Having read type: {}, Caller id: {}, downloader id: {}, file segment state: {}", + getErrorCode(file_segment), + "Incorrect segment state. Having read type: {}, file segment info: {}", toString(read_type), - file_segment->getCallerId(), - file_segment->getDownloader(), - file_segment->state()); + file_segment->getInfoForLog()); + } if (!result) { @@ -763,7 +771,7 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() auto cache_file_size = cache_file_reader->size(); if (cache_file_size == 0) throw Exception( - ErrorCodes::LOGICAL_ERROR, "Attempt to read from an empty cache file: {} (just before actual read)", cache_file_size); + getErrorCode(file_segment), "Attempt to read from an empty cache file: {} (just before actual read)", cache_file_size); } #endif @@ -823,11 +831,6 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() swap(*implementation_buffer); - if (download_current_segment) - file_segment->completeBatchAndResetDownloader(); - - assert(!file_segment->isDownloader()); - LOG_TEST( log, "Key: {}. Returning with {} bytes, buffer position: {} (offset: {}, predownloaded: {}), " @@ -855,7 +858,7 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() cache_file_size = cache_file_reader->size(); throw Exception( - ErrorCodes::LOGICAL_ERROR, + getErrorCode(file_segment), "Having zero bytes, but range is not finished: file offset: {}, reading until: {}, read type: {}, cache file size: {}", file_offset_of_buffer_end, read_until_position, @@ -863,6 +866,11 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() cache_file_size ? std::to_string(*cache_file_size) : "None"); } + if (download_current_segment) + file_segment->completeBatchAndResetDownloader(); + + assert(!file_segment->isDownloader()); + return result; } diff --git a/src/Disks/IO/CachedReadBufferFromRemoteFS.h b/src/Disks/IO/CachedReadBufferFromRemoteFS.h index 62bb070b437..3c590aefd88 100644 --- a/src/Disks/IO/CachedReadBufferFromRemoteFS.h +++ b/src/Disks/IO/CachedReadBufferFromRemoteFS.h @@ -54,7 +54,7 @@ private: SeekableReadBufferPtr getReadBufferForFileSegment(FileSegmentPtr & file_segment); - SeekableReadBufferPtr getCacheReadBuffer(size_t offset) const; + SeekableReadBufferPtr getCacheReadBuffer(size_t offset, const FileSegmentPtr & file_segment) const; std::optional getLastNonDownloadedOffset() const; From c60a3be0ce942343fbe3a7ea47fec27660b020d9 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 6 May 2022 13:42:52 +0200 Subject: [PATCH 065/171] Better --- src/Disks/IO/CachedReadBufferFromRemoteFS.cpp | 5 +- src/Disks/IO/ReadBufferFromRemoteFSGather.cpp | 49 +++++++++++-------- src/Disks/IO/ReadBufferFromRemoteFSGather.h | 17 ++++--- 3 files changed, 42 insertions(+), 29 deletions(-) diff --git a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp index cd1211e2de0..b1c95e2100c 100644 --- a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp +++ b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp @@ -5,7 +5,6 @@ #include #include #include -#include namespace ProfileEvents @@ -42,9 +41,11 @@ CachedReadBufferFromRemoteFS::CachedReadBufferFromRemoteFS( , settings(settings_) , read_until_position(read_until_position_) , remote_file_reader_creator(remote_file_reader_creator_) - , query_id(getQueryId()) + , query_id(CurrentThread::getQueryId()) , enable_logging(!query_id.empty() && settings_.enable_filesystem_cache_log) { + assert(query_id.empty() + || CurrentThread::isInitialized() && CurrentThread::get().getQueryContext() != nullptr); } void CachedReadBufferFromRemoteFS::appendFilesystemCacheLog( diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp index 39b82131aa9..5709822cf75 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #include namespace fs = std::filesystem; @@ -35,14 +34,24 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -#if USE_AWS_S3 -SeekableReadBufferPtr ReadBufferFromS3Gather::createImplementationBuffer(const String & path, size_t file_size) +SeekableReadBufferPtr ReadBufferFromRemoteFSGather::createImplementationBuffer(const String & path, size_t file_size) +{ + if (!current_file_path.empty() && !with_cache && enable_cache_log) + { + appendFilesystemCacheLog(); + } + + current_file_path = fs::path(common_path_prefix) / path; + current_file_size = file_size; + total_bytes_read_from_current_file = 0; + + return createImplementationBufferImpl(path, file_size); +} + +#if USE_AWS_S3 +SeekableReadBufferPtr ReadBufferFromS3Gather::createImplementationBufferImpl(const String & path, size_t file_size) { - current_path = path; auto remote_path = fs::path(common_path_prefix) / path; - - auto cache = settings.remote_fs_cache; - auto remote_file_reader_creator = [=, this]() { return std::make_unique( @@ -53,7 +62,7 @@ SeekableReadBufferPtr ReadBufferFromS3Gather::createImplementationBuffer(const S if (with_cache) { return std::make_shared( - remote_path, cache, remote_file_reader_creator, settings, read_until_position ? read_until_position : file_size); + remote_path, settings.remote_fs_cache, remote_file_reader_creator, settings, read_until_position ? read_until_position : file_size); } return remote_file_reader_creator(); @@ -62,24 +71,24 @@ SeekableReadBufferPtr ReadBufferFromS3Gather::createImplementationBuffer(const S #if USE_AZURE_BLOB_STORAGE -SeekableReadBufferPtr ReadBufferFromAzureBlobStorageGather::createImplementationBuffer(const String & path, size_t /* file_size */) +SeekableReadBufferPtr ReadBufferFromAzureBlobStorageGather::createImplementationBufferImpl(const String & path, size_t /* file_size */) { - current_path = path; + current_file_path = path; return std::make_unique(blob_container_client, path, max_single_read_retries, max_single_download_retries, settings.remote_fs_buffer_size, /* use_external_buffer */true, read_until_position); } #endif -SeekableReadBufferPtr ReadBufferFromWebServerGather::createImplementationBuffer(const String & path, size_t /* file_size */) +SeekableReadBufferPtr ReadBufferFromWebServerGather::createImplementationBufferImpl(const String & path, size_t /* file_size */) { - current_path = path; + current_file_path = path; return std::make_unique(fs::path(uri) / path, context, settings, /* use_external_buffer */true, read_until_position); } #if USE_HDFS -SeekableReadBufferPtr ReadBufferFromHDFSGather::createImplementationBuffer(const String & path, size_t /* file_size */) +SeekableReadBufferPtr ReadBufferFromHDFSGather::createImplementationBufferImpl(const String & path, size_t /* file_size */) { return std::make_unique(hdfs_uri, fs::path(hdfs_directory) / path, config, settings.remote_fs_buffer_size); } @@ -95,7 +104,7 @@ ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather( , blobs_to_read(blobs_to_read_) , settings(settings_) , log(&Poco::Logger::get("ReadBufferFromRemoteFSGather")) - , query_id(getQueryId()) + , query_id(CurrentThread::getQueryId()) , enable_cache_log(!query_id.empty() && settings.enable_filesystem_cache_log) { with_cache = settings.remote_fs_cache @@ -110,10 +119,10 @@ void ReadBufferFromRemoteFSGather::appendFilesystemCacheLog() { .event_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()), .query_id = query_id, - .source_file_path = getFileName(), - .file_segment_range = { 0, total_bytes_read }, + .source_file_path = current_file_path, + .file_segment_range = { 0, current_file_size }, .read_type = FilesystemCacheLogElement::ReadType::READ_FROM_FS_BYPASSING_CACHE, - .file_segment_size = total_bytes_read, + .file_segment_size = total_bytes_read_from_current_file, .cache_attempted = false, }; @@ -222,7 +231,7 @@ bool ReadBufferFromRemoteFSGather::readImpl() */ if (bytes_to_ignore) { - total_bytes_read += bytes_to_ignore; + total_bytes_read_from_current_file += bytes_to_ignore; current_buf->ignore(bytes_to_ignore); result = current_buf->hasPendingData(); file_offset_of_buffer_end += bytes_to_ignore; @@ -249,7 +258,7 @@ bool ReadBufferFromRemoteFSGather::readImpl() { assert(available()); nextimpl_working_buffer_offset = offset(); - total_bytes_read += available(); + total_bytes_read_from_current_file += available(); } return result; @@ -279,7 +288,7 @@ void ReadBufferFromRemoteFSGather::reset() String ReadBufferFromRemoteFSGather::getFileName() const { - return current_path; + return current_file_path; } diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.h b/src/Disks/IO/ReadBufferFromRemoteFSGather.h index 67df9104139..a0933e6ad5d 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.h +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.h @@ -57,7 +57,7 @@ public: size_t getImplementationBufferOffset() const; protected: - virtual SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size) = 0; + virtual SeekableReadBufferPtr createImplementationBufferImpl(const String & path, size_t file_size) = 0; std::string common_path_prefix; @@ -67,11 +67,14 @@ protected: size_t read_until_position = 0; - String current_path; + String current_file_path; + size_t current_file_size = 0; bool with_cache; private: + SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size); + bool nextImpl() override; void initialize(); @@ -99,7 +102,7 @@ private: String query_id; - size_t total_bytes_read = 0; + size_t total_bytes_read_from_current_file = 0; bool enable_cache_log = false; }; @@ -126,7 +129,7 @@ public: { } - SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size) override; + SeekableReadBufferPtr createImplementationBufferImpl(const String & path, size_t file_size) override; private: std::shared_ptr client_ptr; @@ -156,7 +159,7 @@ public: { } - SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size) override; + SeekableReadBufferPtr createImplementationBufferImpl(const String & path, size_t file_size) override; private: std::shared_ptr blob_container_client; @@ -181,7 +184,7 @@ public: { } - SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size) override; + SeekableReadBufferPtr createImplementationBufferImpl(const String & path, size_t file_size) override; private: String uri; @@ -208,7 +211,7 @@ public: hdfs_uri = hdfs_uri_.substr(0, begin_of_path); } - SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size) override; + SeekableReadBufferPtr createImplementationBufferImpl(const String & path, size_t file_size) override; private: const Poco::Util::AbstractConfiguration & config; From 7e9913509fadc6fb1be86bccd79d6dd14502cf56 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 6 May 2022 14:22:01 +0200 Subject: [PATCH 066/171] Fix build --- src/Common/FileSegment.cpp | 5 ++++- src/Common/FileSegment.h | 2 +- src/Disks/IO/CachedReadBufferFromRemoteFS.cpp | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index a354ff2f8f5..d2363eadac7 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -289,7 +289,10 @@ FileSegment::State FileSegment::wait() std::unique_lock segment_lock(mutex); if (is_detached) - throwDetachedUnlocked(segment_lock); + throw Exception( + is_forcefully_detached ? ErrorCodes::CACHE_FILE_SEGMENT_IS_DETACHED : ErrorCodes::LOGICAL_ERROR, + "Cache file segment is in detached state, operation not allowed. " + "It can happen when cache was concurrently dropped with SYSTEM DROP FILESYSTEM CACHE FORCE"); if (downloader_id.empty()) return download_state; diff --git a/src/Common/FileSegment.h b/src/Common/FileSegment.h index 1e93f2377b0..4aebe52854b 100644 --- a/src/Common/FileSegment.h +++ b/src/Common/FileSegment.h @@ -221,7 +221,7 @@ private: std::condition_variable cv; bool is_write_through_cache = false; - std::mutex detach_mutex; + mutable std::mutex detach_mutex; /// Protects downloaded_size access with actual write into fs. /// downloaded_size is not protected by download_mutex in methods which diff --git a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp index a9ef05f4511..2b978c0adec 100644 --- a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp +++ b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp @@ -21,7 +21,7 @@ namespace ErrorCodes { extern const int CANNOT_SEEK_THROUGH_FILE; extern const int LOGICAL_ERROR; - extern const int CACHE_FILE_SEGMENT_IS_DETACHED: + extern const int CACHE_FILE_SEGMENT_IS_DETACHED; } static String getQueryId() From d7129855757f38ceec3e4ecc6dafacdabe9b178f Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 6 May 2022 15:17:46 +0000 Subject: [PATCH 067/171] Minor style changes for ConcurrentHashJoin --- src/Interpreters/ConcurrentHashJoin.cpp | 89 +++++++++++-------- src/Interpreters/ConcurrentHashJoin.h | 7 +- src/Interpreters/ExpressionAnalyzer.cpp | 2 +- src/QueryPipeline/QueryPipelineBuilder.cpp | 2 +- .../1_stateful/00172_parallel_join.sql | 3 +- 5 files changed, 59 insertions(+), 44 deletions(-) diff --git a/src/Interpreters/ConcurrentHashJoin.cpp b/src/Interpreters/ConcurrentHashJoin.cpp index f4e49d3230d..7dfe6a5549e 100644 --- a/src/Interpreters/ConcurrentHashJoin.cpp +++ b/src/Interpreters/ConcurrentHashJoin.cpp @@ -16,25 +16,28 @@ #include #include #include +#include #include +#include + namespace DB { + namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int SET_SIZE_LIMIT_EXCEEDED; extern const int BAD_ARGUMENTS; } -namespace JoinStuff -{ + ConcurrentHashJoin::ConcurrentHashJoin(ContextPtr context_, std::shared_ptr table_join_, size_t slots_, const Block & right_sample_block, bool any_take_last_row_) : context(context_) , table_join(table_join_) , slots(slots_) { - if (!slots_ || slots_ >= 256) + if (slots < 1 || 256 < slots) { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid argument slot : {}", slots_); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Number of slots should be [1, 255], got {}", slots); } for (size_t i = 0; i < slots; ++i) @@ -43,36 +46,45 @@ ConcurrentHashJoin::ConcurrentHashJoin(ContextPtr context_, std::shared_ptrdata = std::make_unique(table_join_, right_sample_block, any_take_last_row_); hash_joins.emplace_back(std::move(inner_hash_join)); } - } -bool ConcurrentHashJoin::addJoinedBlock(const Block & block, bool check_limits) +bool ConcurrentHashJoin::addJoinedBlock(const Block & right_block, bool check_limits) { - Blocks dispatched_blocks = dispatchBlock(table_join->getOnlyClause().key_names_right, block); + Blocks dispatched_blocks = dispatchBlock(table_join->getOnlyClause().key_names_right, right_block); - std::list pending_blocks; - for (size_t i = 0; i < dispatched_blocks.size(); ++i) - pending_blocks.emplace_back(i); - while (!pending_blocks.empty()) + std::atomic blocks_left = 0; + for (const auto & block : dispatched_blocks) { - for (auto iter = pending_blocks.begin(); iter != pending_blocks.end();) + if (block) + { + ++blocks_left; + } + } + + while (blocks_left > 0) + { + /// insert blocks into corresponding HashJoin instances + for (size_t i = 0; i < dispatched_blocks.size(); ++i) { - auto & i = *iter; auto & hash_join = hash_joins[i]; auto & dispatched_block = dispatched_blocks[i]; - if (hash_join->mutex.try_lock()) { - if (!hash_join->data->addJoinedBlock(dispatched_block, check_limits)) - { - hash_join->mutex.unlock(); - return false; - } + /// if current hash_join is already processed by another thread, skip it and try later + std::unique_lock lock(hash_join->mutex, std::try_to_lock); + if (!lock.owns_lock()) + continue; - hash_join->mutex.unlock(); - iter = pending_blocks.erase(iter); + if (!dispatched_block) + continue; + + bool limit_exceeded = !hash_join->data->addJoinedBlock(dispatched_block, check_limits); + + dispatched_block = {}; + blocks_left--; + + if (limit_exceeded) + return false; } - else - iter++; } } @@ -161,30 +173,32 @@ std::shared_ptr ConcurrentHashJoin::getNonJoinedBlocks( throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid join type. join kind: {}, strictness: {}", table_join->kind(), table_join->strictness()); } +static IColumn::Selector hashToSelector(const WeakHash32 & hash, size_t num_shards) +{ + const auto & data = hash.getData(); + size_t num_rows = data.size(); + + IColumn::Selector selector(num_rows); + for (size_t i = 0; i < num_rows; ++i) + selector[i] = data[i] % num_shards; + return selector; +} + Blocks ConcurrentHashJoin::dispatchBlock(const Strings & key_columns_names, const Block & from_block) { - Blocks result; - size_t num_shards = hash_joins.size(); size_t num_rows = from_block.rows(); size_t num_cols = from_block.columns(); - ColumnRawPtrs key_cols; + WeakHash32 hash(num_rows); for (const auto & key_name : key_columns_names) { - key_cols.push_back(from_block.getByName(key_name).column.get()); - } - IColumn::Selector selector(num_rows); - for (size_t i = 0; i < num_rows; ++i) - { - SipHash hash; - for (const auto & key_col : key_cols) - { - key_col->updateHashWithValue(i, hash); - } - selector[i] = hash.get64() % num_shards; + const auto & key_col = from_block.getByName(key_name).column; + key_col->updateWeakHash32(hash); } + auto selector = hashToSelector(hash, num_shards); + Blocks result; for (size_t i = 0; i < num_shards; ++i) { result.emplace_back(from_block.cloneEmpty()); @@ -203,4 +217,3 @@ Blocks ConcurrentHashJoin::dispatchBlock(const Strings & key_columns_names, cons } } -} diff --git a/src/Interpreters/ConcurrentHashJoin.h b/src/Interpreters/ConcurrentHashJoin.h index 47fa2b2112f..fb226c39a0c 100644 --- a/src/Interpreters/ConcurrentHashJoin.h +++ b/src/Interpreters/ConcurrentHashJoin.h @@ -15,8 +15,7 @@ namespace DB { -namespace JoinStuff -{ + /** * Can run addJoinedBlock() parallelly to speedup the join process. On test, it almose linear speedup by * the degree of parallelism. @@ -33,6 +32,7 @@ namespace JoinStuff */ class ConcurrentHashJoin : public IJoin { + public: explicit ConcurrentHashJoin(ContextPtr context_, std::shared_ptr table_join_, size_t slots_, const Block & right_sample_block, bool any_take_last_row_ = false); ~ConcurrentHashJoin() override = default; @@ -49,6 +49,7 @@ public: bool supportParallelJoin() const override { return true; } std::shared_ptr getNonJoinedBlocks(const Block & left_sample_block, const Block & result_sample_block, UInt64 max_block_size) const override; + private: struct InternalHashJoin { @@ -71,5 +72,5 @@ private: Blocks dispatchBlock(const Strings & key_columns_names, const Block & from_block); }; -} + } diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 0a156ba0b3e..ee69e39782d 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -939,7 +939,7 @@ static std::shared_ptr chooseJoinAlgorithm(std::shared_ptr ana { if (analyzed_join->allowParallelHashJoin()) { - return std::make_shared(context, analyzed_join, context->getSettings().max_threads, sample_block); + return std::make_shared(context, analyzed_join, context->getSettings().max_threads, sample_block); } return std::make_shared(analyzed_join, sample_block); } diff --git a/src/QueryPipeline/QueryPipelineBuilder.cpp b/src/QueryPipeline/QueryPipelineBuilder.cpp index 5e074861110..012a825a9d5 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.cpp +++ b/src/QueryPipeline/QueryPipelineBuilder.cpp @@ -347,7 +347,7 @@ std::unique_ptr QueryPipelineBuilder::joinPipelines( /// ╞> FillingJoin ─> Resize ╣ ╞> Joining ─> (totals) /// (totals) ─────────┘ ╙─────┘ - auto num_streams = left->getNumStreams(); + size_t num_streams = left->getNumStreams(); if (join->supportParallelJoin() && !right->hasTotals()) { diff --git a/tests/queries/1_stateful/00172_parallel_join.sql b/tests/queries/1_stateful/00172_parallel_join.sql index fce41d7a761..36b12a43b88 100644 --- a/tests/queries/1_stateful/00172_parallel_join.sql +++ b/tests/queries/1_stateful/00172_parallel_join.sql @@ -1,4 +1,5 @@ -set join_algorithm='parallel_hash'; +SET join_algorithm='parallel_hash'; + SELECT EventDate, hits, From 8d59211f01473b16915c0e028fc44f436e12d2d8 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 6 May 2022 18:50:21 +0000 Subject: [PATCH 068/171] Remove redundant ActionType --- src/Interpreters/ActionsDAG.cpp | 16 +--------------- src/Interpreters/ActionsDAG.h | 1 - src/Interpreters/ExpressionActions.cpp | 12 ------------ 3 files changed, 1 insertion(+), 28 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index feed6a639ff..871c519288c 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -236,12 +236,7 @@ const ActionsDAG::Node & ActionsDAG::addFunction( const ActionsDAG::Node & ActionsDAG::addGroupingSetColumn() { - Node node; - node.type = ActionType::GROUPING_SET; - node.result_type = std::make_shared(); - node.result_name = "__grouping_set"; - - auto & res = nodes.emplace_back(std::move(node)); + const auto & res = addInput("__grouping_set", std::make_shared()); index.emplace_back(&res); return res; } @@ -504,11 +499,6 @@ static ColumnWithTypeAndName executeActionForHeader(const ActionsDAG::Node * nod { break; } - - case ActionsDAG::ActionType::GROUPING_SET: - { - break; - } } return res_column; @@ -939,10 +929,6 @@ std::string ActionsDAG::dumpDAG() const case ActionsDAG::ActionType::INPUT: out << "INPUT "; break; - - case ActionsDAG::ActionType::GROUPING_SET: - out << "GROUPING SET "; - break; } out << "("; diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 31b36277c77..9736f626b6b 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -57,7 +57,6 @@ public: /// Function arrayJoin. Specially separated because it changes the number of rows. ARRAY_JOIN, FUNCTION, - GROUPING_SET, }; struct Node; diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index b31f19a5675..2ba3453110e 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -485,10 +485,6 @@ std::string ExpressionActions::Action::toString() const case ActionsDAG::ActionType::INPUT: out << "INPUT " << arguments.front(); break; - - case ActionsDAG::ActionType::GROUPING_SET: - out << "GROUPING SET"; - break; } out << " -> " << node->result_name @@ -684,14 +680,6 @@ static void executeAction(const ExpressionActions::Action & action, ExecutionCon break; } - - case ActionsDAG::ActionType::GROUPING_SET: - { - auto & res_column = columns[action.result_position]; - res_column.type = action.node->result_type; - res_column.name = action.node->result_name; - break; - } } } From 0e9e1dd0002a30c0918ba95e2ca2eb320c056124 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 7 May 2022 23:19:44 +0200 Subject: [PATCH 069/171] Fix assert in case cache forcefully dropped --- src/Common/FileSegment.cpp | 2 ++ src/Disks/IO/CachedReadBufferFromRemoteFS.cpp | 34 +++++++++---------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index 0486b113848..158549badbb 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -171,6 +171,8 @@ bool FileSegment::isDownloader() const { std::lock_guard segment_lock(mutex); + assertNotDetached(segment_lock); + return getCallerId() == downloader_id; } diff --git a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp index 2b978c0adec..676b174e4d3 100644 --- a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp +++ b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp @@ -676,18 +676,18 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() return false; SCOPE_EXIT({ - /// Save state of current file segment before it is completed. - nextimpl_step_log_info = getInfoForLog(); - - if (current_file_segment_it == file_segments_holder->file_segments.end()) - return; - - auto & file_segment = *current_file_segment_it; - - bool download_current_segment = read_type == ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE; - if (download_current_segment) + try { - try + /// Save state of current file segment before it is completed. + nextimpl_step_log_info = getInfoForLog(); + + if (current_file_segment_it == file_segments_holder->file_segments.end()) + return; + + auto & file_segment = *current_file_segment_it; + + bool download_current_segment = read_type == ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE; + if (download_current_segment) { bool need_complete_file_segment = file_segment->isDownloader(); if (need_complete_file_segment) @@ -696,13 +696,13 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() file_segment->completeBatchAndResetDownloader(); } } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } - } - assert(!file_segment->isDownloader()); + assert(!file_segment->isDownloader()); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } }); bytes_to_predownload = 0; From 50d229651f5117f3a4369a0094f20f75ab04ff9f Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 7 May 2022 23:33:28 +0200 Subject: [PATCH 070/171] Fix exception message --- src/Common/FileSegment.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index 158549badbb..171bb8ff4ac 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -633,7 +633,9 @@ void FileSegment::assertDetachedStatus(std::lock_guard & segment_loc if (download_state != State::EMPTY && !hasFinalizedState()) { throw Exception( - ErrorCodes::LOGICAL_ERROR, "Cache get into inconsistent state {}", getInfoForLogImpl(segment_lock)); + ErrorCodes::LOGICAL_ERROR, + "Detached file segment has incorrent state: {}", + getInfoForLogImpl(segment_lock)); } } From f43c09b53c733e41b674cd95e1881958babb8201 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 8 May 2022 14:18:43 +0200 Subject: [PATCH 071/171] Fix incorrect assetion --- src/Common/FileSegment.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index 171bb8ff4ac..1d4ec4a9d8e 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -171,7 +171,8 @@ bool FileSegment::isDownloader() const { std::lock_guard segment_lock(mutex); - assertNotDetached(segment_lock); + if (is_forcefully_detached) + throwDetachedUnlocked(segment_lock); return getCallerId() == downloader_id; } From a17da05bdaff44045b449220f15782c87bdadeb5 Mon Sep 17 00:00:00 2001 From: qieqieplus Date: Tue, 19 Apr 2022 18:30:20 +0800 Subject: [PATCH 072/171] use return type as matrix value type --- src/Functions/array/arrayDistance.cpp | 122 ++++++++++-------- .../02280_array_distance.reference | 16 +-- .../0_stateless/02280_array_distance.sql | 3 + 3 files changed, 78 insertions(+), 63 deletions(-) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index 8004552a840..d405b9a4a94 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -1,30 +1,31 @@ -#include +#include #include #include #include #include #include - #include namespace DB { namespace ErrorCodes { - extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int LOGICAL_ERROR; + extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; } template struct LpDistance { static inline String name = "L" + std::to_string(N); - template - static void compute(const Eigen::MatrixBase & left, const Eigen::MatrixBase & right, PaddedPODArray & array) + template + static void compute(const Eigen::MatrixX & left, const Eigen::MatrixX & right, PaddedPODArray & array) { auto & norms = (left - right).colwise().template lpNorm(); + // array.insert() failed to work with Eigen iterators for (auto n : norms) - array.emplace_back(n); + array.push_back(n); } }; @@ -36,19 +37,16 @@ struct LinfDistance : LpDistance struct CosineDistance { static inline String name = "Cosine"; - template - static void compute(const Eigen::MatrixBase & left, const Eigen::MatrixBase & right, PaddedPODArray & array) + template + static void compute(const Eigen::MatrixX & left, const Eigen::MatrixX & right, PaddedPODArray & array) { - // auto & nx = left.colwise().normalized().eval(); - // auto & ny = right.colwise().normalized().eval(); - // auto & dist = 1.0 - x.cwiseProduct(y).colwise().sum().array(); auto & prod = left.cwiseProduct(right).colwise().sum(); auto & nx = left.colwise().norm(); auto & ny = right.colwise().norm(); auto & nm = nx.cwiseProduct(ny).cwiseInverse(); auto & dist = 1.0 - prod.cwiseProduct(nm).array(); for (auto d : dist) - array.emplace_back(d); + array.push_back(d); } }; @@ -63,30 +61,18 @@ public: bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } bool useDefaultImplementationForConstants() const override { return true; } - DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & /*arguments*/) const override + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - return std::make_shared(); - } - - ColumnPtr - executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override - { - const auto & left = arguments[0]; - const auto & right = arguments[1]; - - const auto & col_x = left.column->convertToFullColumnIfConst(); - const auto & col_y = right.column->convertToFullColumnIfConst(); - const auto * arr_x = checkAndGetColumn(col_x.get()); - const auto * arr_y = checkAndGetColumn(col_y.get()); - if (!arr_x || !arr_y) + DataTypes types; + for (const auto & argument : arguments) { - throw Exception("Argument of function " + String(name) + " must be array. ", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + const auto * array_type = checkAndGetDataType(argument.type.get()); + if (!array_type) + throw Exception("Argument of function " + getName() + " must be array. ", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + types.push_back(array_type->getNestedType()); } - - const auto & type_x = checkAndGetDataType(left.type.get())->getNestedType(); - const auto & type_y = checkAndGetDataType(right.type.get())->getNestedType(); - - const auto & common_type = getLeastSupertype(DataTypes{type_x, type_y}); + const auto & common_type = getLeastSupertype(types); switch (common_type->getTypeId()) { case TypeIndex::UInt8: @@ -96,31 +82,55 @@ public: case TypeIndex::Int16: case TypeIndex::Int32: case TypeIndex::Float32: - return executeWithType(*arr_x, *arr_y, type_x, type_y, result_type); + return std::make_shared(); case TypeIndex::UInt64: case TypeIndex::Int64: case TypeIndex::Float64: - return executeWithType(*arr_x, *arr_y, type_x, type_y, result_type); + return std::make_shared(); default: throw Exception( - "Array nested type " + common_type->getName() + "Arguments of function " + getName() + " has nested type " + common_type->getName() + ". Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } } + ColumnPtr + executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override + { + const auto & type_x = typeid_cast(arguments[0].type.get())->getNestedType(); + const auto & type_y = typeid_cast(arguments[1].type.get())->getNestedType(); + + const auto & col_x = arguments[0].column->convertToFullColumnIfConst(); + const auto & col_y = arguments[1].column->convertToFullColumnIfConst(); + + const auto * arr_x = assert_cast(col_x.get()); + const auto * arr_y = assert_cast(col_y.get()); + + auto result = result_type->createColumn(); + switch (result_type->getTypeId()) + { + case TypeIndex::Float32: + executeWithType(*arr_x, *arr_y, type_x, type_y, result); + break; + case TypeIndex::Float64: + executeWithType(*arr_x, *arr_y, type_x, type_y, result); + break; + default: + throw Exception("Unexpected result type.", ErrorCodes::LOGICAL_ERROR); + } + return result; + } + private: template - static ColumnPtr executeWithType( + void executeWithType( const ColumnArray & array_x, const ColumnArray & array_y, const DataTypePtr & type_x, const DataTypePtr & type_y, - const DataTypePtr & result_type) + MutableColumnPtr & column) const { - auto result = result_type->createColumn(); - auto & array = typeid_cast(*result).getData(); - Eigen::MatrixX mx, my; columnToMatrix(array_x, type_x, mx); columnToMatrix(array_y, type_y, my); @@ -128,15 +138,14 @@ private: if (mx.rows() && my.rows() && mx.rows() != my.rows()) { throw Exception( - "Arguments of function " + String(name) + " have different array sizes.", ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH); + "Arguments of function " + getName() + " have different array sizes.", ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH); } - - Kernel::compute(mx, my, array); - return result; + auto & data = assert_cast &>(*column).getData(); + Kernel::compute(mx, my, data); } template - static void columnToMatrix(const ColumnArray & array, const DataTypePtr & nested_type, Eigen::MatrixX & mat) + void columnToMatrix(const ColumnArray & array, const DataTypePtr & nested_type, Eigen::MatrixX & mat) const { switch (nested_type->getTypeId()) { @@ -172,7 +181,7 @@ private: break; default: throw Exception( - "Array nested type " + nested_type->getName() + "Arguments of function " + getName() + " has nested type " + nested_type->getName() + ". Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } @@ -180,25 +189,28 @@ private: // optimize for float/ double template ::value>::type> - static void fillMatrix(Eigen::MatrixX & mat, const ColumnArray & array) + void fillMatrix(Eigen::MatrixX & mat, const ColumnArray & array) const { - const auto & vec = typeid_cast &>(array.getData()); - const auto & data = vec.getData(); + const auto & data = typeid_cast &>(array.getData()).getData(); const auto & offsets = array.getOffsets(); mat = Eigen::Map>(data.data(), offsets.front(), offsets.size()); } template - static void fillMatrix(Eigen::MatrixX & mat, const ColumnArray & array) + void fillMatrix(Eigen::MatrixX & mat, const ColumnArray & array) const { - const auto & vec = typeid_cast &>(array.getData()); - const auto & data = vec.getData(); + const auto & data = typeid_cast &>(array.getData()).getData(); const auto & offsets = array.getOffsets(); - mat.resize(offsets.front(), offsets.size()); + size_t rows = offsets.front(), cols = offsets.size(); + mat.resize(rows, cols); ColumnArray::Offset prev = 0, col = 0; - for (auto off : offsets) + for (ColumnArray::Offset off : offsets) { + if (off - prev != rows) + throw Exception( + "Arguments of function " + getName() + " have different array sizes.", ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH); + for (ColumnArray::Offset row = 0; row < off - prev; ++row) { mat(row, col) = static_cast(data[prev + row]); diff --git a/tests/queries/0_stateless/02280_array_distance.reference b/tests/queries/0_stateless/02280_array_distance.reference index 9de9521befe..75e24833e1e 100644 --- a/tests/queries/0_stateless/02280_array_distance.reference +++ b/tests/queries/0_stateless/02280_array_distance.reference @@ -19,12 +19,12 @@ nan 6 8 9 -0.020204103 -0.118082896 +0.020204102886728692 +0.11808289631180302 0 -1 1 218.74643 -1 2 1348.2118 -2 1 219.28064 -2 2 1347.4009 -3 1 214.35251 -3 2 1342.8857 +1 1 218.74642854227358 +1 2 1348.2117786164013 +2 1 219.28064210048274 +2 2 1347.4008312302617 +3 1 214.35251339790725 +3 2 1342.8856987845243 diff --git a/tests/queries/0_stateless/02280_array_distance.sql b/tests/queries/0_stateless/02280_array_distance.sql index 0f269b11179..835b2cf15ae 100644 --- a/tests/queries/0_stateless/02280_array_distance.sql +++ b/tests/queries/0_stateless/02280_array_distance.sql @@ -22,6 +22,9 @@ SELECT arrayCosineDistance(v, materialize([1., 1., 1.])) FROM vec1; INSERT INTO vec2 VALUES (1, [100, 200, 0]), (2, [888, 777, 666]); SELECT v1.id, v2.id, arrayL2Distance(v1.v, v2.v) as dist FROM vec1 v1, vec2 v2; +INSERT INTO vec2 VALUES (3, [123]); +SELECT v1.id, v2.id, arrayL2Distance(v1.v, v2.v) as dist FROM vec1 v1, vec2 v2; -- { serverError 190 } + SELECT arrayL1Distance([0, 0], [1]); -- { serverError 190 } SELECT arrayL2Distance((1, 2), (3,4)); -- { serverError 43 } From 307511aab4f618d013671d468c2f23db5660e953 Mon Sep 17 00:00:00 2001 From: qieqieplus Date: Mon, 9 May 2022 14:41:12 +0800 Subject: [PATCH 073/171] impl norm functions for array --- src/Functions/array/arrayNorm.cpp | 195 ++++++++++++++++++ .../array/registerFunctionsArray.cpp | 2 + 2 files changed, 197 insertions(+) create mode 100644 src/Functions/array/arrayNorm.cpp diff --git a/src/Functions/array/arrayNorm.cpp b/src/Functions/array/arrayNorm.cpp new file mode 100644 index 00000000000..cf8ae7524dc --- /dev/null +++ b/src/Functions/array/arrayNorm.cpp @@ -0,0 +1,195 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int LOGICAL_ERROR; +} + +template +struct LpNorm +{ + static inline String name = "L" + std::to_string(N); + template + static void compute(const std::vector> & vec, PaddedPODArray & array) + { + for (const auto & v : vec) + { + array.push_back(v.template lpNorm()); + } + } +}; + +struct LinfNorm : LpNorm +{ + static inline String name = "Linf"; +}; + +template +class FunctionArrayNorm : public IFunction +{ +public: + static inline auto name = "array" + Kernel::name + "Norm"; + String getName() const override { return name; } + static FunctionPtr create(ContextPtr) { return std::make_shared>(); } + size_t getNumberOfArguments() const override { return 1; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + DataTypes types; + for (const auto & argument : arguments) + { + const auto * array_type = checkAndGetDataType(argument.type.get()); + if (!array_type) + throw Exception("Argument of function " + getName() + " must be array. ", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + types.push_back(array_type->getNestedType()); + } + const auto & common_type = getLeastSupertype(types); + switch (common_type->getTypeId()) + { + case TypeIndex::UInt8: + case TypeIndex::UInt16: + case TypeIndex::UInt32: + case TypeIndex::Int8: + case TypeIndex::Int16: + case TypeIndex::Int32: + case TypeIndex::Float32: + return std::make_shared(); + case TypeIndex::UInt64: + case TypeIndex::Int64: + case TypeIndex::Float64: + return std::make_shared(); + default: + throw Exception( + "Arguments of function " + getName() + " has nested type " + common_type->getName() + + ". Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + } + + ColumnPtr + executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override + { + const auto & type = typeid_cast(arguments[0].type.get())->getNestedType(); + const auto column = arguments[0].column->convertToFullColumnIfConst(); + const auto * arr = assert_cast(column.get()); + + auto result = result_type->createColumn(); + switch (result_type->getTypeId()) + { + case TypeIndex::Float32: + executeWithType(*arr, type, result); + break; + case TypeIndex::Float64: + executeWithType(*arr, type, result); + break; + default: + throw Exception("Unexpected result type.", ErrorCodes::LOGICAL_ERROR); + } + return result; + } + +private: + template + void executeWithType(const ColumnArray & array, const DataTypePtr & type, MutableColumnPtr & column) const + { + std::vector> v; + columnToVectors(array, type, v); + auto & data = assert_cast &>(*column).getData(); + Kernel::compute(v, data); + } + + template + void columnToVectors(const ColumnArray & array, const DataTypePtr & nested_type, std::vector> & vec) const + { + switch (nested_type->getTypeId()) + { + case TypeIndex::UInt8: + fillVectors(vec, array); + break; + case TypeIndex::UInt16: + fillVectors(vec, array); + break; + case TypeIndex::UInt32: + fillVectors(vec, array); + break; + case TypeIndex::UInt64: + fillVectors(vec, array); + break; + case TypeIndex::Int8: + fillVectors(vec, array); + break; + case TypeIndex::Int16: + fillVectors(vec, array); + break; + case TypeIndex::Int32: + fillVectors(vec, array); + break; + case TypeIndex::Int64: + fillVectors(vec, array); + break; + case TypeIndex::Float32: + fillVectors(vec, array); + break; + case TypeIndex::Float64: + fillVectors(vec, array); + break; + default: + throw Exception( + "Arguments of function " + getName() + " has nested type " + nested_type->getName() + + ". Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + } + template ::value>::type> + void fillVectors(std::vector> & vec, const ColumnArray & array) const + { + const auto & data = typeid_cast &>(array.getData()).getData(); + const auto & offsets = array.getOffsets(); + ColumnArray::Offset prev = 0; + for (auto off : offsets) + { + vec.emplace_back(Eigen::Map>(data.data() + prev, off - prev)); + prev = off; + } + } + + template + void fillVectors(std::vector> & vec, const ColumnArray & array) const + { + const auto & data = typeid_cast &>(array.getData()).getData(); + const auto & offsets = array.getOffsets(); + + ColumnArray::Offset prev = 0; + for (auto off : offsets) + { + Eigen::VectorX mat(off - prev); + for (ColumnArray::Offset row = 0; row + prev < off; ++row) + { + mat[row] = static_cast(data[prev + row]); + } + prev = off; + vec.emplace_back(mat); + } + } +}; + +void registerFunctionArrayNorm(FunctionFactory & factory) +{ + factory.registerFunction>>(); + factory.registerFunction>>(); + factory.registerFunction>(); +} + +} diff --git a/src/Functions/array/registerFunctionsArray.cpp b/src/Functions/array/registerFunctionsArray.cpp index f24a2023d40..e2e8b08fbf2 100644 --- a/src/Functions/array/registerFunctionsArray.cpp +++ b/src/Functions/array/registerFunctionsArray.cpp @@ -38,6 +38,7 @@ void registerFunctionArrayReduceInRanges(FunctionFactory &); void registerFunctionMapOp(FunctionFactory &); void registerFunctionMapPopulateSeries(FunctionFactory &); void registerFunctionArrayDistance(FunctionFactory &); +void registerFunctionArrayNorm(FunctionFactory &); void registerFunctionsArray(FunctionFactory & factory) { @@ -77,6 +78,7 @@ void registerFunctionsArray(FunctionFactory & factory) registerFunctionMapOp(factory); registerFunctionMapPopulateSeries(factory); registerFunctionArrayDistance(factory); + registerFunctionArrayNorm(factory); } } From 96352034a3e3ba0671bc918117e9010eb9ba29a9 Mon Sep 17 00:00:00 2001 From: qieqieplus Date: Mon, 9 May 2022 15:11:18 +0800 Subject: [PATCH 074/171] add array norm tests --- .../queries/0_stateless/02280_array_norm.reference | 11 +++++++++++ tests/queries/0_stateless/02280_array_norm.sql | 13 +++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 tests/queries/0_stateless/02280_array_norm.reference create mode 100644 tests/queries/0_stateless/02280_array_norm.sql diff --git a/tests/queries/0_stateless/02280_array_norm.reference b/tests/queries/0_stateless/02280_array_norm.reference new file mode 100644 index 00000000000..2d22f2967a2 --- /dev/null +++ b/tests/queries/0_stateless/02280_array_norm.reference @@ -0,0 +1,11 @@ +6 +7.0710678118654755 +2 +1 5 +2 2 +3 5.196152 +4 0 +1 11 +2 11 +3 11 +4 11 diff --git a/tests/queries/0_stateless/02280_array_norm.sql b/tests/queries/0_stateless/02280_array_norm.sql new file mode 100644 index 00000000000..e5a310bfa1e --- /dev/null +++ b/tests/queries/0_stateless/02280_array_norm.sql @@ -0,0 +1,13 @@ +SELECT arrayL1Norm([1, 2, 3]); +SELECT arrayL2Norm([3., 4., 5.]); +SELECT arrayLinfNorm([0, 0, 2]); + +DROP TABLE IF EXISTS vec; +CREATE TABLE vec (id UInt64, v Array(UInt8)) ENGINE = Memory; +INSERT INTO vec VALUES (1, [3, 4]), (2, [2]), (3, [3, 3, 3]), (4, NULL); + +SELECT id, arrayL2Norm(v) FROM vec; +SELECT id, arrayL1Norm(materialize([5., 6.])) FROM vec; +SELECT arrayL1Norm((1, 2,)); -- { serverError 43 } + +DROP TABLE vec; From 2f0f76369ab925b497cdf0956cb204c9252f44f2 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 9 May 2022 21:20:10 +0200 Subject: [PATCH 075/171] Better --- src/Common/ProfileEvents.cpp | 6 ++ src/Disks/IO/CachedReadBufferFromRemoteFS.cpp | 90 ++++++++++++++++--- src/Disks/IO/CachedReadBufferFromRemoteFS.h | 4 + src/IO/ReadBufferFromFile.h | 2 - src/IO/ReadBufferFromFileDescriptor.h | 2 + src/Interpreters/FilesystemCacheLog.cpp | 18 ++++ src/Interpreters/FilesystemCacheLog.h | 3 + 7 files changed, 112 insertions(+), 13 deletions(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 7d811d6c1ee..32a186903ff 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -275,6 +275,12 @@ M(RemoteFSSeeksWithReset, "Number of seeks which lead to a new connection") \ M(RemoteFSBuffers, "Number of buffers created for asynchronous reading from remote filesystem") \ \ + M(FileSegmentWaitReadBufferMicroseconds, "") \ + M(FileSegmentReadMicroseconds, "") \ + M(FileSegmentCacheWriteMicroseconds, "") \ + M(FileSegmentPredownloadMicroseconds, "") \ + M(FileSegmentResultBytes, "") \ + \ M(ReadBufferSeekCancelConnection, "Number of seeks which lead to new connection (s3, http)") \ \ M(SleepFunctionCalls, "Number of times a sleep function (sleep, sleepEachRow) has been called.") \ diff --git a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp index 7ffac5cd08e..e4bee1761f6 100644 --- a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp +++ b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace ProfileEvents @@ -12,6 +13,11 @@ namespace ProfileEvents extern const Event RemoteFSReadBytes; extern const Event RemoteFSCacheReadBytes; extern const Event RemoteFSCacheDownloadBytes; +extern const Event FileSegmentWaitReadBufferMicroseconds; +extern const Event FileSegmentReadMicroseconds; +extern const Event FileSegmentCacheWriteMicroseconds; +extern const Event FileSegmentPredownloadMicroseconds; +extern const Event FileSegmentResultBytes; } namespace DB @@ -50,6 +56,7 @@ CachedReadBufferFromRemoteFS::CachedReadBufferFromRemoteFS( , remote_file_reader_creator(remote_file_reader_creator_) , query_id(getQueryId()) , enable_logging(!query_id.empty() && CurrentThread::get().getQueryContext()->getSettingsRef().enable_filesystem_cache_log) + , current_buffer_id(getRandomASCIIString(8)) { } @@ -62,9 +69,14 @@ void CachedReadBufferFromRemoteFS::appendFilesystemCacheLog( .query_id = query_id, .source_file_path = remote_fs_object_path, .file_segment_range = { file_segment_range.left, file_segment_range.right }, + .requested_range = { first_offset, read_until_position }, .file_segment_size = file_segment_range.size(), + .read_buffer_id = current_buffer_id, + .profile_counters = std::make_shared(current_file_segment_counters.getPartiallyAtomicSnapshot()), }; + current_file_segment_counters.reset(); + switch (type) { case CachedReadBufferFromRemoteFS::ReadType::CACHED: @@ -109,9 +121,16 @@ void CachedReadBufferFromRemoteFS::initialize(size_t offset, size_t size) SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getCacheReadBuffer(size_t offset) const { auto path = cache->getPathInLocalCache(cache_key, offset); - auto buf = std::make_shared(path, settings.local_fs_buffer_size); - if (buf->size() == 0) + + ReadSettings local_read_settings{settings}; + /// Do not allow to use asynchronous version of LocalFSReadMethod. + local_read_settings.local_fs_method = LocalFSReadMethod::pread; + + auto buf = createReadBufferFromFileBase(path, local_read_settings); + auto from_fd = dynamic_cast(buf.get()); + if (from_fd && from_fd->size() == 0) throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read from an empty cache file: {}", path); + return buf; } @@ -340,8 +359,13 @@ SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getImplementationBuffer(File auto range = file_segment->range(); bytes_to_predownload = 0; + Stopwatch watch(CLOCK_MONOTONIC); + auto read_buffer_for_file_segment = getReadBufferForFileSegment(file_segment); + watch.stop(); + current_file_segment_counters.increment(ProfileEvents::FileSegmentWaitReadBufferMicroseconds, watch.elapsedMicroseconds()); + [[maybe_unused]] auto download_current_segment = read_type == ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE; assert(download_current_segment == file_segment->isDownloader()); @@ -362,7 +386,7 @@ SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getImplementationBuffer(File case ReadType::CACHED: { #ifndef NDEBUG - auto * file_reader = assert_cast(read_buffer_for_file_segment.get()); + auto * file_reader = dynamic_cast(read_buffer_for_file_segment.get()); size_t file_size = file_reader->size(); if (file_size == 0 || range.left + file_size <= file_offset_of_buffer_end) @@ -436,6 +460,9 @@ bool CachedReadBufferFromRemoteFS::completeFileSegmentAndGetNext() { LOG_TEST(log, "Completed segment: {}", (*current_file_segment_it)->range().toString()); + if (enable_logging) + appendFilesystemCacheLog((*current_file_segment_it)->range(), read_type); + auto file_segment_it = current_file_segment_it++; auto & file_segment = *file_segment_it; @@ -460,15 +487,29 @@ bool CachedReadBufferFromRemoteFS::completeFileSegmentAndGetNext() if (read_type == ReadType::CACHED) (*current_file_segment_it)->incrementHitsCount(); - if (enable_logging) - appendFilesystemCacheLog((*current_file_segment_it)->range(), read_type); LOG_TEST(log, "New segment: {}", (*current_file_segment_it)->range().toString()); return true; } +CachedReadBufferFromRemoteFS::~CachedReadBufferFromRemoteFS() +{ + if (enable_logging + && file_segments_holder + && current_file_segment_it != file_segments_holder->file_segments.end()) + { + appendFilesystemCacheLog((*current_file_segment_it)->range(), read_type); + } +} + void CachedReadBufferFromRemoteFS::predownload(FileSegmentPtr & file_segment) { + Stopwatch predownload_watch(CLOCK_MONOTONIC); + SCOPE_EXIT({ + predownload_watch.stop(); + current_file_segment_counters.increment(ProfileEvents::FileSegmentPredownloadMicroseconds, predownload_watch.elapsedMicroseconds()); + }); + if (bytes_to_predownload) { /// Consider this case. Some user needed segment [a, b] and downloaded it partially. @@ -484,7 +525,17 @@ void CachedReadBufferFromRemoteFS::predownload(FileSegmentPtr & file_segment) while (true) { - if (!bytes_to_predownload || implementation_buffer->eof()) + bool has_more_data; + { + Stopwatch watch(CLOCK_MONOTONIC); + + has_more_data = !implementation_buffer->eof(); + + watch.stop(); + current_file_segment_counters.increment(ProfileEvents::FileSegmentReadMicroseconds, watch.elapsedMicroseconds()); + } + + if (!bytes_to_predownload || !has_more_data) { if (bytes_to_predownload) throw Exception( @@ -531,7 +582,13 @@ void CachedReadBufferFromRemoteFS::predownload(FileSegmentPtr & file_segment) assert(file_segment->getDownloadOffset() == static_cast(implementation_buffer->getPosition())); + Stopwatch watch(CLOCK_MONOTONIC); + file_segment->write(implementation_buffer->buffer().begin(), current_predownload_size, current_offset); + + watch.stop(); + current_file_segment_counters.increment(ProfileEvents::FileSegmentCacheWriteMicroseconds, watch.elapsedMicroseconds()); + ProfileEvents::increment(ProfileEvents::RemoteFSCacheDownloadBytes, current_predownload_size); current_offset += current_predownload_size; @@ -711,9 +768,6 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() if (read_type == ReadType::CACHED) (*current_file_segment_it)->incrementHitsCount(); - - if (enable_logging) - appendFilesystemCacheLog((*current_file_segment_it)->range(), read_type); } assert(!internal_buffer.empty()); @@ -758,7 +812,7 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() if (!result) { #ifndef NDEBUG - if (auto * cache_file_reader = typeid_cast(implementation_buffer.get())) + if (auto * cache_file_reader = dynamic_cast(implementation_buffer.get())) { auto cache_file_size = cache_file_reader->size(); if (cache_file_size == 0) @@ -767,7 +821,13 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() } #endif + Stopwatch watch(CLOCK_MONOTONIC); + result = implementation_buffer->next(); + + watch.stop(); + current_file_segment_counters.increment(ProfileEvents::FileSegmentReadMicroseconds, watch.elapsedMicroseconds()); + size = implementation_buffer->buffer().size(); if (read_type == ReadType::CACHED) @@ -786,11 +846,16 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() { assert(file_segment->getDownloadOffset() == static_cast(implementation_buffer->getPosition())); + Stopwatch watch(CLOCK_MONOTONIC); + file_segment->write( needed_to_predownload ? implementation_buffer->position() : implementation_buffer->buffer().begin(), size, file_offset_of_buffer_end); + watch.stop(); + current_file_segment_counters.increment(ProfileEvents::FileSegmentCacheWriteMicroseconds, watch.elapsedMicroseconds()); + ProfileEvents::increment(ProfileEvents::RemoteFSCacheDownloadBytes, size); assert(file_segment->getDownloadOffset() <= file_segment->range().right + 1); @@ -819,10 +884,13 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() } file_offset_of_buffer_end += size; + } swap(*implementation_buffer); + current_file_segment_counters.increment(ProfileEvents::FileSegmentResultBytes, available()); + if (download_current_segment) file_segment->completeBatchAndResetDownloader(); @@ -851,7 +919,7 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() if (size == 0 && file_offset_of_buffer_end < read_until_position) { std::optional cache_file_size; - if (auto * cache_file_reader = dynamic_cast(implementation_buffer.get())) + if (auto * cache_file_reader = dynamic_cast(implementation_buffer.get())) cache_file_size = cache_file_reader->size(); throw Exception( diff --git a/src/Disks/IO/CachedReadBufferFromRemoteFS.h b/src/Disks/IO/CachedReadBufferFromRemoteFS.h index 62bb070b437..7eb5479fbc7 100644 --- a/src/Disks/IO/CachedReadBufferFromRemoteFS.h +++ b/src/Disks/IO/CachedReadBufferFromRemoteFS.h @@ -28,6 +28,8 @@ public: const ReadSettings & settings_, size_t read_until_position_); + ~CachedReadBufferFromRemoteFS() override; + bool nextImpl() override; off_t seek(off_t off, int whence) override; @@ -116,8 +118,10 @@ private: String query_id; bool enable_logging = false; + String current_buffer_id; CurrentMetrics::Increment metric_increment{CurrentMetrics::FilesystemCacheReadBuffers}; + ProfileEvents::Counters current_file_segment_counters; }; } diff --git a/src/IO/ReadBufferFromFile.h b/src/IO/ReadBufferFromFile.h index 52b18b94616..a720f8dd36d 100644 --- a/src/IO/ReadBufferFromFile.h +++ b/src/IO/ReadBufferFromFile.h @@ -50,8 +50,6 @@ public: return file_name; } - Range getRemainingReadRange() const override { return Range{ .left = file_offset_of_buffer_end, .right = std::nullopt }; } - size_t getFileOffsetOfBufferEnd() const override { return file_offset_of_buffer_end; } }; diff --git a/src/IO/ReadBufferFromFileDescriptor.h b/src/IO/ReadBufferFromFileDescriptor.h index ba1502fb9aa..a1d19c08087 100644 --- a/src/IO/ReadBufferFromFileDescriptor.h +++ b/src/IO/ReadBufferFromFileDescriptor.h @@ -49,6 +49,8 @@ public: return file_offset_of_buffer_end - (working_buffer.end() - pos); } + Range getRemainingReadRange() const override { return Range{ .left = file_offset_of_buffer_end, .right = std::nullopt }; } + /// If 'offset' is small enough to stay in buffer after seek, then true seek in file does not happen. off_t seek(off_t off, int whence) override; diff --git a/src/Interpreters/FilesystemCacheLog.cpp b/src/Interpreters/FilesystemCacheLog.cpp index 9eec65a38ab..cae80453b35 100644 --- a/src/Interpreters/FilesystemCacheLog.cpp +++ b/src/Interpreters/FilesystemCacheLog.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include @@ -35,8 +37,11 @@ NamesAndTypesList FilesystemCacheLogElement::getNamesAndTypes() {"query_id", std::make_shared()}, {"source_file_path", std::make_shared()}, {"file_segment_range", std::make_shared(std::move(types))}, + {"total_requested_range", std::make_shared(std::move(types))}, {"size", std::make_shared()}, {"read_type", std::make_shared()}, + {"ProfileEvents", std::make_shared(std::make_shared(), std::make_shared())}, + {"read_buffer_id", std::make_shared()}, }; } @@ -51,8 +56,21 @@ void FilesystemCacheLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(source_file_path); columns[i++]->insert(Tuple{file_segment_range.first, file_segment_range.second}); + columns[i++]->insert(Tuple{requested_range.first, requested_range.second}); columns[i++]->insert(file_segment_size); columns[i++]->insert(typeToString(read_type)); + + if (profile_counters) + { + auto * column = columns[i++].get(); + ProfileEvents::dumpToMapColumn(*profile_counters, column, true); + } + else + { + columns[i++]->insertDefault(); + } + + columns[i++]->insert(read_buffer_id); } }; diff --git a/src/Interpreters/FilesystemCacheLog.h b/src/Interpreters/FilesystemCacheLog.h index 208f53a76cf..a89dde12014 100644 --- a/src/Interpreters/FilesystemCacheLog.h +++ b/src/Interpreters/FilesystemCacheLog.h @@ -37,8 +37,11 @@ struct FilesystemCacheLogElement String source_file_path; std::pair file_segment_range{}; + std::pair requested_range{}; ReadType read_type{}; size_t file_segment_size; + String read_buffer_id; + std::shared_ptr profile_counters; static std::string name() { return "FilesystemCacheLog"; } From 43935f71f7ca0e5ff72db3d6222cfd00805c498b Mon Sep 17 00:00:00 2001 From: Vladimir C Date: Tue, 10 May 2022 14:09:49 +0200 Subject: [PATCH 076/171] Update ConcurrentHashJoin.cpp --- src/Interpreters/ConcurrentHashJoin.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/ConcurrentHashJoin.cpp b/src/Interpreters/ConcurrentHashJoin.cpp index 7dfe6a5549e..f4ab8555d5f 100644 --- a/src/Interpreters/ConcurrentHashJoin.cpp +++ b/src/Interpreters/ConcurrentHashJoin.cpp @@ -18,7 +18,6 @@ #include #include #include -#include namespace DB { @@ -52,7 +51,7 @@ bool ConcurrentHashJoin::addJoinedBlock(const Block & right_block, bool check_li { Blocks dispatched_blocks = dispatchBlock(table_join->getOnlyClause().key_names_right, right_block); - std::atomic blocks_left = 0; + size_t blocks_left = 0; for (const auto & block : dispatched_blocks) { if (block) @@ -68,15 +67,14 @@ bool ConcurrentHashJoin::addJoinedBlock(const Block & right_block, bool check_li { auto & hash_join = hash_joins[i]; auto & dispatched_block = dispatched_blocks[i]; + + if (dispatched_block) { /// if current hash_join is already processed by another thread, skip it and try later std::unique_lock lock(hash_join->mutex, std::try_to_lock); if (!lock.owns_lock()) continue; - if (!dispatched_block) - continue; - bool limit_exceeded = !hash_join->data->addJoinedBlock(dispatched_block, check_limits); dispatched_block = {}; From 4065b83ba99c6b394e5b4d9ec7fa905d420c795a Mon Sep 17 00:00:00 2001 From: Vladimir C Date: Tue, 10 May 2022 14:14:58 +0200 Subject: [PATCH 077/171] Update ConcurrentHashJoin.cpp --- src/Interpreters/ConcurrentHashJoin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/ConcurrentHashJoin.cpp b/src/Interpreters/ConcurrentHashJoin.cpp index f4ab8555d5f..f6ba9f95bbc 100644 --- a/src/Interpreters/ConcurrentHashJoin.cpp +++ b/src/Interpreters/ConcurrentHashJoin.cpp @@ -34,7 +34,7 @@ ConcurrentHashJoin::ConcurrentHashJoin(ContextPtr context_, std::shared_ptr Date: Tue, 10 May 2022 14:59:52 +0200 Subject: [PATCH 078/171] Disable log_query_threads by default --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 76d2dca117b..6fb52c0591d 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -379,7 +379,7 @@ static constexpr UInt64 operator""_Gb(unsigned long long value) \ M(Bool, log_profile_events, true, "Log query performance statistics into the query_log, query_thread_log and query_views_log.", 0) \ M(Bool, log_query_settings, true, "Log query settings into the query_log.", 0) \ - M(Bool, log_query_threads, true, "Log query threads into system.query_thread_log table. This setting have effect only when 'log_queries' is true.", 0) \ + M(Bool, log_query_threads, false, "Log query threads into system.query_thread_log table. This setting have effect only when 'log_queries' is true.", 0) \ M(Bool, log_query_views, true, "Log query dependent views into system.query_views_log table. This setting have effect only when 'log_queries' is true.", 0) \ M(String, log_comment, "", "Log comment into system.query_log table and server log. It can be set to arbitrary string no longer than max_query_size.", 0) \ M(LogsLevel, send_logs_level, LogsLevel::fatal, "Send server text logs with specified minimum level to client. Valid values: 'trace', 'debug', 'information', 'warning', 'error', 'fatal', 'none'", 0) \ From 0feda67ec4f30e1dff8cd80564c6f3cd99da5276 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 6 May 2022 17:04:03 +0200 Subject: [PATCH 079/171] Fix progress for insert select --- programs/client/Client.cpp | 2 +- src/Client/ClientBase.cpp | 12 +++--- src/Client/Connection.cpp | 3 +- src/Client/Connection.h | 3 +- src/Client/HedgedConnections.cpp | 2 +- src/Client/IServerConnection.h | 3 +- src/Client/LocalConnection.cpp | 21 +++++++++-- src/Client/LocalConnection.h | 3 +- src/Client/MultiplexedConnections.cpp | 4 +- src/Client/Suggest.cpp | 2 +- src/Common/ProgressIndication.cpp | 11 +++--- src/IO/Progress.cpp | 41 +++++++++++++++------ src/IO/Progress.h | 14 +++---- src/Processors/Sources/SourceWithProgress.h | 2 + src/QueryPipeline/RemoteInserter.cpp | 2 +- src/Server/GRPCServer.cpp | 2 +- src/Server/TCPHandler.cpp | 3 +- src/Storages/StorageFile.cpp | 20 ++++++++-- 18 files changed, 101 insertions(+), 49 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index d8636484cba..660ef7a5de1 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -107,7 +107,7 @@ std::vector Client::loadWarningMessages() connection->sendQuery(connection_parameters.timeouts, "SELECT message FROM system.warnings", "" /* query_id */, QueryProcessingStage::Complete, &global_context->getSettingsRef(), - &global_context->getClientInfo(), false); + &global_context->getClientInfo(), false, {}); while (true) { Packet packet = connection->receivePacket(); diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index def74fa0ec9..86a6f64b6ff 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -719,7 +719,8 @@ void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr pa query_processing_stage, &global_context->getSettingsRef(), &global_context->getClientInfo(), - true); + true, + [&](const Progress & progress) { onProgress(progress); }); if (send_external_tables) sendExternalTables(parsed_query); @@ -1072,7 +1073,8 @@ void ClientBase::processInsertQuery(const String & query_to_execute, ASTPtr pars query_processing_stage, &global_context->getSettingsRef(), &global_context->getClientInfo(), - true); + true, + [&](const Progress & progress) { onProgress(progress); }); if (send_external_tables) sendExternalTables(parsed_query); @@ -1104,7 +1106,9 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des if (!parsed_insert_query) return; - if (need_render_progress) + bool have_data_in_stdin = !is_interactive && !stdin_is_a_tty && !std_in.eof(); + + if (need_render_progress && have_data_in_stdin) { /// Set total_bytes_to_read for current fd. FileProgress file_progress(0, std_in.size()); @@ -1114,8 +1118,6 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des progress_indication.setFileProgressCallback(global_context, true); } - bool have_data_in_stdin = !is_interactive && !stdin_is_a_tty && !std_in.eof(); - /// If data fetched from file (maybe compressed file) if (parsed_insert_query->infile) { diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 021ae2142a1..e53d55f6964 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -451,7 +451,8 @@ void Connection::sendQuery( UInt64 stage, const Settings * settings, const ClientInfo * client_info, - bool with_pending_data) + bool with_pending_data, + std::function) { if (!connected) connect(timeouts); diff --git a/src/Client/Connection.h b/src/Client/Connection.h index e34a0a22f42..d00a5760a8d 100644 --- a/src/Client/Connection.h +++ b/src/Client/Connection.h @@ -100,7 +100,8 @@ public: UInt64 stage/* = QueryProcessingStage::Complete */, const Settings * settings/* = nullptr */, const ClientInfo * client_info/* = nullptr */, - bool with_pending_data/* = false */) override; + bool with_pending_data/* = false */, + std::function process_progress_callback) override; void sendCancel() override; diff --git a/src/Client/HedgedConnections.cpp b/src/Client/HedgedConnections.cpp index 665f98a88d6..954396af0fa 100644 --- a/src/Client/HedgedConnections.cpp +++ b/src/Client/HedgedConnections.cpp @@ -179,7 +179,7 @@ void HedgedConnections::sendQuery( modified_settings.parallel_replica_offset = fd_to_replica_location[replica.packet_receiver->getFileDescriptor()].offset; } - replica.connection->sendQuery(timeouts, query, query_id, stage, &modified_settings, &client_info, with_pending_data); + replica.connection->sendQuery(timeouts, query, query_id, stage, &modified_settings, &client_info, with_pending_data, {}); replica.change_replica_timeout.setRelative(timeouts.receive_data_timeout); replica.packet_receiver->setReceiveTimeout(hedged_connections_factory.getConnectionTimeouts().receive_timeout); }; diff --git a/src/Client/IServerConnection.h b/src/Client/IServerConnection.h index 861630a942b..dfa4873e426 100644 --- a/src/Client/IServerConnection.h +++ b/src/Client/IServerConnection.h @@ -90,7 +90,8 @@ public: UInt64 stage, const Settings * settings, const ClientInfo * client_info, - bool with_pending_data) = 0; + bool with_pending_data, + std::function process_progress_callback) = 0; virtual void sendCancel() = 0; diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index 49e40ef8571..77519423763 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -74,13 +74,14 @@ void LocalConnection::sendQuery( UInt64 stage, const Settings *, const ClientInfo *, - bool) + bool, + std::function process_progress_callback) { query_context = session.makeQueryContext(); query_context->setCurrentQueryId(query_id); if (send_progress) { - query_context->setProgressCallback([this] (const Progress & value) { return this->updateProgress(value); }); + query_context->setProgressCallback([this] (const Progress & value) { this->updateProgress(value); }); query_context->setFileProgressCallback([this](const FileProgress & value) { this->updateProgress(Progress(value)); }); } if (!current_database.empty()) @@ -143,6 +144,19 @@ void LocalConnection::sendQuery( else if (state->io.pipeline.completed()) { CompletedPipelineExecutor executor(state->io.pipeline); + if (process_progress_callback) + { + auto callback = [this, &process_progress_callback]() + { + if (state->is_cancelled) + return true; + + process_progress_callback(state->progress.fetchAndResetPiecewiseAtomically()); + return false; + }; + + executor.setCancelCallback(callback, query_context->getSettingsRef().interactive_delay / 1000); + } executor.execute(); } @@ -185,6 +199,7 @@ void LocalConnection::sendData(const Block & block, const String &, bool) void LocalConnection::sendCancel() { + state->is_cancelled = true; if (state->executor) state->executor->cancel(); } @@ -440,7 +455,7 @@ Packet LocalConnection::receivePacket() } case Protocol::Server::Progress: { - packet.progress = std::move(state->progress); + packet.progress = state->progress.fetchAndResetPiecewiseAtomically(); state->progress.reset(); next_packet_type.reset(); break; diff --git a/src/Client/LocalConnection.h b/src/Client/LocalConnection.h index ad6f94122cc..1ad6ad73238 100644 --- a/src/Client/LocalConnection.h +++ b/src/Client/LocalConnection.h @@ -98,7 +98,8 @@ public: UInt64 stage/* = QueryProcessingStage::Complete */, const Settings * settings/* = nullptr */, const ClientInfo * client_info/* = nullptr */, - bool with_pending_data/* = false */) override; + bool with_pending_data/* = false */, + std::function process_progress_callback) override; void sendCancel() override; diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp index 31fbc609bdc..b14ff9f2c8d 100644 --- a/src/Client/MultiplexedConnections.cpp +++ b/src/Client/MultiplexedConnections.cpp @@ -161,14 +161,14 @@ void MultiplexedConnections::sendQuery( modified_settings.parallel_replica_offset = i; replica_states[i].connection->sendQuery(timeouts, query, query_id, - stage, &modified_settings, &client_info, with_pending_data); + stage, &modified_settings, &client_info, with_pending_data, {}); } } else { /// Use single replica. replica_states[0].connection->sendQuery(timeouts, query, query_id, - stage, &modified_settings, &client_info, with_pending_data); + stage, &modified_settings, &client_info, with_pending_data, {}); } sent_query = true; diff --git a/src/Client/Suggest.cpp b/src/Client/Suggest.cpp index 84625a768bf..de09c07f4c1 100644 --- a/src/Client/Suggest.cpp +++ b/src/Client/Suggest.cpp @@ -132,7 +132,7 @@ void Suggest::load(ContextPtr context, const ConnectionParameters & connection_p void Suggest::fetch(IServerConnection & connection, const ConnectionTimeouts & timeouts, const std::string & query) { - connection.sendQuery(timeouts, query, "" /* query_id */, QueryProcessingStage::Complete, nullptr, nullptr, false); + connection.sendQuery(timeouts, query, "" /* query_id */, QueryProcessingStage::Complete, nullptr, nullptr, false, {}); while (true) { diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp index 7a8cff2d58f..ffc90807060 100644 --- a/src/Common/ProgressIndication.cpp +++ b/src/Common/ProgressIndication.cpp @@ -165,18 +165,17 @@ void ProgressIndication::writeProgress() message << '\r'; size_t prefix_size = message.count(); - size_t read_bytes = progress.read_raw_bytes ? progress.read_raw_bytes : progress.read_bytes; message << indicator << " Progress: "; message << formatReadableQuantity(progress.read_rows) << " rows, " - << formatReadableSizeWithDecimalSuffix(read_bytes); + << formatReadableSizeWithDecimalSuffix(progress.read_bytes); auto elapsed_ns = watch.elapsed(); if (elapsed_ns) message << " (" << formatReadableQuantity(progress.read_rows * 1000000000.0 / elapsed_ns) << " rows/s., " - << formatReadableSizeWithDecimalSuffix(read_bytes * 1000000000.0 / elapsed_ns) << "/s.) "; + << formatReadableSizeWithDecimalSuffix(progress.read_bytes * 1000000000.0 / elapsed_ns) << "/s.) "; else message << ". "; @@ -206,7 +205,7 @@ void ProgressIndication::writeProgress() int64_t remaining_space = static_cast(terminal_width) - written_progress_chars; /// If the approximate number of rows to process is known, we can display a progress bar and percentage. - if (progress.total_rows_to_read || progress.total_raw_bytes_to_read) + if (progress.total_rows_to_read || progress.total_bytes_to_read) { size_t current_count, max_count; if (progress.total_rows_to_read) @@ -216,8 +215,8 @@ void ProgressIndication::writeProgress() } else { - current_count = progress.read_raw_bytes; - max_count = std::max(progress.read_raw_bytes, progress.total_raw_bytes_to_read); + current_count = progress.read_bytes; + max_count = std::max(progress.read_bytes, progress.total_bytes_to_read); } /// To avoid flicker, display progress bar only if .5 seconds have passed since query execution start diff --git a/src/IO/Progress.cpp b/src/IO/Progress.cpp index 1d16f54de7b..29cf6c52d13 100644 --- a/src/IO/Progress.cpp +++ b/src/IO/Progress.cpp @@ -68,10 +68,12 @@ bool Progress::incrementPiecewiseAtomically(const Progress & rhs) { read_rows += rhs.read_rows; read_bytes += rhs.read_bytes; - read_raw_bytes += rhs.read_raw_bytes; - total_rows_to_read += rhs.total_rows_to_read; - total_raw_bytes_to_read += rhs.total_raw_bytes_to_read; + if (rhs.total_rows_to_read) + total_rows_to_read.store(rhs.total_rows_to_read); + + if (rhs.total_bytes_to_read) + total_bytes_to_read.store(rhs.total_bytes_to_read); written_rows += rhs.written_rows; written_bytes += rhs.written_bytes; @@ -83,10 +85,9 @@ void Progress::reset() { read_rows = 0; read_bytes = 0; - read_raw_bytes = 0; total_rows_to_read = 0; - total_raw_bytes_to_read = 0; + total_bytes_to_read = 0; written_rows = 0; written_bytes = 0; @@ -98,10 +99,9 @@ ProgressValues Progress::getValues() const res.read_rows = read_rows.load(std::memory_order_relaxed); res.read_bytes = read_bytes.load(std::memory_order_relaxed); - res.read_raw_bytes = read_raw_bytes.load(std::memory_order_relaxed); res.total_rows_to_read = total_rows_to_read.load(std::memory_order_relaxed); - res.total_raw_bytes_to_read = total_raw_bytes_to_read.load(std::memory_order_relaxed); + res.total_bytes_to_read = total_bytes_to_read.load(std::memory_order_relaxed); res.written_rows = written_rows.load(std::memory_order_relaxed); res.written_bytes = written_bytes.load(std::memory_order_relaxed); @@ -109,16 +109,31 @@ ProgressValues Progress::getValues() const return res; } -ProgressValues Progress::fetchAndResetPiecewiseAtomically() +ProgressValues Progress::fetchValuesAndResetPiecewiseAtomically() { ProgressValues res; res.read_rows = read_rows.fetch_and(0); res.read_bytes = read_bytes.fetch_and(0); - res.read_raw_bytes = read_raw_bytes.fetch_and(0); res.total_rows_to_read = total_rows_to_read.fetch_and(0); - res.total_raw_bytes_to_read = total_raw_bytes_to_read.fetch_and(0); + res.total_bytes_to_read = total_bytes_to_read.fetch_and(0); + + res.written_rows = written_rows.fetch_and(0); + res.written_bytes = written_bytes.fetch_and(0); + + return res; +} + +Progress Progress::fetchAndResetPiecewiseAtomically() +{ + Progress res; + + res.read_rows = read_rows.fetch_and(0); + res.read_bytes = read_bytes.fetch_and(0); + + res.total_rows_to_read = total_rows_to_read.fetch_and(0); + res.total_bytes_to_read = total_bytes_to_read.fetch_and(0); res.written_rows = written_rows.fetch_and(0); res.written_bytes = written_bytes.fetch_and(0); @@ -130,10 +145,9 @@ Progress & Progress::operator=(Progress && other) noexcept { read_rows = other.read_rows.load(std::memory_order_relaxed); read_bytes = other.read_bytes.load(std::memory_order_relaxed); - read_raw_bytes = other.read_raw_bytes.load(std::memory_order_relaxed); total_rows_to_read = other.total_rows_to_read.load(std::memory_order_relaxed); - total_raw_bytes_to_read = other.total_raw_bytes_to_read.load(std::memory_order_relaxed); + total_bytes_to_read = other.total_bytes_to_read.load(std::memory_order_relaxed); written_rows = other.written_rows.load(std::memory_order_relaxed); written_bytes = other.written_bytes.load(std::memory_order_relaxed); @@ -148,7 +162,10 @@ void Progress::read(ReadBuffer & in, UInt64 server_revision) read_rows.store(values.read_rows, std::memory_order_relaxed); read_bytes.store(values.read_bytes, std::memory_order_relaxed); + total_rows_to_read.store(values.total_rows_to_read, std::memory_order_relaxed); + total_bytes_to_read.store(values.total_bytes_to_read, std::memory_order_relaxed); + written_rows.store(values.written_rows, std::memory_order_relaxed); written_bytes.store(values.written_bytes, std::memory_order_relaxed); } diff --git a/src/IO/Progress.h b/src/IO/Progress.h index 4f1a3df0ffd..f04822f26bb 100644 --- a/src/IO/Progress.h +++ b/src/IO/Progress.h @@ -18,10 +18,9 @@ struct ProgressValues { size_t read_rows; size_t read_bytes; - size_t read_raw_bytes; size_t total_rows_to_read; - size_t total_raw_bytes_to_read; + size_t total_bytes_to_read; size_t written_rows; size_t written_bytes; @@ -68,15 +67,12 @@ struct Progress { std::atomic read_rows {0}; /// Rows (source) processed. std::atomic read_bytes {0}; /// Bytes (uncompressed, source) processed. - std::atomic read_raw_bytes {0}; /// Raw bytes processed. /** How much rows/bytes must be processed, in total, approximately. Non-zero value is sent when there is information about * some new part of job. Received values must be summed to get estimate of total rows to process. - * `total_raw_bytes_to_process` is used for file table engine or when reading from file descriptor. - * Used for rendering progress bar on client. */ std::atomic total_rows_to_read {0}; - std::atomic total_raw_bytes_to_read {0}; + std::atomic total_bytes_to_read {0}; std::atomic written_rows {0}; std::atomic written_bytes {0}; @@ -93,7 +89,7 @@ struct Progress : written_rows(write_progress.written_rows), written_bytes(write_progress.written_bytes) {} explicit Progress(FileProgress file_progress) - : read_raw_bytes(file_progress.read_bytes), total_raw_bytes_to_read(file_progress.total_bytes_to_read) {} + : read_bytes(file_progress.read_bytes), total_bytes_to_read(file_progress.total_bytes_to_read) {} void read(ReadBuffer & in, UInt64 server_revision); @@ -109,7 +105,9 @@ struct Progress ProgressValues getValues() const; - ProgressValues fetchAndResetPiecewiseAtomically(); + ProgressValues fetchValuesAndResetPiecewiseAtomically(); + + Progress fetchAndResetPiecewiseAtomically(); Progress & operator=(Progress && other) noexcept; diff --git a/src/Processors/Sources/SourceWithProgress.h b/src/Processors/Sources/SourceWithProgress.h index 912a548f977..57002006957 100644 --- a/src/Processors/Sources/SourceWithProgress.h +++ b/src/Processors/Sources/SourceWithProgress.h @@ -42,6 +42,7 @@ public: /// Set the approximate total number of rows to read. virtual void addTotalRowsApprox(size_t value) = 0; + virtual void setTotalRowsApprox(size_t value) = 0; }; /// Implementation for ISourceWithProgress @@ -58,6 +59,7 @@ public: void setProcessListElement(QueryStatus * elem) final; void setProgressCallback(const ProgressCallback & callback) final { progress_callback = callback; } void addTotalRowsApprox(size_t value) final { total_rows_approx += value; } + void setTotalRowsApprox(size_t value) final { total_rows_approx = value; } protected: /// Call this method to provide information about progress. diff --git a/src/QueryPipeline/RemoteInserter.cpp b/src/QueryPipeline/RemoteInserter.cpp index aec7562e133..d5cef72b020 100644 --- a/src/QueryPipeline/RemoteInserter.cpp +++ b/src/QueryPipeline/RemoteInserter.cpp @@ -50,7 +50,7 @@ RemoteInserter::RemoteInserter( /** Send query and receive "header", that describes table structure. * Header is needed to know, what structure is required for blocks to be passed to 'write' method. */ - connection.sendQuery(timeouts, query, "", QueryProcessingStage::Complete, &settings_, &modified_client_info, false); + connection.sendQuery(timeouts, query, "", QueryProcessingStage::Complete, &settings_, &modified_client_info, false, {}); while (true) { diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index 99ee1b7f7f3..e60b85ee7a3 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -1482,7 +1482,7 @@ namespace void Call::addProgressToResult() { - auto values = progress.fetchAndResetPiecewiseAtomically(); + auto values = progress.fetchValuesAndResetPiecewiseAtomically(); if (!values.read_rows && !values.read_bytes && !values.total_rows_to_read && !values.written_rows && !values.written_bytes) return; auto & grpc_progress = *result.mutable_progress(); diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index a038dcb3e6c..c71b3834726 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1329,6 +1329,7 @@ void TCPHandler::receiveQuery() query_context->getIgnoredPartUUIDs()->add(*state.part_uuids_to_ignore); query_context->setProgressCallback([this] (const Progress & value) { return this->updateProgress(value); }); + query_context->setFileProgressCallback([this](const FileProgress & value) { this->updateProgress(Progress(value)); }); /// /// Settings @@ -1736,7 +1737,7 @@ void TCPHandler::updateProgress(const Progress & value) void TCPHandler::sendProgress() { writeVarUInt(Protocol::Server::Progress, *out); - auto increment = state.progress.fetchAndResetPiecewiseAtomically(); + auto increment = state.progress.fetchValuesAndResetPiecewiseAtomically(); increment.write(*out, client_tcp_protocol_version); out->next(); } diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index c460b8a4c67..e177a8ffd76 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -433,6 +433,8 @@ public: bool need_path_column = false; bool need_file_column = false; + + size_t total_bytes_to_read = 0; }; using FilesInfoPtr = std::shared_ptr; @@ -573,6 +575,15 @@ public: chunk.addColumn(column->convertToFullColumnIfConst()); } + if (num_rows) + { + auto bytes_per_row = std::ceil(static_cast(chunk.bytes()) / num_rows); + size_t total_rows_approx = std::ceil(static_cast(files_info->total_bytes_to_read) / bytes_per_row); + total_rows_approx_accumulated += total_rows_approx; + ++total_rows_count_times; + total_rows_approx = total_rows_approx_accumulated / total_rows_count_times; + setTotalRowsApprox(total_rows_approx); + } return chunk; } @@ -608,6 +619,9 @@ private: bool finished_generate = false; std::shared_lock shared_lock; + + UInt64 total_rows_approx_accumulated = 0; + size_t total_rows_count_times = 0; }; @@ -635,6 +649,7 @@ Pipe StorageFile::read( auto files_info = std::make_shared(); files_info->files = paths; + files_info->total_bytes_to_read = total_bytes_to_read; for (const auto & column : column_names) { @@ -654,9 +669,8 @@ Pipe StorageFile::read( /// Set total number of bytes to process. For progress bar. auto progress_callback = context->getFileProgressCallback(); - if ((context->getApplicationType() == Context::ApplicationType::LOCAL - || context->getApplicationType() == Context::ApplicationType::CLIENT) - && progress_callback) + + if (progress_callback) progress_callback(FileProgress(0, total_bytes_to_read)); for (size_t i = 0; i < num_streams; ++i) From f4b45a580564635885ab4e1c3d9735c9e1d51d76 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 10 May 2022 16:00:53 +0200 Subject: [PATCH 080/171] Better profile events --- src/Common/ProfileEvents.cpp | 15 ++++-- ...chronousReadIndirectBufferFromRemoteFS.cpp | 9 ++-- src/Disks/IO/CachedReadBufferFromRemoteFS.cpp | 47 ++++++++++++------- src/Disks/IO/ThreadPoolRemoteFSReader.cpp | 8 ++-- src/IO/WriteBufferFromS3.cpp | 4 +- src/Interpreters/FilesystemCacheLog.cpp | 4 +- 6 files changed, 55 insertions(+), 32 deletions(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 32a186903ff..e7f40bfe9f4 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -260,10 +260,12 @@ \ M(QueryMemoryLimitExceeded, "Number of times when memory limit exceeded for query.") \ \ - M(RemoteFSReadMicroseconds, "Time of reading from remote filesystem.") \ - M(RemoteFSReadBytes, "Read bytes from remote filesystem.") \ - M(RemoteFSCacheReadBytes, "Read bytes from cache of remote filesystem.") \ - M(RemoteFSCacheDownloadBytes, "Bytes downloaded to cache from remote filesystem.") \ + M(CachedReadBufferReadFromSourceMicroseconds, "") \ + M(CachedReadBufferReadFromCacheMicroseconds, "") \ + M(CachedReadBufferReadFromSourceBytes, "") \ + M(CachedReadBufferReadFromCacheBytes, "") \ + M(CachedReadBufferCacheWriteBytes, "") \ + M(CachedReadBufferCacheWriteMicroseconds, "") \ \ M(RemoteFSSeeks, "Total number of seeks for async buffer") \ M(RemoteFSPrefetches, "Number of prefetches made with asynchronous reading from remote filesystem") \ @@ -275,11 +277,14 @@ M(RemoteFSSeeksWithReset, "Number of seeks which lead to a new connection") \ M(RemoteFSBuffers, "Number of buffers created for asynchronous reading from remote filesystem") \ \ + M(ThreadpoolReaderTaskMicroseconds, "") \ + M(ThreadpoolReaderReadBytes, "") \ + \ M(FileSegmentWaitReadBufferMicroseconds, "") \ M(FileSegmentReadMicroseconds, "") \ M(FileSegmentCacheWriteMicroseconds, "") \ M(FileSegmentPredownloadMicroseconds, "") \ - M(FileSegmentResultBytes, "") \ + M(FileSegmentUsedBytes, "") \ \ M(ReadBufferSeekCancelConnection, "Number of seeks which lead to new connection (s3, http)") \ \ diff --git a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp index d1e4b9fa21e..2f0f9d529cc 100644 --- a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp @@ -169,6 +169,9 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::nextImpl() if (!hasPendingDataToRead()) return false; + Stopwatch watch; + CurrentMetrics::Increment metric_increment{CurrentMetrics::AsynchronousReadWait}; + size_t size = 0; if (prefetch_future.valid()) { @@ -176,8 +179,6 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::nextImpl() size_t offset = 0; { - Stopwatch watch; - CurrentMetrics::Increment metric_increment{CurrentMetrics::AsynchronousReadWait}; auto result = prefetch_future.get(); size = result.size; offset = result.offset; @@ -185,7 +186,6 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::nextImpl() /// If prefetch_future is valid, size should always be greater than zero. assert(offset < size); - ProfileEvents::increment(ProfileEvents::AsynchronousReadWaitMicroseconds, watch.elapsedMicroseconds()); } prefetch_buffer.swap(memory); @@ -210,6 +210,9 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::nextImpl() } } + watch.stop(); + ProfileEvents::increment(ProfileEvents::AsynchronousReadWaitMicroseconds, watch.elapsedMicroseconds()); + file_offset_of_buffer_end = impl->getFileOffsetOfBufferEnd(); assert(file_offset_of_buffer_end == impl->getImplementationBufferOffset()); diff --git a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp index e4bee1761f6..49b961ee26a 100644 --- a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp +++ b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp @@ -10,14 +10,18 @@ namespace ProfileEvents { -extern const Event RemoteFSReadBytes; -extern const Event RemoteFSCacheReadBytes; -extern const Event RemoteFSCacheDownloadBytes; extern const Event FileSegmentWaitReadBufferMicroseconds; extern const Event FileSegmentReadMicroseconds; extern const Event FileSegmentCacheWriteMicroseconds; extern const Event FileSegmentPredownloadMicroseconds; -extern const Event FileSegmentResultBytes; +extern const Event FileSegmentUsedBytes; + +extern const Event CachedReadBufferReadFromSourceMicroseconds; +extern const Event CachedReadBufferReadFromCacheMicroseconds; +extern const Event CachedReadBufferCacheWriteMicroseconds; +extern const Event CachedReadBufferReadFromSourceBytes; +extern const Event CachedReadBufferReadFromCacheBytes; +extern const Event CachedReadBufferCacheWriteBytes; } namespace DB @@ -532,7 +536,9 @@ void CachedReadBufferFromRemoteFS::predownload(FileSegmentPtr & file_segment) has_more_data = !implementation_buffer->eof(); watch.stop(); - current_file_segment_counters.increment(ProfileEvents::FileSegmentReadMicroseconds, watch.elapsedMicroseconds()); + auto elapsed = watch.elapsedMicroseconds(); + current_file_segment_counters.increment(ProfileEvents::FileSegmentReadMicroseconds, elapsed); + ProfileEvents::increment(ProfileEvents::CachedReadBufferReadFromSourceMicroseconds, elapsed); } if (!bytes_to_predownload || !has_more_data) @@ -574,7 +580,7 @@ void CachedReadBufferFromRemoteFS::predownload(FileSegmentPtr & file_segment) size_t current_impl_buffer_size = implementation_buffer->buffer().size(); size_t current_predownload_size = std::min(current_impl_buffer_size, bytes_to_predownload); - ProfileEvents::increment(ProfileEvents::RemoteFSReadBytes, current_impl_buffer_size); + ProfileEvents::increment(ProfileEvents::CachedReadBufferReadFromSourceBytes, current_impl_buffer_size); if (file_segment->reserve(current_predownload_size)) { @@ -587,9 +593,10 @@ void CachedReadBufferFromRemoteFS::predownload(FileSegmentPtr & file_segment) file_segment->write(implementation_buffer->buffer().begin(), current_predownload_size, current_offset); watch.stop(); - current_file_segment_counters.increment(ProfileEvents::FileSegmentCacheWriteMicroseconds, watch.elapsedMicroseconds()); - - ProfileEvents::increment(ProfileEvents::RemoteFSCacheDownloadBytes, current_predownload_size); + auto elapsed = watch.elapsedMicroseconds(); + current_file_segment_counters.increment(ProfileEvents::FileSegmentCacheWriteMicroseconds, elapsed); + ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteMicroseconds, elapsed); + ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteBytes, current_predownload_size); current_offset += current_predownload_size; @@ -826,14 +833,21 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() result = implementation_buffer->next(); watch.stop(); - current_file_segment_counters.increment(ProfileEvents::FileSegmentReadMicroseconds, watch.elapsedMicroseconds()); + auto elapsed = watch.elapsedMicroseconds(); + current_file_segment_counters.increment(ProfileEvents::FileSegmentReadMicroseconds, elapsed); size = implementation_buffer->buffer().size(); if (read_type == ReadType::CACHED) - ProfileEvents::increment(ProfileEvents::RemoteFSCacheReadBytes, size); + { + ProfileEvents::increment(ProfileEvents::CachedReadBufferReadFromCacheBytes, size); + ProfileEvents::increment(ProfileEvents::CachedReadBufferReadFromCacheMicroseconds, elapsed); + } else - ProfileEvents::increment(ProfileEvents::RemoteFSReadBytes, size); + { + ProfileEvents::increment(ProfileEvents::CachedReadBufferReadFromSourceBytes, size); + ProfileEvents::increment(ProfileEvents::CachedReadBufferReadFromSourceMicroseconds, elapsed); + } } if (result) @@ -854,9 +868,10 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() file_offset_of_buffer_end); watch.stop(); - current_file_segment_counters.increment(ProfileEvents::FileSegmentCacheWriteMicroseconds, watch.elapsedMicroseconds()); - - ProfileEvents::increment(ProfileEvents::RemoteFSCacheDownloadBytes, size); + auto elapsed = watch.elapsedMicroseconds(); + current_file_segment_counters.increment(ProfileEvents::FileSegmentCacheWriteMicroseconds, elapsed); + ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteMicroseconds, elapsed); + ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteBytes, size); assert(file_segment->getDownloadOffset() <= file_segment->range().right + 1); assert( @@ -889,7 +904,7 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() swap(*implementation_buffer); - current_file_segment_counters.increment(ProfileEvents::FileSegmentResultBytes, available()); + current_file_segment_counters.increment(ProfileEvents::FileSegmentUsedBytes, available()); if (download_current_segment) file_segment->completeBatchAndResetDownloader(); diff --git a/src/Disks/IO/ThreadPoolRemoteFSReader.cpp b/src/Disks/IO/ThreadPoolRemoteFSReader.cpp index b1ae42d03d6..f8050b8a8b0 100644 --- a/src/Disks/IO/ThreadPoolRemoteFSReader.cpp +++ b/src/Disks/IO/ThreadPoolRemoteFSReader.cpp @@ -16,8 +16,8 @@ namespace ProfileEvents { - extern const Event RemoteFSReadMicroseconds; - extern const Event RemoteFSReadBytes; + extern const Event ThreadpoolReaderTaskMicroseconds; + extern const Event ThreadpoolReaderReadBytes; } namespace CurrentMetrics @@ -83,8 +83,8 @@ std::future ThreadPoolRemoteFSReader::submit(Reques watch.stop(); - ProfileEvents::increment(ProfileEvents::RemoteFSReadMicroseconds, watch.elapsedMicroseconds()); - ProfileEvents::increment(ProfileEvents::RemoteFSReadBytes, result.offset ? result.size - result.offset : result.size); + ProfileEvents::increment(ProfileEvents::ThreadpoolReaderTaskMicroseconds, watch.elapsedMicroseconds()); + ProfileEvents::increment(ProfileEvents::ThreadpoolReaderReadBytes, result.offset ? result.size - result.offset : result.size); thread_status.detachQuery(/* if_not_detached */true); diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 9471b5d319b..5dcb7673829 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -21,7 +21,7 @@ namespace ProfileEvents { extern const Event WriteBufferFromS3Bytes; - extern const Event RemoteFSCacheDownloadBytes; + extern const Event CachedReadBufferCacheWriteBytes; } namespace DB @@ -480,7 +480,7 @@ void WriteBufferFromS3::finalizeCacheIfNeeded(std::optional size_t size = (*file_segment_it)->finalizeWrite(); file_segment_it = file_segments.erase(file_segment_it); - ProfileEvents::increment(ProfileEvents::RemoteFSCacheDownloadBytes, size); + ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteBytes, size); } catch (...) { diff --git a/src/Interpreters/FilesystemCacheLog.cpp b/src/Interpreters/FilesystemCacheLog.cpp index cae80453b35..be2aae9c550 100644 --- a/src/Interpreters/FilesystemCacheLog.cpp +++ b/src/Interpreters/FilesystemCacheLog.cpp @@ -36,8 +36,8 @@ NamesAndTypesList FilesystemCacheLogElement::getNamesAndTypes() {"event_time", std::make_shared()}, {"query_id", std::make_shared()}, {"source_file_path", std::make_shared()}, - {"file_segment_range", std::make_shared(std::move(types))}, - {"total_requested_range", std::make_shared(std::move(types))}, + {"file_segment_range", std::make_shared(types)}, + {"total_requested_range", std::make_shared(types)}, {"size", std::make_shared()}, {"read_type", std::make_shared()}, {"ProfileEvents", std::make_shared(std::make_shared(), std::make_shared())}, From 1feba64a4ec335bbe59218915bad987e31653575 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 10 May 2022 11:33:48 +0000 Subject: [PATCH 081/171] Continue fixing #36199. --- src/Interpreters/join_common.cpp | 32 +++++++++++++++++++ ...2267_join_dup_columns_issue36199.reference | 2 ++ .../02267_join_dup_columns_issue36199.sql | 4 +++ 3 files changed, 38 insertions(+) diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index 522244d76f0..fbefcd183e3 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -733,6 +733,38 @@ Block NotJoinedBlocks::read() copySameKeys(result_block); correctLowcardAndNullability(result_block); + /// This is a very dirty hack. + /// Sometimes, columns from right joined table may be duplicating, and we can't properly process it. + /// So, in case if column is empty in result, try to find non-empty column with the same name. + /// Repro: + /// select * from (select 2 as x) as t1 right join (select count('x'), count('y'), 0 as x) as t2 on t1.x = t2.x + /// + /// Please, somebody, fix it in a proper way. + { + std::unordered_map> result_block_positions; + for (size_t i = 0; i < result_block.columns(); ++i) + { + const auto & col = result_block.getByPosition(i); + result_block_positions[col.name].push_back(i); + } + + for (auto & col : result_block) + { + if (col.column->empty()) + { + for (auto pos : result_block_positions[col.name]) + { + const auto & same_col = result_block.getByPosition(pos); + if (!same_col.column->empty()) + { + col.column = same_col.column; + break; + } + } + } + } + } + #ifndef NDEBUG assertBlocksHaveEqualStructure(result_block, result_sample_block, "NotJoinedBlocks"); #endif diff --git a/tests/queries/0_stateless/02267_join_dup_columns_issue36199.reference b/tests/queries/0_stateless/02267_join_dup_columns_issue36199.reference index 47e37dc8fc8..7c9573c94f1 100644 --- a/tests/queries/0_stateless/02267_join_dup_columns_issue36199.reference +++ b/tests/queries/0_stateless/02267_join_dup_columns_issue36199.reference @@ -1,2 +1,4 @@ 2 1 2 +0 1 0 2 1 2 +0 1 0 diff --git a/tests/queries/0_stateless/02267_join_dup_columns_issue36199.sql b/tests/queries/0_stateless/02267_join_dup_columns_issue36199.sql index 118d475207a..5cab2b0d9e3 100644 --- a/tests/queries/0_stateless/02267_join_dup_columns_issue36199.sql +++ b/tests/queries/0_stateless/02267_join_dup_columns_issue36199.sql @@ -5,9 +5,13 @@ FROM ( SELECT 2 AS x ) AS t1 RIGHT JOIN ( SELECT count('x'), count('y'), 2 AS x ) AS t2 ON t1.x = t2.x; +select * from (select 2 as x) as t1 right join (select count('x'), count('y'), 0 as x) as t2 on t1.x = t2.x; + set join_algorithm = 'partial_merge'; SELECT * FROM ( SELECT 2 AS x ) AS t1 RIGHT JOIN ( SELECT count('x'), count('y'), 2 AS x ) AS t2 ON t1.x = t2.x; + +select * from (select 2 as x) as t1 right join (select count('x'), count('y'), 0 as x) as t2 on t1.x = t2.x; From 837d4bd13f9015328064f093d28ed6e76f17a0d9 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 10 May 2022 14:30:02 +0000 Subject: [PATCH 082/171] Revert "Continue fixing #36199." This reverts commit 1feba64a4ec335bbe59218915bad987e31653575. --- src/Interpreters/join_common.cpp | 32 ------------------- ...2267_join_dup_columns_issue36199.reference | 2 -- .../02267_join_dup_columns_issue36199.sql | 4 --- 3 files changed, 38 deletions(-) diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index fbefcd183e3..522244d76f0 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -733,38 +733,6 @@ Block NotJoinedBlocks::read() copySameKeys(result_block); correctLowcardAndNullability(result_block); - /// This is a very dirty hack. - /// Sometimes, columns from right joined table may be duplicating, and we can't properly process it. - /// So, in case if column is empty in result, try to find non-empty column with the same name. - /// Repro: - /// select * from (select 2 as x) as t1 right join (select count('x'), count('y'), 0 as x) as t2 on t1.x = t2.x - /// - /// Please, somebody, fix it in a proper way. - { - std::unordered_map> result_block_positions; - for (size_t i = 0; i < result_block.columns(); ++i) - { - const auto & col = result_block.getByPosition(i); - result_block_positions[col.name].push_back(i); - } - - for (auto & col : result_block) - { - if (col.column->empty()) - { - for (auto pos : result_block_positions[col.name]) - { - const auto & same_col = result_block.getByPosition(pos); - if (!same_col.column->empty()) - { - col.column = same_col.column; - break; - } - } - } - } - } - #ifndef NDEBUG assertBlocksHaveEqualStructure(result_block, result_sample_block, "NotJoinedBlocks"); #endif diff --git a/tests/queries/0_stateless/02267_join_dup_columns_issue36199.reference b/tests/queries/0_stateless/02267_join_dup_columns_issue36199.reference index 7c9573c94f1..47e37dc8fc8 100644 --- a/tests/queries/0_stateless/02267_join_dup_columns_issue36199.reference +++ b/tests/queries/0_stateless/02267_join_dup_columns_issue36199.reference @@ -1,4 +1,2 @@ 2 1 2 -0 1 0 2 1 2 -0 1 0 diff --git a/tests/queries/0_stateless/02267_join_dup_columns_issue36199.sql b/tests/queries/0_stateless/02267_join_dup_columns_issue36199.sql index 5cab2b0d9e3..118d475207a 100644 --- a/tests/queries/0_stateless/02267_join_dup_columns_issue36199.sql +++ b/tests/queries/0_stateless/02267_join_dup_columns_issue36199.sql @@ -5,13 +5,9 @@ FROM ( SELECT 2 AS x ) AS t1 RIGHT JOIN ( SELECT count('x'), count('y'), 2 AS x ) AS t2 ON t1.x = t2.x; -select * from (select 2 as x) as t1 right join (select count('x'), count('y'), 0 as x) as t2 on t1.x = t2.x; - set join_algorithm = 'partial_merge'; SELECT * FROM ( SELECT 2 AS x ) AS t1 RIGHT JOIN ( SELECT count('x'), count('y'), 2 AS x ) AS t2 ON t1.x = t2.x; - -select * from (select 2 as x) as t1 right join (select count('x'), count('y'), 0 as x) as t2 on t1.x = t2.x; From ff001382fb24103921b3013f104b234cc6dea573 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 10 May 2022 14:22:22 +0000 Subject: [PATCH 083/171] Handle non unique column names in NotJoinedBlocks --- src/Interpreters/join_common.cpp | 42 ++++++++++++-------------------- 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index 522244d76f0..37f46d6a65e 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -597,41 +597,29 @@ NotJoinedBlocks::NotJoinedBlocks(std::unique_ptr filler_, column_indices_left.emplace_back(left_pos); } - /// `sample_block_names` may contain non unique column names + /// `saved_block_sample` may contains non unique column names, get any of them /// (e.g. in case of `... JOIN (SELECT a, a, b FROM table) as t2`) - /// proper fix is to get rid of it - std::unordered_set sample_block_names; - for (size_t right_pos = 0; right_pos < saved_block_sample.columns(); ++right_pos) + for (const auto & [name, right_pos] : saved_block_sample.getNamesToIndexesMap()) { - const String & name = saved_block_sample.getByPosition(right_pos).name; - - auto [_, inserted] = sample_block_names.insert(name); - /// skip columns with same names - if (!inserted) - continue; - - if (!result_sample_block.has(name)) - continue; - - size_t result_position = result_sample_block.getPositionByName(name); - - /// Don't remap left keys twice. We need only qualified right keys here - if (result_position < left_columns_count) - continue; - - setRightIndex(right_pos, result_position); + /// Start from left_columns_count to don't remap left keys twice. We need only qualified right keys here + /// `result_sample_block` may contains non unique column names, need to set index for all of them + for (size_t result_pos = left_columns_count; result_pos < result_sample_block.columns(); ++result_pos) + { + const auto & result_name = result_sample_block.getByPosition(result_pos).name; + if (result_name == name) + setRightIndex(right_pos, result_pos); + } } - /// `result_sample_block` also may contains non unique column names - const auto & result_names = result_sample_block.getNames(); - size_t unique_names_count = std::unordered_set(result_names.begin(), result_names.end()).size(); - if (column_indices_left.size() + column_indices_right.size() + same_result_keys.size() != unique_names_count) + if (column_indices_left.size() + column_indices_right.size() + same_result_keys.size() != result_sample_block.columns()) + { throw Exception( ErrorCodes::LOGICAL_ERROR, - "Error in columns mapping in JOIN. (assertion failed {} + {} + {} != {}) " + "Error in columns mapping in JOIN: assertion failed {} + {} + {} != {}; " "Result block [{}], Saved block [{}]", - column_indices_left.size(), column_indices_right.size(), same_result_keys.size(), unique_names_count, + column_indices_left.size(), column_indices_right.size(), same_result_keys.size(), result_sample_block.columns(), result_sample_block.dumpNames(), saved_block_sample.dumpNames()); + } } void NotJoinedBlocks::setRightIndex(size_t right_pos, size_t result_position) From 9b24e0d260b3f0b96190225cbe33e62a7786153c Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 10 May 2022 14:24:20 +0000 Subject: [PATCH 084/171] Apply optimizeCountConstantAndSumOne before JOINs --- src/Interpreters/TreeOptimizer.cpp | 11 +++++------ src/Interpreters/TreeOptimizer.h | 1 + src/Interpreters/TreeRewriter.cpp | 4 ++++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index 19041a19aa4..391b970f2b8 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -646,12 +646,6 @@ void optimizeSumIfFunctions(ASTPtr & query) RewriteSumIfFunctionVisitor(data).visit(query); } -void optimizeCountConstantAndSumOne(ASTPtr & query) -{ - RewriteCountVariantsVisitor::visit(query); -} - - void optimizeInjectiveFunctionsInsideUniq(ASTPtr & query, ContextPtr context) { RemoveInjectiveFunctionsVisitor::Data data(context); @@ -740,6 +734,11 @@ void TreeOptimizer::optimizeIf(ASTPtr & query, Aliases & aliases, bool if_chain_ OptimizeIfChainsVisitor().visit(query); } +void TreeOptimizer::optimizeCountConstantAndSumOne(ASTPtr & query) +{ + RewriteCountVariantsVisitor::visit(query); +} + void TreeOptimizer::apply(ASTPtr & query, TreeRewriterResult & result, const std::vector & tables_with_columns, ContextPtr context) { diff --git a/src/Interpreters/TreeOptimizer.h b/src/Interpreters/TreeOptimizer.h index 1d9a29bd0ac..ced185373cc 100644 --- a/src/Interpreters/TreeOptimizer.h +++ b/src/Interpreters/TreeOptimizer.h @@ -24,6 +24,7 @@ public: ContextPtr context); static void optimizeIf(ASTPtr & query, Aliases & aliases, bool if_chain_to_multiif); + static void optimizeCountConstantAndSumOne(ASTPtr & query); }; } diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 9a00c0e7135..c90421d6f4f 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -1151,6 +1151,10 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( if (remove_duplicates) renameDuplicatedColumns(select_query); + /// Perform it before analyzing JOINs, because it may change number of columns with names unique and break some login inside JOINs + if (settings.optimize_normalize_count_variants) + TreeOptimizer::optimizeCountConstantAndSumOne(query); + if (tables_with_columns.size() > 1) { const auto & right_table = tables_with_columns[1]; From cadd614ce89d6777198d57dd7c762213bfe62f0f Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 10 May 2022 14:24:36 +0000 Subject: [PATCH 085/171] Add cases to join_dup_columns_issue36199 --- ...2267_join_dup_columns_issue36199.reference | 12 ++++++++ .../02267_join_dup_columns_issue36199.sql | 29 ++++++++++++------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/tests/queries/0_stateless/02267_join_dup_columns_issue36199.reference b/tests/queries/0_stateless/02267_join_dup_columns_issue36199.reference index 47e37dc8fc8..c075b08e533 100644 --- a/tests/queries/0_stateless/02267_join_dup_columns_issue36199.reference +++ b/tests/queries/0_stateless/02267_join_dup_columns_issue36199.reference @@ -1,2 +1,14 @@ 2 1 2 +0 1 0 +0 1 1 0 +0 1 0 +0 0 1 0 +y 0 0 1 3 +\N 0 \N 10000000000 \N 2 1 2 +0 1 0 +0 1 1 0 +0 1 0 +0 0 1 0 +y 0 0 1 3 +\N 0 \N 10000000000 \N diff --git a/tests/queries/0_stateless/02267_join_dup_columns_issue36199.sql b/tests/queries/0_stateless/02267_join_dup_columns_issue36199.sql index 118d475207a..b51b3cc22e2 100644 --- a/tests/queries/0_stateless/02267_join_dup_columns_issue36199.sql +++ b/tests/queries/0_stateless/02267_join_dup_columns_issue36199.sql @@ -1,13 +1,22 @@ -set join_algorithm = 'hash'; +SET join_algorithm = 'hash'; -SELECT * -FROM ( SELECT 2 AS x ) AS t1 -RIGHT JOIN ( SELECT count('x'), count('y'), 2 AS x ) AS t2 -ON t1.x = t2.x; +SELECT * FROM ( SELECT 2 AS x ) AS t1 RIGHT JOIN ( SELECT count('x'), count('y'), 2 AS x ) AS t2 ON t1.x = t2.x; +SELECT * FROM ( SELECT 2 AS x ) as t1 RIGHT JOIN ( SELECT count('x'), count('y'), 0 AS x ) AS t2 ON t1.x = t2.x; +SELECT * FROM ( SELECT 2 AS x ) as t1 RIGHT JOIN ( SELECT count('x') :: Nullable(Int32), count('y'), 0 AS x ) AS t2 ON t1.x = t2.x; +SELECT * FROM ( SELECT 2 AS x ) as t1 RIGHT JOIN ( SELECT count('x') :: Nullable(Int32), count('y') :: Nullable(Int32), 0 AS x ) AS t2 ON t1.x = t2.x; +SELECT * FROM ( SELECT count('a'), count('b'), count('c'), 2 AS x ) as t1 RIGHT JOIN ( SELECT count('x'), count('y'), 0 AS x ) AS t2 ON t1.x = t2.x; -set join_algorithm = 'partial_merge'; +SELECT 'y', * FROM (SELECT count('y'), count('y'), 2 AS x) AS t1 RIGHT JOIN (SELECT count('x'), count('y'), 3 AS x) AS t2 ON t1.x = t2.x; +SELECT * FROM (SELECT arrayJoin([NULL]), 9223372036854775806, arrayJoin([NULL]), NULL AS x) AS t1 RIGHT JOIN (SELECT arrayJoin([arrayJoin([10000000000.])]), NULL AS x) AS t2 ON t1.x = t2.x; -SELECT * -FROM ( SELECT 2 AS x ) AS t1 -RIGHT JOIN ( SELECT count('x'), count('y'), 2 AS x ) AS t2 -ON t1.x = t2.x; +SET join_algorithm = 'partial_merge'; + +SELECT * FROM ( SELECT 2 AS x ) AS t1 RIGHT JOIN ( SELECT count('x'), count('y'), 2 AS x ) AS t2 ON t1.x = t2.x; +SELECT * FROM ( SELECT 2 AS x ) as t1 RIGHT JOIN ( SELECT count('x'), count('y'), 0 AS x ) AS t2 ON t1.x = t2.x; +SELECT * FROM ( SELECT 2 AS x ) as t1 RIGHT JOIN ( SELECT count('x') :: Nullable(Int32), count('y'), 0 AS x ) AS t2 ON t1.x = t2.x; +SELECT * FROM ( SELECT 2 AS x ) as t1 RIGHT JOIN ( SELECT count('x') :: Nullable(Int32), count('y') :: Nullable(Int32), 0 AS x ) AS t2 ON t1.x = t2.x; + +SELECT * FROM ( SELECT count('a'), count('b'), count('c'), 2 AS x ) as t1 RIGHT JOIN ( SELECT count('x'), count('y'), 0 AS x ) AS t2 ON t1.x = t2.x; + +SELECT 'y', * FROM (SELECT count('y'), count('y'), 2 AS x) AS t1 RIGHT JOIN (SELECT count('x'), count('y'), 3 AS x) AS t2 ON t1.x = t2.x; +SELECT * FROM (SELECT arrayJoin([NULL]), 9223372036854775806, arrayJoin([NULL]), NULL AS x) AS t1 RIGHT JOIN (SELECT arrayJoin([arrayJoin([10000000000.])]), NULL AS x) AS t2 ON t1.x = t2.x; From a02e1d2f4a81abb3dc80c85adc73f4f3c1e28f16 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 10 May 2022 16:00:00 +0000 Subject: [PATCH 086/171] Simplify code a little bit. --- src/Interpreters/ActionsDAG.cpp | 7 ----- src/Interpreters/ActionsDAG.h | 1 - src/Interpreters/Aggregator.cpp | 4 --- src/Interpreters/Aggregator.h | 12 +++------ src/Interpreters/ExpressionAnalyzer.cpp | 14 +++++----- src/Interpreters/InterpreterSelectQuery.cpp | 22 +++++++--------- src/Interpreters/InterpreterSelectQuery.h | 3 +-- src/Processors/QueryPlan/AggregatingStep.cpp | 11 +++++++- .../Transforms/GroupingSetsTransform.cpp | 26 ++++++++++++++----- .../Transforms/GroupingSetsTransform.h | 3 ++- 10 files changed, 52 insertions(+), 51 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 871c519288c..f796a55ff72 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -234,13 +234,6 @@ const ActionsDAG::Node & ActionsDAG::addFunction( return addNode(std::move(node)); } -const ActionsDAG::Node & ActionsDAG::addGroupingSetColumn() -{ - const auto & res = addInput("__grouping_set", std::make_shared()); - index.emplace_back(&res); - return res; -} - const ActionsDAG::Node & ActionsDAG::findInIndex(const std::string & name) const { if (const auto * node = tryFindInIndex(name)) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 9736f626b6b..d11dfc3b8ad 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -132,7 +132,6 @@ public: const FunctionOverloadResolverPtr & function, NodeRawConstPtrs children, std::string result_name); - const Node & addGroupingSetColumn(); /// Index can contain any column returned from DAG. /// You may manually change it if needed. diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index cacd8d841f5..dae00951dc2 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -366,7 +366,6 @@ Block Aggregator::Params::getHeader( const Block & intermediate_header, const ColumnNumbers & keys, const AggregateDescriptions & aggregates, - bool has_grouping_sets, bool final) { Block res; @@ -391,9 +390,6 @@ Block Aggregator::Params::getHeader( for (const auto & key : keys) res.insert(src_header.safeGetByPosition(key).cloneEmpty()); - if (has_grouping_sets) - res.insert({ std::make_shared(), "__grouping_set" }); - for (const auto & aggregate : aggregates) { size_t arguments_size = aggregate.arguments.size(); diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index e6a00b9a191..a0863822bae 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -966,8 +966,6 @@ public: }; StatsCollectingParams stats_collecting_params; - bool has_grouping_sets; - Params( const Block & src_header_, const ColumnNumbers & keys_, @@ -985,8 +983,7 @@ public: bool compile_aggregate_expressions_, size_t min_count_to_compile_aggregate_expression_, const Block & intermediate_header_ = {}, - const StatsCollectingParams & stats_collecting_params_ = {}, - bool has_grouping_sets_ = false) + const StatsCollectingParams & stats_collecting_params_ = {}) : src_header(src_header_) , intermediate_header(intermediate_header_) , keys(keys_) @@ -1006,7 +1003,6 @@ public: , compile_aggregate_expressions(compile_aggregate_expressions_) , min_count_to_compile_aggregate_expression(min_count_to_compile_aggregate_expression_) , stats_collecting_params(stats_collecting_params_) - , has_grouping_sets(has_grouping_sets_) { } @@ -1016,9 +1012,8 @@ public: const ColumnNumbers & keys_, const AggregateDescriptions & aggregates_, bool overflow_row_, - bool has_grouping_sets_, size_t max_threads_) - : Params(Block(), keys_, aggregates_, overflow_row_, 0, OverflowMode::THROW, 0, 0, 0, false, nullptr, max_threads_, 0, false, 0, {}, {}, has_grouping_sets_) + : Params(Block(), keys_, aggregates_, overflow_row_, 0, OverflowMode::THROW, 0, 0, 0, false, nullptr, max_threads_, 0, false, 0, {}, {}) { intermediate_header = intermediate_header_; } @@ -1028,12 +1023,11 @@ public: const Block & intermediate_header, const ColumnNumbers & keys, const AggregateDescriptions & aggregates, - bool has_grouping_sets, bool final); Block getHeader(bool final) const { - return getHeader(src_header, intermediate_header, keys, aggregates, has_grouping_sets, final); + return getHeader(src_header, intermediate_header, keys, aggregates, final); } /// Returns keys and aggregated for EXPLAIN query diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index a6eda5a8514..25967919183 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -327,6 +327,12 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) { NameSet unique_keys; ASTs & group_asts = group_by_ast->children; + + /// For GROUPING SETS with multiple groups we always add virtual __grouping_set column + /// With set number, which is used as an additional key at the stage of merging aggregating data. + if (select_query->group_by_with_grouping_sets && group_asts.size() > 1) + aggregated_columns.emplace_back("__grouping_set", std::make_shared()); + for (ssize_t i = 0; i < static_cast(group_asts.size()); ++i) { ssize_t size = group_asts.size(); @@ -450,9 +456,6 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) else aggregated_columns = temp_actions->getNamesAndTypesList(); - if (aggregation_keys_list.size() > 1) - aggregated_columns.emplace_back("__grouping_set", std::make_shared()); - for (const auto & desc : aggregate_descriptions) aggregated_columns.emplace_back(desc.column_name, desc.function->getReturnType()); } @@ -1243,11 +1246,6 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain getRootActions(ast_element, only_types, step.actions()); } } - if (useGroupingSetKey()) - { - step.addRequiredOutput("__grouping_set"); - step.actions()->addGroupingSetColumn(); - } } else { diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 42a78d99da1..e8c1be5330e 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -738,12 +738,12 @@ Block InterpreterSelectQuery::getSampleBlockImpl() Block res; - for (const auto & key : query_analyzer->aggregationKeys()) - res.insert({nullptr, header.getByName(key.name).type, key.name}); - if (analysis_result.use_grouping_set_key) res.insert({ nullptr, std::make_shared(), "__grouping_set" }); + for (const auto & key : query_analyzer->aggregationKeys()) + res.insert({nullptr, header.getByName(key.name).type, key.name}); + for (const auto & aggregate : query_analyzer->aggregates()) { size_t arguments_size = aggregate.argument_names.size(); @@ -1582,10 +1582,10 @@ static void executeMergeAggregatedImpl( const auto & header_before_merge = query_plan.getCurrentDataStream().header; ColumnNumbers keys; - for (const auto & key : aggregation_keys) - keys.push_back(header_before_merge.getPositionByName(key.name)); if (has_grouping_sets) keys.push_back(header_before_merge.getPositionByName("__grouping_set")); + for (const auto & key : aggregation_keys) + keys.push_back(header_before_merge.getPositionByName(key.name)); /** There are two modes of distributed aggregation. * @@ -1602,7 +1602,7 @@ static void executeMergeAggregatedImpl( * but it can work more slowly. */ - Aggregator::Params params(header_before_merge, keys, aggregates, overflow_row, settings.max_threads, has_grouping_sets); + Aggregator::Params params(header_before_merge, keys, aggregates, overflow_row, settings.max_threads); auto transform_params = std::make_shared(params, final); @@ -2192,8 +2192,7 @@ Aggregator::Params InterpreterSelectQuery::getAggregatorParams( const ColumnNumbers & keys, const AggregateDescriptions & aggregates, bool overflow_row, const Settings & settings, - size_t group_by_two_level_threshold, size_t group_by_two_level_threshold_bytes, - bool use_grouping_set_key) + size_t group_by_two_level_threshold, size_t group_by_two_level_threshold_bytes) { const auto stats_collecting_params = Aggregator::Params::StatsCollectingParams( query_ptr, @@ -2220,8 +2219,7 @@ Aggregator::Params InterpreterSelectQuery::getAggregatorParams( settings.compile_aggregate_expressions, settings.min_count_to_compile_aggregate_expression, Block{}, - stats_collecting_params, - use_grouping_set_key + stats_collecting_params }; } @@ -2284,7 +2282,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac keys.push_back(header_before_aggregation.getPositionByName(key.name)); auto aggregator_params = getAggregatorParams(header_before_aggregation, keys, aggregates, overflow_row, settings, - settings.group_by_two_level_threshold, settings.group_by_two_level_threshold_bytes, query_analyzer->useGroupingSetKey()); + settings.group_by_two_level_threshold, settings.group_by_two_level_threshold_bytes); auto grouping_sets_params = getAggregatorGroupingSetsParams(header_before_aggregation, keys); @@ -2376,7 +2374,7 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPlan & query_plan, Modific for (const auto & key : query_analyzer->aggregationKeys()) keys.push_back(header_before_transform.getPositionByName(key.name)); - auto params = getAggregatorParams(header_before_transform, keys, query_analyzer->aggregates(), false, settings, 0, 0, false); + auto params = getAggregatorParams(header_before_transform, keys, query_analyzer->aggregates(), false, settings, 0, 0); auto transform_params = std::make_shared(std::move(params), true); QueryPlanStepPtr step; diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 532eced9dc1..237b305ae03 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -149,8 +149,7 @@ private: const ColumnNumbers & keys, const AggregateDescriptions & aggregates, bool overflow_row, const Settings & settings, - size_t group_by_two_level_threshold, size_t group_by_two_level_threshold_bytes, - bool use_grouping_set_key); + size_t group_by_two_level_threshold, size_t group_by_two_level_threshold_bytes); GroupingSetsParamsList getAggregatorGroupingSetsParams( const Block & header_before_aggregation, const ColumnNumbers & all_keys); diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index a6bc40120e6..e7c87843f16 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace DB { @@ -32,6 +33,14 @@ static ITransformingStep::Traits getTraits() }; } +static Block appendGroupingColumn(Block block, const GroupingSetsParamsList & params) +{ + if (params.empty()) + return block; + + return GroupingSetsTransform::appendGroupingColumn(std::move(block)); +} + AggregatingStep::AggregatingStep( const DataStream & input_stream_, Aggregator::Params params_, @@ -44,7 +53,7 @@ AggregatingStep::AggregatingStep( bool storage_has_evenly_distributed_read_, InputOrderInfoPtr group_by_info_, SortDescription group_by_sort_description_) - : ITransformingStep(input_stream_, params_.getHeader(final_), getTraits(), false) + : ITransformingStep(input_stream_, appendGroupingColumn(params_.getHeader(final_), grouping_sets_params_), getTraits(), false) , params(std::move(params_)) , grouping_sets_params(std::move(grouping_sets_params_)) , final(std::move(final_)) diff --git a/src/Processors/Transforms/GroupingSetsTransform.cpp b/src/Processors/Transforms/GroupingSetsTransform.cpp index 824dde18827..74494d54c2f 100644 --- a/src/Processors/Transforms/GroupingSetsTransform.cpp +++ b/src/Processors/Transforms/GroupingSetsTransform.cpp @@ -8,6 +8,21 @@ namespace DB { +Block GroupingSetsTransform::appendGroupingColumn(Block block) +{ + Block res; + + size_t rows = block.rows(); + auto column = ColumnUInt64::create(rows); + + res.insert({ColumnPtr(std::move(column)), std::make_shared(), "__grouping_set"}); + + for (auto & col : block) + res.insert(std::move(col)); + + return res; +} + GroupingSetsTransform::GroupingSetsTransform( Block input_header, Block output_header, @@ -15,11 +30,10 @@ GroupingSetsTransform::GroupingSetsTransform( ColumnNumbers const & missing_columns_, size_t set_id_ ) - : ISimpleTransform(std::move(input_header), std::move(output_header), true) + : ISimpleTransform(std::move(input_header), appendGroupingColumn(std::move(output_header)), true) , params(std::move(params_)) , missing_columns(missing_columns_) , set_id(set_id_) - , output_size(getOutputPort().getHeader().columns()) {} void GroupingSetsTransform::transform(Chunk & chunk) @@ -31,18 +45,18 @@ void GroupingSetsTransform::transform(Chunk & chunk) auto const & output_header = getOutputPort().getHeader(); result_columns.reserve(output_header.columns()); - auto grouping_set_pos = output_header.getPositionByName("__grouping_set"); + const size_t grouping_set_pos = 0; size_t real_column_index = 0, missign_column_index = 0; for (size_t i = 0; i < output_header.columns(); ++i) { if (i == grouping_set_pos) { - result_columns.push_back(ColumnUInt64::create(rows, set_id)); + result_columns.push_back(ColumnConst::create(ColumnUInt64::create(1, set_id), rows)); continue; } - if (missign_column_index < missing_columns.size() && missing_columns[missign_column_index] == i) - result_columns.push_back(output_header.getByPosition(missing_columns[missign_column_index++]).column->cloneResized(rows)); + if (missign_column_index < missing_columns.size() && missing_columns[missign_column_index] + 1 == i) + result_columns.push_back(output_header.getByPosition(missing_columns[missign_column_index++] + 1).column->cloneResized(rows)); else result_columns.push_back(std::move(columns[real_column_index++])); } diff --git a/src/Processors/Transforms/GroupingSetsTransform.h b/src/Processors/Transforms/GroupingSetsTransform.h index 95bd3fe9779..7eaadfa25b6 100644 --- a/src/Processors/Transforms/GroupingSetsTransform.h +++ b/src/Processors/Transforms/GroupingSetsTransform.h @@ -24,6 +24,8 @@ public: size_t set_id); String getName() const override { return "GroupingSetsTransform"; } + static Block appendGroupingColumn(Block block); + protected: void transform(Chunk & chunk) override; @@ -31,7 +33,6 @@ private: AggregatingTransformParamsPtr params; const ColumnNumbers missing_columns; const size_t set_id; - const size_t output_size; }; } From f7dbd48ee560e370b99355a1e1e0c2251ae9e58e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 10 May 2022 16:12:03 +0000 Subject: [PATCH 087/171] Simplify code a little bit. --- src/Interpreters/Aggregator.cpp | 2 -- src/Interpreters/Aggregator.h | 17 +---------------- src/Interpreters/InterpreterSelectQuery.h | 3 +++ src/Processors/QueryPlan/AggregatingStep.h | 15 +++++++++++++++ 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index dae00951dc2..a76558cbc9d 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include @@ -8,7 +7,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index a0863822bae..1806465db4a 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -872,21 +872,6 @@ using ManyAggregatedDataVariantsPtr = std::shared_ptr; - /** How are "total" values calculated with WITH TOTALS? * (For more details, see TotalsHavingTransform.) * @@ -913,7 +898,7 @@ public: Block intermediate_header; /// What to count. - ColumnNumbers keys; + const ColumnNumbers keys; const AggregateDescriptions aggregates; const size_t keys_size; const size_t aggregates_size; diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 237b305ae03..ed6320963ed 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -29,6 +29,9 @@ class InterpreterSelectWithUnionQuery; class Context; class QueryPlan; +struct GroupingSetsParams; +using GroupingSetsParamsList = std::vector; + struct TreeRewriterResult; using TreeRewriterResultPtr = std::shared_ptr; using AggregatorParamsPtr = std::unique_ptr; diff --git a/src/Processors/QueryPlan/AggregatingStep.h b/src/Processors/QueryPlan/AggregatingStep.h index bb3020c084c..b933daaa474 100644 --- a/src/Processors/QueryPlan/AggregatingStep.h +++ b/src/Processors/QueryPlan/AggregatingStep.h @@ -10,6 +10,21 @@ namespace DB struct AggregatingTransformParams; using AggregatingTransformParamsPtr = std::shared_ptr; +struct GroupingSetsParams +{ + GroupingSetsParams() = default; + + GroupingSetsParams(ColumnNumbers used_keys_, ColumnNumbers missing_keys_) + : used_keys(std::move(used_keys_)) + , missing_keys(std::move(missing_keys_)) + {} + + ColumnNumbers used_keys; + ColumnNumbers missing_keys; +}; + +using GroupingSetsParamsList = std::vector; + /// Aggregation. See AggregatingTransform. class AggregatingStep : public ITransformingStep { From b654eda2b40f973a462c2b981d277f14692fad41 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 10 May 2022 17:55:09 +0200 Subject: [PATCH 088/171] Better test, remove file --- src/Common/getQueryId.h | 14 -------------- ...40_system_remote_filesystem_cache_log.reference | 9 ++++----- .../02240_system_remote_filesystem_cache_log.sql | 8 ++++---- 3 files changed, 8 insertions(+), 23 deletions(-) delete mode 100644 src/Common/getQueryId.h diff --git a/src/Common/getQueryId.h b/src/Common/getQueryId.h deleted file mode 100644 index 96dcbb5cb22..00000000000 --- a/src/Common/getQueryId.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once -#include - -namespace DB -{ - -static String getQueryId() -{ - if (!CurrentThread::isInitialized() || !CurrentThread::get().getQueryContext()) - return ""; - return CurrentThread::getQueryId().toString(); -} - -} diff --git a/tests/queries/0_stateless/02240_system_remote_filesystem_cache_log.reference b/tests/queries/0_stateless/02240_system_remote_filesystem_cache_log.reference index 9b0d13a1625..0525b8e3937 100644 --- a/tests/queries/0_stateless/02240_system_remote_filesystem_cache_log.reference +++ b/tests/queries/0_stateless/02240_system_remote_filesystem_cache_log.reference @@ -7,12 +7,11 @@ DROP TABLE IF EXISTS test; DROP TABLE IF EXISTS system.filesystem_cache_log; CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache', min_bytes_for_wide_part = 10485760; INSERT INTO test SELECT number, toString(number) FROM numbers(100000); -SELECT * FROM test FORMAT Null; +SELECT 2240, * FROM test FORMAT Null; SYSTEM FLUSH LOGS; -SELECT file_segment_range, read_type FROM system.filesystem_cache_log WHERE read_type='READ_FROM_FS_AND_DOWNLOADED_TO_CACHE'; -(0,519) READ_FROM_FS_AND_DOWNLOADED_TO_CACHE +SELECT file_segment_range, read_type FROM system.filesystem_cache_log WHERE read_type='READ_FROM_FS_AND_DOWNLOADED_TO_CACHE' AND query_id = (SELECT query_id from system.query_log where query LIKE '%SELECT 2240%' ORDER BY event_time desc LIMIT 1); (0,808110) READ_FROM_FS_AND_DOWNLOADED_TO_CACHE -SELECT * FROM test FORMAT Null; +SELECT 2241, * FROM test FORMAT Null; SYSTEM FLUSH LOGS; -SELECT file_segment_range, read_type FROM system.filesystem_cache_log WHERE read_type='READ_FROM_CACHE'; +SELECT file_segment_range, read_type FROM system.filesystem_cache_log WHERE read_type='READ_FROM_CACHE' AND query_id = (SELECT query_id from system.query_log where query LIKE '%SELECT 2241%' ORDER BY event_time desc LIMIT 1); (0,808110) READ_FROM_CACHE diff --git a/tests/queries/0_stateless/02240_system_remote_filesystem_cache_log.sql b/tests/queries/0_stateless/02240_system_remote_filesystem_cache_log.sql index 59e164d6f9f..4214aee6d93 100644 --- a/tests/queries/0_stateless/02240_system_remote_filesystem_cache_log.sql +++ b/tests/queries/0_stateless/02240_system_remote_filesystem_cache_log.sql @@ -11,10 +11,10 @@ DROP TABLE IF EXISTS system.filesystem_cache_log; CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache', min_bytes_for_wide_part = 10485760; INSERT INTO test SELECT number, toString(number) FROM numbers(100000); -SELECT * FROM test FORMAT Null; +SELECT 2240, * FROM test FORMAT Null; SYSTEM FLUSH LOGS; -SELECT file_segment_range, read_type FROM system.filesystem_cache_log WHERE read_type='READ_FROM_FS_AND_DOWNLOADED_TO_CACHE'; +SELECT file_segment_range, read_type FROM system.filesystem_cache_log WHERE read_type='READ_FROM_FS_AND_DOWNLOADED_TO_CACHE' AND query_id = (SELECT query_id from system.query_log where query LIKE '%SELECT 2240%' ORDER BY event_time desc LIMIT 1); -SELECT * FROM test FORMAT Null; +SELECT 2241, * FROM test FORMAT Null; SYSTEM FLUSH LOGS; -SELECT file_segment_range, read_type FROM system.filesystem_cache_log WHERE read_type='READ_FROM_CACHE'; +SELECT file_segment_range, read_type FROM system.filesystem_cache_log WHERE read_type='READ_FROM_CACHE' AND query_id = (SELECT query_id from system.query_log where query LIKE '%SELECT 2241%' ORDER BY event_time desc LIMIT 1); From b6075031d848d827b84f087dbd31fe0f206d2e7b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 10 May 2022 17:54:36 +0000 Subject: [PATCH 089/171] Delete GroupingSetsTransform. --- src/Processors/QueryPlan/AggregatingStep.cpp | 58 +++++++++++++--- .../Transforms/GroupingSetsTransform.cpp | 67 ------------------- .../Transforms/GroupingSetsTransform.h | 38 ----------- .../01883_with_grouping_sets.reference | 4 +- 4 files changed, 52 insertions(+), 115 deletions(-) delete mode 100644 src/Processors/Transforms/GroupingSetsTransform.cpp delete mode 100644 src/Processors/Transforms/GroupingSetsTransform.h diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index e7c87843f16..f090f78e41a 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -7,11 +7,11 @@ #include #include #include +#include #include #include #include #include -#include #include namespace DB @@ -38,7 +38,17 @@ static Block appendGroupingColumn(Block block, const GroupingSetsParamsList & pa if (params.empty()) return block; - return GroupingSetsTransform::appendGroupingColumn(std::move(block)); + Block res; + + size_t rows = block.rows(); + auto column = ColumnUInt64::create(rows); + + res.insert({ColumnPtr(std::move(column)), std::make_shared(), "__grouping_set"}); + + for (auto & col : block) + res.insert(std::move(col)); + + return res; } AggregatingStep::AggregatingStep( @@ -67,7 +77,7 @@ AggregatingStep::AggregatingStep( { } -void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) +void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) { QueryPipelineProcessorsCollector collector(pipeline, this); @@ -120,7 +130,6 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B return copiers; }); - std::vector transform_params_per_set; pipeline.transform([&](OutputPortRawPtrs ports) { assert(streams * grouping_sets_size == ports.size()); @@ -148,7 +157,6 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B transform_params->params.stats_collecting_params }; auto transform_params_for_set = std::make_shared(std::move(params_for_set), final); - transform_params_per_set.push_back(transform_params_for_set); if (streams > 1) { @@ -179,9 +187,10 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B Processors resizes; for (size_t i = 0; i < grouping_sets_size; ++i) { - auto resize = std::make_shared(transform_params_per_set[i]->getHeader(), streams, 1); - auto & inputs = resize->getInputs(); auto output_it = ports.begin() + i * streams; + auto resize = std::make_shared((*output_it)->getHeader(), streams, 1); + auto & inputs = resize->getInputs(); + for (auto input_it = inputs.begin(); input_it != inputs.end(); ++output_it, ++input_it) connect(**output_it, *input_it); resizes.push_back(resize); @@ -195,7 +204,40 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B auto output_header = transform_params->getHeader(); pipeline.addSimpleTransform([&](const Block & header) { - auto transform = std::make_shared(header, output_header, transform_params_per_set[set_counter], grouping_sets_params[set_counter].missing_keys, set_counter); + /// Here we create a DAG which fills missing keys and adds `__grouping_set` column + auto dag = std::make_shared(header.getColumnsWithTypeAndName()); + ActionsDAG::NodeRawConstPtrs index; + index.reserve(output_header.columns() + 1); + + auto grouping_col = ColumnConst::create(ColumnUInt64::create(1, set_counter), 0); + const auto * grouping_node = &dag->addColumn( + {ColumnPtr(std::move(grouping_col)), std::make_shared(), "__grouping_set"}); + + grouping_node = &dag->materializeNode(*grouping_node); + index.push_back(grouping_node); + + size_t real_column_index = 0; + size_t missign_column_index = 0; + const auto & missing_columns = grouping_sets_params[set_counter].missing_keys; + + for (size_t i = 0; i < output_header.columns(); ++i) + { + if (missign_column_index < missing_columns.size() && missing_columns[missign_column_index] == i) + { + auto missing = output_header.getByPosition(missing_columns[missign_column_index++]); + auto column = ColumnConst::create(missing.column->cloneResized(1), 0); + const auto * node = &dag->addColumn({ColumnPtr(std::move(column)), missing.type, missing.name}); + node = &dag->materializeNode(*node); + index.push_back(node); + } + else + index.push_back(dag->getIndex()[real_column_index++]); + } + + dag->getIndex().swap(index); + auto expression = std::make_shared(dag, settings.getActionsSettings()); + auto transform = std::make_shared(header, expression); + ++set_counter; return transform; }); diff --git a/src/Processors/Transforms/GroupingSetsTransform.cpp b/src/Processors/Transforms/GroupingSetsTransform.cpp deleted file mode 100644 index 74494d54c2f..00000000000 --- a/src/Processors/Transforms/GroupingSetsTransform.cpp +++ /dev/null @@ -1,67 +0,0 @@ -#include -#include -#include -#include -#include -#include - -namespace DB -{ - -Block GroupingSetsTransform::appendGroupingColumn(Block block) -{ - Block res; - - size_t rows = block.rows(); - auto column = ColumnUInt64::create(rows); - - res.insert({ColumnPtr(std::move(column)), std::make_shared(), "__grouping_set"}); - - for (auto & col : block) - res.insert(std::move(col)); - - return res; -} - -GroupingSetsTransform::GroupingSetsTransform( - Block input_header, - Block output_header, - AggregatingTransformParamsPtr params_, - ColumnNumbers const & missing_columns_, - size_t set_id_ -) - : ISimpleTransform(std::move(input_header), appendGroupingColumn(std::move(output_header)), true) - , params(std::move(params_)) - , missing_columns(missing_columns_) - , set_id(set_id_) -{} - -void GroupingSetsTransform::transform(Chunk & chunk) -{ - size_t rows = chunk.getNumRows(); - - auto columns = chunk.detachColumns(); - Columns result_columns; - auto const & output_header = getOutputPort().getHeader(); - - result_columns.reserve(output_header.columns()); - const size_t grouping_set_pos = 0; - - size_t real_column_index = 0, missign_column_index = 0; - for (size_t i = 0; i < output_header.columns(); ++i) - { - if (i == grouping_set_pos) - { - result_columns.push_back(ColumnConst::create(ColumnUInt64::create(1, set_id), rows)); - continue; - } - if (missign_column_index < missing_columns.size() && missing_columns[missign_column_index] + 1 == i) - result_columns.push_back(output_header.getByPosition(missing_columns[missign_column_index++] + 1).column->cloneResized(rows)); - else - result_columns.push_back(std::move(columns[real_column_index++])); - } - - chunk.setColumns(std::move(result_columns), rows); -} - -} diff --git a/src/Processors/Transforms/GroupingSetsTransform.h b/src/Processors/Transforms/GroupingSetsTransform.h deleted file mode 100644 index 7eaadfa25b6..00000000000 --- a/src/Processors/Transforms/GroupingSetsTransform.h +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once -#include -#include -#include - -namespace DB -{ - -/* - * This transform is used as the last step of aggregation with GROUPING SETS modifier. - * The main purpose is to add a '__grouping_set' column which stores information - * about grouping keys set used to generate rows. - * '__grouping_set' column is required during MergeAggregated step to distinguish - * generated on different shards rows. -*/ -class GroupingSetsTransform : public ISimpleTransform -{ -public: - GroupingSetsTransform( - Block input_header, - Block output_header, - AggregatingTransformParamsPtr params, - ColumnNumbers const & missing_columns, - size_t set_id); - String getName() const override { return "GroupingSetsTransform"; } - - static Block appendGroupingColumn(Block block); - -protected: - void transform(Chunk & chunk) override; - -private: - AggregatingTransformParamsPtr params; - const ColumnNumbers missing_columns; - const size_t set_id; -}; - -} diff --git a/tests/queries/0_stateless/01883_with_grouping_sets.reference b/tests/queries/0_stateless/01883_with_grouping_sets.reference index ff2b70783c9..a036ccb0796 100644 --- a/tests/queries/0_stateless/01883_with_grouping_sets.reference +++ b/tests/queries/0_stateless/01883_with_grouping_sets.reference @@ -8,7 +8,7 @@ ExpressionTransform (Expression) ExpressionTransform × 2 (Aggregating) - GroupingSetsTransform × 2 + ExpressionTransform × 2 AggregatingTransform × 2 Copy 1 → 2 (Expression) @@ -87,7 +87,7 @@ ExpressionTransform (Expression) ExpressionTransform × 2 (Aggregating) - GroupingSetsTransform × 2 + ExpressionTransform × 2 Resize × 2 3 → 1 AggregatingTransform × 6 Copy × 3 1 → 2 From bde2733aa2ed67b83b24b0ff2909c4e75452a678 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 11 May 2022 01:25:41 +0200 Subject: [PATCH 090/171] Update tests --- .../queries/0_stateless/01461_query_start_time_microseconds.sql | 1 + tests/queries/0_stateless/01473_event_time_microseconds.sql | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/01461_query_start_time_microseconds.sql b/tests/queries/0_stateless/01461_query_start_time_microseconds.sql index be1d9897053..9f78a1f95e1 100644 --- a/tests/queries/0_stateless/01461_query_start_time_microseconds.sql +++ b/tests/queries/0_stateless/01461_query_start_time_microseconds.sql @@ -23,6 +23,7 @@ WITH ( ) AS t) SELECT if(dateDiff('second', toDateTime(time_with_microseconds), toDateTime(t)) = 0, 'ok', 'fail'); -- +SET log_query_threads = 1; SELECT '01461_query_thread_log_query_start_time_milliseconds_test'; SYSTEM FLUSH LOGS; -- assumes that the query_start_time field is already accurate. diff --git a/tests/queries/0_stateless/01473_event_time_microseconds.sql b/tests/queries/0_stateless/01473_event_time_microseconds.sql index 932acf48cc4..63ecfde62e2 100644 --- a/tests/queries/0_stateless/01473_event_time_microseconds.sql +++ b/tests/queries/0_stateless/01473_event_time_microseconds.sql @@ -41,6 +41,7 @@ WITH ( ) AS time SELECT if(dateDiff('second', toDateTime(time.1), toDateTime(time.2)) = 0, 'ok', toString(time)); +SET log_query_threads = 1; SELECT '01473_query_thread_log_table_event_start_time_microseconds_test'; WITH ( SELECT event_time_microseconds, event_time From b793bff179c4684effa6b43f2596be2bc7be15dd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 11 May 2022 03:09:53 +0200 Subject: [PATCH 091/171] Update test --- tests/queries/0_stateless/01473_event_time_microseconds.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01473_event_time_microseconds.sql b/tests/queries/0_stateless/01473_event_time_microseconds.sql index 63ecfde62e2..2b443cf82eb 100644 --- a/tests/queries/0_stateless/01473_event_time_microseconds.sql +++ b/tests/queries/0_stateless/01473_event_time_microseconds.sql @@ -6,6 +6,7 @@ -- Refer: tests/integration/test_asynchronous_metric_log_table. SET log_queries = 1; +SET log_query_threads = 1; SET query_profiler_real_time_period_ns = 100000000; -- a long enough query to trigger the query profiler and to record trace log SELECT sleep(2) FORMAT Null; @@ -41,7 +42,6 @@ WITH ( ) AS time SELECT if(dateDiff('second', toDateTime(time.1), toDateTime(time.2)) = 0, 'ok', toString(time)); -SET log_query_threads = 1; SELECT '01473_query_thread_log_table_event_start_time_microseconds_test'; WITH ( SELECT event_time_microseconds, event_time From 175175a16e9b3898980a66e1cbb96321a8a980ac Mon Sep 17 00:00:00 2001 From: fenglv Date: Wed, 11 May 2022 04:16:05 +0000 Subject: [PATCH 092/171] fix wrong argument in proxy resolver of DiskS3 --- src/Disks/S3/ProxyResolverConfiguration.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/S3/ProxyResolverConfiguration.cpp b/src/Disks/S3/ProxyResolverConfiguration.cpp index bb558cf4b72..7449c5200de 100644 --- a/src/Disks/S3/ProxyResolverConfiguration.cpp +++ b/src/Disks/S3/ProxyResolverConfiguration.cpp @@ -63,7 +63,7 @@ Aws::Client::ClientConfigurationPerRequest ProxyResolverConfiguration::getConfig { auto resolved_endpoint = endpoint; resolved_endpoint.setHost(resolved_hosts[i].toString()); - session = makeHTTPSession(endpoint, timeouts, false); + session = makeHTTPSession(resolved_endpoint, timeouts, false); try { From 5f9eee976f91827a94f0bc3e0c0d2fe7550ee8dd Mon Sep 17 00:00:00 2001 From: qieqieplus Date: Mon, 9 May 2022 15:26:46 +0800 Subject: [PATCH 093/171] fix & format --- .gitmodules | 2 +- contrib/eigen | 2 +- contrib/eigen-cmake/CMakeLists.txt | 2 +- src/Functions/array/arrayDistance.cpp | 2 +- src/Functions/array/arrayNorm.cpp | 2 +- ...array_distance.reference => 02282_array_distance.reference} | 0 .../{02280_array_distance.sql => 02282_array_distance.sql} | 3 --- .../{02280_array_norm.reference => 02283_array_norm.reference} | 0 .../0_stateless/{02280_array_norm.sql => 02283_array_norm.sql} | 0 9 files changed, 5 insertions(+), 8 deletions(-) rename tests/queries/0_stateless/{02280_array_distance.reference => 02282_array_distance.reference} (100%) rename tests/queries/0_stateless/{02280_array_distance.sql => 02282_array_distance.sql} (89%) rename tests/queries/0_stateless/{02280_array_norm.reference => 02283_array_norm.reference} (100%) rename tests/queries/0_stateless/{02280_array_norm.sql => 02283_array_norm.sql} (100%) diff --git a/.gitmodules b/.gitmodules index 84507a0935b..cc03d05622f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -264,4 +264,4 @@ url = https://github.com/zlib-ng/minizip-ng [submodule "contrib/eigen"] path = contrib/eigen - url = https://gitlab.com/libeigen/eigen.git + url = https://github.com/eigen-mirror/eigen diff --git a/contrib/eigen b/contrib/eigen index 51a0b4e2d2d..3147391d946 160000 --- a/contrib/eigen +++ b/contrib/eigen @@ -1 +1 @@ -Subproject commit 51a0b4e2d2d6f22ec88529b520e9d954615fec1d +Subproject commit 3147391d946bb4b6c68edd901f2add6ac1f31f8c diff --git a/contrib/eigen-cmake/CMakeLists.txt b/contrib/eigen-cmake/CMakeLists.txt index 58b41d122e0..c8548b7fbc1 100644 --- a/contrib/eigen-cmake/CMakeLists.txt +++ b/contrib/eigen-cmake/CMakeLists.txt @@ -11,7 +11,7 @@ if (ENABLE_MKL) message("MKL INCLUDE: ${MKL_INCLUDE}") message("MKL LIBRARIES: ${MKL_LIBRARIES}") target_compile_definitions(_eigen INTERFACE EIGEN_USE_MKL_ALL) - target_include_directories(_eigen PRIVATE ${MKL_INCLUDE}) + target_include_directories(_eigen INTERFACE ${MKL_INCLUDE}) target_link_libraries(_eigen INTERFACE ${MKL_LIBRARIES}) endif() endif() diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index d405b9a4a94..8996236058a 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -2,9 +2,9 @@ #include #include #include +#include #include #include -#include namespace DB { diff --git a/src/Functions/array/arrayNorm.cpp b/src/Functions/array/arrayNorm.cpp index cf8ae7524dc..aefa219a720 100644 --- a/src/Functions/array/arrayNorm.cpp +++ b/src/Functions/array/arrayNorm.cpp @@ -2,9 +2,9 @@ #include #include #include +#include #include #include -#include namespace DB { diff --git a/tests/queries/0_stateless/02280_array_distance.reference b/tests/queries/0_stateless/02282_array_distance.reference similarity index 100% rename from tests/queries/0_stateless/02280_array_distance.reference rename to tests/queries/0_stateless/02282_array_distance.reference diff --git a/tests/queries/0_stateless/02280_array_distance.sql b/tests/queries/0_stateless/02282_array_distance.sql similarity index 89% rename from tests/queries/0_stateless/02280_array_distance.sql rename to tests/queries/0_stateless/02282_array_distance.sql index 835b2cf15ae..0f269b11179 100644 --- a/tests/queries/0_stateless/02280_array_distance.sql +++ b/tests/queries/0_stateless/02282_array_distance.sql @@ -22,9 +22,6 @@ SELECT arrayCosineDistance(v, materialize([1., 1., 1.])) FROM vec1; INSERT INTO vec2 VALUES (1, [100, 200, 0]), (2, [888, 777, 666]); SELECT v1.id, v2.id, arrayL2Distance(v1.v, v2.v) as dist FROM vec1 v1, vec2 v2; -INSERT INTO vec2 VALUES (3, [123]); -SELECT v1.id, v2.id, arrayL2Distance(v1.v, v2.v) as dist FROM vec1 v1, vec2 v2; -- { serverError 190 } - SELECT arrayL1Distance([0, 0], [1]); -- { serverError 190 } SELECT arrayL2Distance((1, 2), (3,4)); -- { serverError 43 } diff --git a/tests/queries/0_stateless/02280_array_norm.reference b/tests/queries/0_stateless/02283_array_norm.reference similarity index 100% rename from tests/queries/0_stateless/02280_array_norm.reference rename to tests/queries/0_stateless/02283_array_norm.reference diff --git a/tests/queries/0_stateless/02280_array_norm.sql b/tests/queries/0_stateless/02283_array_norm.sql similarity index 100% rename from tests/queries/0_stateless/02280_array_norm.sql rename to tests/queries/0_stateless/02283_array_norm.sql From 18dd74be0c8cd6ab505f09722134e47c0827df75 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 10 May 2022 19:50:43 +0200 Subject: [PATCH 094/171] Remove unneede changes --- src/Common/FileCache.cpp | 44 ++-- src/Common/FileCache.h | 10 +- src/Common/FileSegment.cpp | 219 +++++++----------- src/Common/FileSegment.h | 53 +---- src/Disks/IO/CachedReadBufferFromRemoteFS.cpp | 37 ++- src/Disks/IO/CachedReadBufferFromRemoteFS.h | 2 +- src/IO/WriteBufferFromS3.cpp | 112 ++++++--- src/IO/WriteBufferFromS3.h | 7 +- src/Interpreters/InterpreterSystemQuery.cpp | 4 +- src/Parsers/ASTSystemQuery.cpp | 2 - src/Parsers/ASTSystemQuery.h | 2 - .../02293_force_drop_cache.reference | 1 - .../0_stateless/02293_force_drop_cache.sh | 45 ---- 13 files changed, 217 insertions(+), 321 deletions(-) delete mode 100644 tests/queries/0_stateless/02293_force_drop_cache.reference delete mode 100755 tests/queries/0_stateless/02293_force_drop_cache.sh diff --git a/src/Common/FileCache.cpp b/src/Common/FileCache.cpp index 5ac6ef0e588..ae1b1afdd09 100644 --- a/src/Common/FileCache.cpp +++ b/src/Common/FileCache.cpp @@ -426,6 +426,25 @@ LRUFileCache::FileSegmentCell * LRUFileCache::addCell( return &(it->second); } +FileSegmentsHolder LRUFileCache::setDownloading(const Key & key, size_t offset, size_t size) +{ + std::lock_guard cache_lock(mutex); + +#ifndef NDEBUG + assertCacheCorrectness(key, cache_lock); +#endif + + auto * cell = getCell(key, offset, cache_lock); + if (cell) + throw Exception( + ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, + "Cache cell already exists for key `{}` and offset {}", + keyToStr(key), offset); + + auto file_segments = splitRangeIntoCells(key, offset, size, FileSegment::State::DOWNLOADING, cache_lock); + return FileSegmentsHolder(std::move(file_segments)); +} + bool LRUFileCache::tryReserve( const Key & key, size_t offset, size_t size, std::lock_guard & cache_lock) { @@ -590,7 +609,7 @@ void LRUFileCache::remove(const Key & key) #endif } -void LRUFileCache::remove(bool force_remove_unreleasable) +void LRUFileCache::remove() { /// Try remove all cached files by cache_base_path. /// Only releasable file segments are evicted. @@ -607,15 +626,13 @@ void LRUFileCache::remove(bool force_remove_unreleasable) ErrorCodes::LOGICAL_ERROR, "Cache is in inconsistent state: LRU queue contains entries with no cache cell"); - if (cell->releasable() || force_remove_unreleasable) + if (cell->releasable()) { auto file_segment = cell->file_segment; if (file_segment) { - std::lock_guard detach_lock(file_segment->detach_mutex); std::lock_guard segment_lock(file_segment->mutex); - - file_segment->detach(force_remove_unreleasable, cache_lock, detach_lock, segment_lock); + file_segment->detach(cache_lock, segment_lock); remove(file_segment->key(), file_segment->offset(), cache_lock, segment_lock); } } @@ -813,23 +830,6 @@ std::vector LRUFileCache::tryGetCachePaths(const Key & key) return cache_paths; } -FileSegmentPtr LRUFileCache::setDownloading(const Key & key, size_t offset, size_t size, std::lock_guard & cache_lock) -{ - auto * cell = getCell(key, offset, cache_lock); - if (cell) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Cache cell already exists for key `{}` and offset {}", - keyToStr(key), offset); - - cell = addCell(key, offset, size, FileSegment::State::DOWNLOADING, cache_lock); - - if (!cell) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to add a new cell for download"); - - return cell->file_segment; -} - size_t LRUFileCache::getUsedCacheSize() const { std::lock_guard cache_lock(mutex); diff --git a/src/Common/FileCache.h b/src/Common/FileCache.h index 0218fb897d3..ff65b579470 100644 --- a/src/Common/FileCache.h +++ b/src/Common/FileCache.h @@ -43,7 +43,7 @@ public: virtual void remove(const Key & key) = 0; - virtual void remove(bool force_remove_unreleasable) = 0; + virtual void remove() = 0; static bool isReadOnly(); @@ -84,6 +84,8 @@ public: */ virtual FileSegmentsHolder get(const Key & key, size_t offset, size_t size) = 0; + virtual FileSegmentsHolder setDownloading(const Key & key, size_t offset, size_t size) = 0; + virtual FileSegments getSnapshot() const = 0; /// For debug. @@ -124,8 +126,6 @@ protected: std::lock_guard & cache_lock, std::lock_guard & segment_lock) = 0; - virtual FileSegmentPtr setDownloading(const Key & key, size_t offset, size_t size, std::lock_guard & cache_lock) = 0; - void assertInitialized() const; }; @@ -148,7 +148,7 @@ public: void remove(const Key & key) override; - void remove(bool force_remove_unreleasable) override; + void remove() override; std::vector tryGetCachePaths(const Key & key) override; @@ -271,7 +271,7 @@ private: void fillHolesWithEmptyFileSegments( FileSegments & file_segments, const Key & key, const FileSegment::Range & range, bool fill_with_detached_file_segments, std::lock_guard & cache_lock); - FileSegmentPtr setDownloading(const Key & key, size_t offset, size_t size, std::lock_guard & cache_lock) override; + FileSegmentsHolder setDownloading(const Key & key, size_t offset, size_t size) override; size_t getUsedCacheSizeUnlocked(std::lock_guard & cache_lock) const; diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index 1d4ec4a9d8e..0fbfcd42e3b 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -57,8 +57,6 @@ FileSegment::FileSegment( /// needed, downloader is set on file segment creation). case (State::DOWNLOADING): { - /// On write-through cache we do not check downloader id. - is_write_through_cache = true; downloader_id = getCallerId(); break; } @@ -170,10 +168,6 @@ String FileSegment::getDownloader() const bool FileSegment::isDownloader() const { std::lock_guard segment_lock(mutex); - - if (is_forcefully_detached) - throwDetachedUnlocked(segment_lock); - return getCallerId() == downloader_id; } @@ -222,7 +216,7 @@ void FileSegment::write(const char * from, size_t size, size_t offset_) ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Not enough space is reserved. Available: {}, expected: {}", availableSize(), size); - if (!isDownloader() && !is_write_through_cache) + if (!isDownloader()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Only downloader can do the downloading. (CallerId: {}, DownloaderId: {})", getCallerId(), downloader_id); @@ -238,8 +232,6 @@ void FileSegment::write(const char * from, size_t size, size_t offset_) "Attempt to write {} bytes to offset: {}, but current download offset is {}", size, offset_, download_offset); - std::lock_guard detach_lock(detach_mutex); - { std::lock_guard segment_lock(mutex); assertNotDetached(segment_lock); @@ -282,13 +274,87 @@ void FileSegment::write(const char * from, size_t size, size_t offset_) assert(getDownloadOffset() == offset_ + size); } +void FileSegment::writeInMemory(const char * from, size_t size) +{ + if (!size) + throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Attempt to write zero size cache file"); + + if (availableSize() < size) + throw Exception( + ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, + "Not enough space is reserved. Available: {}, expected: {}", availableSize(), size); + + std::lock_guard segment_lock(mutex); + assertNotDetached(segment_lock); + + if (cache_writer) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache writer already initialized"); + + auto download_path = cache->getPathInLocalCache(key(), offset()); + cache_writer = std::make_unique(download_path, size + 1); + + try + { + cache_writer->write(from, size); + } + catch (Exception & e) + { + wrapWithCacheInfo(e, "while writing into cache", segment_lock); + + setDownloadFailed(segment_lock); + + cv.notify_all(); + + throw; + } +} + +size_t FileSegment::finalizeWrite() +{ + std::lock_guard segment_lock(mutex); + + if (!cache_writer) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache writer not initialized"); + + size_t size = cache_writer->offset(); + + if (size == 0) + throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Writing zero size is not allowed"); + + assertNotDetached(segment_lock); + + try + { + cache_writer->next(); + } + catch (Exception & e) + { + wrapWithCacheInfo(e, "while writing into cache", segment_lock); + + setDownloadFailed(segment_lock); + + cv.notify_all(); + + throw; + } + + downloaded_size += size; + + if (downloaded_size != range().size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected downloaded size to equal file segment size ({} == {})", downloaded_size, range().size()); + + setDownloaded(segment_lock); + + return size; +} + FileSegment::State FileSegment::wait() { std::unique_lock segment_lock(mutex); if (is_detached) throw Exception( - is_forcefully_detached ? ErrorCodes::CACHE_FILE_SEGMENT_IS_DETACHED : ErrorCodes::LOGICAL_ERROR, + ErrorCodes::LOGICAL_ERROR, "Cache file segment is in detached state, operation not allowed. " "It can happen when cache was concurrently dropped with SYSTEM DROP FILESYSTEM CACHE FORCE"); @@ -316,16 +382,13 @@ bool FileSegment::reserve(size_t size) if (!size) throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Zero space reservation is not allowed"); - std::lock_guard cache_lock(cache->mutex); - std::lock_guard detach_lock(detach_mutex); - { std::lock_guard segment_lock(mutex); assertNotDetached(segment_lock); auto caller_id = getCallerId(); bool is_downloader = caller_id == downloader_id; - if (!is_downloader && !is_write_through_cache) + if (!is_downloader) throw Exception(ErrorCodes::LOGICAL_ERROR, "Space can be reserved only by downloader (current: {}, expected: {})", caller_id, downloader_id); if (downloaded_size + size > range().size()) @@ -344,6 +407,8 @@ bool FileSegment::reserve(size_t size) size_t free_space = reserved_size - downloaded_size; size_t size_to_reserve = size - free_space; + std::lock_guard cache_lock(cache->mutex); + bool reserved = cache->tryReserve(key(), offset(), size_to_reserve, cache_lock); if (reserved) @@ -415,7 +480,7 @@ void FileSegment::complete(State state) assertNotDetached(segment_lock); bool is_downloader = isDownloaderImpl(segment_lock); - if (!is_downloader && !is_write_through_cache) + if (!is_downloader) { cv.notify_all(); throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, @@ -616,7 +681,7 @@ void FileSegment::throwDetached() const void FileSegment::throwDetachedUnlocked(std::lock_guard & segment_lock) const { throw Exception( - is_forcefully_detached ? ErrorCodes::CACHE_FILE_SEGMENT_IS_DETACHED : ErrorCodes::LOGICAL_ERROR, + ErrorCodes::LOGICAL_ERROR, "Cache file segment is in detached state, operation not allowed. " "It can happen when cache was concurrently dropped with SYSTEM DROP FILESYSTEM CACHE FORCE. " "Please, retry. File segment info: {}", getInfoForLogImpl(segment_lock)); @@ -664,23 +729,14 @@ bool FileSegment::hasFinalizedState() const || download_state == State::SKIP_CACHE; } -bool FileSegment::isForcefullyDetached() const -{ - std::lock_guard detach_lock(detach_mutex); - return is_forcefully_detached; -} - void FileSegment::detach( - bool forced_detach, std::lock_guard & /* cache_lock */, - std::lock_guard & /* detach_lock */, std::lock_guard & segment_lock) { if (is_detached) return; markAsDetached(segment_lock); - is_forcefully_detached = forced_detach; download_state = State::PARTIALLY_DOWNLOADED_NO_CONTINUATION; downloader_id.clear(); @@ -762,117 +818,4 @@ String FileSegmentsHolder::toString() return ranges; } - -FileSegmentRangeWriter::FileSegmentRangeWriter( - IFileCache * cache_, - const FileSegment::Key & key_, - size_t max_file_segment_size_) - : cache(cache_) - , key(key_) - , max_file_segment_size(max_file_segment_size_) -{ -} - -void FileSegmentRangeWriter::allocateFileSegment(size_t offset) -{ - std::lock_guard cache_lock(cache->mutex); - - auto file_segment = cache->setDownloading(key, offset, max_file_segment_size, cache_lock); - current_file_segment = file_segment; - - file_segments_holder.add(std::move(file_segment)); -} - -bool FileSegmentRangeWriter::write(char * data, size_t size, size_t offset) -{ - /** - * We want to write eventually some size, which is not known until the very end. - * Therefore we allocate file segments lazily. Each file segment is assigned capacity - * of max_file_segment_size, but reserved_size remains 0, until call to tryReserve(). - * Once current file segment is full (reached max_file_segment_size), we allocate a - * new file segment. All allocated file segments resize in file segments holder. - * If at the end of all writes, the last file segment is not full, then it is resized. - */ - - std::lock_guard lock(mutex); - - if (finalized) - return false; - - if (current_file_segment.expired()) - allocateFileSegment(current_file_segment_start_offset); - - auto file_segment = current_file_segment.lock(); - if (file_segment->getAvailableSize() == 0) - { - file_segment->complete(FileSegment::State::DOWNLOADED); - - allocateFileSegment(current_file_segment_start_offset); - file_segment = current_file_segment.lock(); - } - - bool reserved = file_segment->reserve(size); - if (!reserved) - return false; - - file_segment->write(data, size, offset); - current_file_segment_start_offset += size; - - return true; -} - -void FileSegmentRangeWriter::finalize() -{ - std::lock_guard lock(mutex); - - if (finalized) - return; - - if (file_segments_holder.file_segments.empty() || current_file_segment.expired()) - return; - - auto file_segment = current_file_segment.lock(); - - std::lock_guard cache_lock(cache->mutex); - file_segment->complete(cache_lock); - - finalized = true; -} - -FileSegmentRangeWriter::~FileSegmentRangeWriter() -{ - try - { - if (!finalized) - finalize(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } -} - -void FileSegmentRangeWriter::clearDownloaded() -{ - std::lock_guard lock(mutex); - - current_file_segment.reset(); - - auto & file_segments = file_segments_holder.file_segments; - if (file_segments.empty()) - return; - - std::lock_guard cache_lock(cache->mutex); - - for (auto file_segment_it = file_segments.begin(); file_segment_it != file_segments.end();) - { - auto file_segment = *file_segment_it; - - std::lock_guard segment_lock(file_segment->mutex); - cache->remove(key, file_segment->offset(), cache_lock, segment_lock); - } - - file_segments.clear(); -} - } diff --git a/src/Common/FileSegment.h b/src/Common/FileSegment.h index 55a2680c0db..b521209e671 100644 --- a/src/Common/FileSegment.h +++ b/src/Common/FileSegment.h @@ -113,6 +113,8 @@ public: */ void writeInMemory(const char * from, size_t size); + size_t finalizeWrite(); + RemoteFileReaderPtr getRemoteFileReader(); void setRemoteFileReader(RemoteFileReaderPtr remote_file_reader_); @@ -154,13 +156,9 @@ public: static FileSegmentPtr getSnapshot(const FileSegmentPtr & file_segment, std::lock_guard & cache_lock); void detach( - bool forced_detach, std::lock_guard & cache_lock, - std::lock_guard & /* detach_lock */, std::lock_guard & segment_lock); - bool isForcefullyDetached() const; - [[noreturn]] void throwDetached() const; private: @@ -212,15 +210,11 @@ private: /// global locking order rule: /// 1. cache lock - /// 2. detach lock (if needed) - /// 3. segment lock + /// 2. segment lock mutable std::mutex mutex; std::condition_variable cv; - bool is_write_through_cache = false; - mutable std::mutex detach_mutex; - /// Protects downloaded_size access with actual write into fs. /// downloaded_size is not protected by download_mutex in methods which /// can never be run in parallel to FileSegment::write() method @@ -248,56 +242,15 @@ private: struct FileSegmentsHolder : private boost::noncopyable { - FileSegmentsHolder() = default; - explicit FileSegmentsHolder(FileSegments && file_segments_) : file_segments(std::move(file_segments_)) {} FileSegmentsHolder(FileSegmentsHolder && other) noexcept : file_segments(std::move(other.file_segments)) {} ~FileSegmentsHolder(); - void add(FileSegmentPtr && file_segment) - { - file_segments.push_back(file_segment); - } - FileSegments file_segments{}; String toString(); }; -class FileSegmentRangeWriter -{ -public: - FileSegmentRangeWriter( - IFileCache * cache_, - const FileSegment::Key & key_, - size_t max_file_segment_size_); - - ~FileSegmentRangeWriter(); - - bool write(char * data, size_t size, size_t offset); - - void finalize(); - - /// If exception happened on remote fs write, we consider current cache invalid. - void clearDownloaded(); - -private: - void allocateFileSegment(size_t offset); - - IFileCache * cache; - FileSegment::Key key; - size_t max_file_segment_size; - - FileSegmentsHolder file_segments_holder; - - std::weak_ptr current_file_segment; - size_t current_file_segment_start_offset = 0; - - bool finalized = false; - - std::mutex mutex; -}; - } diff --git a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp index 676b174e4d3..584939b0554 100644 --- a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp +++ b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp @@ -21,7 +21,6 @@ namespace ErrorCodes { extern const int CANNOT_SEEK_THROUGH_FILE; extern const int LOGICAL_ERROR; - extern const int CACHE_FILE_SEGMENT_IS_DETACHED; } static String getQueryId() @@ -31,14 +30,6 @@ static String getQueryId() return CurrentThread::getQueryId().toString(); } -static int getErrorCode(const FileSegmentPtr & file_segment) -{ - if (file_segment->isForcefullyDetached()) - return ErrorCodes::CACHE_FILE_SEGMENT_IS_DETACHED; - else - return ErrorCodes::LOGICAL_ERROR; -} - CachedReadBufferFromRemoteFS::CachedReadBufferFromRemoteFS( const String & remote_fs_object_path_, FileCachePtr cache_, @@ -115,12 +106,12 @@ void CachedReadBufferFromRemoteFS::initialize(size_t offset, size_t size) initialized = true; } -SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getCacheReadBuffer(size_t offset, const FileSegmentPtr & file_segment) const +SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getCacheReadBuffer(size_t offset) const { auto path = cache->getPathInLocalCache(cache_key, offset); auto buf = std::make_shared(path, settings.local_fs_buffer_size); if (buf->size() == 0) - throw Exception(getErrorCode(file_segment), "Attempt to read from an empty cache file: {}", path); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read from an empty cache file: {}", path); return buf; } @@ -185,8 +176,9 @@ SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getReadBufferForFileSegment( if (download_state == FileSegment::State::DOWNLOADED) { read_type = ReadType::CACHED; - return getCacheReadBuffer(range.left, file_segment); - } else + return getCacheReadBuffer(range.left); + } + else { read_type = ReadType::REMOTE_FS_READ_BYPASS_CACHE; return getRemoteFSReadBuffer(file_segment, read_type); @@ -251,7 +243,7 @@ SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getReadBufferForFileSegment( /// file_offset_of_buffer_end read_type = ReadType::CACHED; - return getCacheReadBuffer(range.left, file_segment); + return getCacheReadBuffer(range.left); } if (wait_download_tries++ < wait_download_max_tries) @@ -268,7 +260,7 @@ SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getReadBufferForFileSegment( case FileSegment::State::DOWNLOADED: { read_type = ReadType::CACHED; - return getCacheReadBuffer(range.left, file_segment); + return getCacheReadBuffer(range.left); } case FileSegment::State::PARTIALLY_DOWNLOADED: { @@ -292,7 +284,7 @@ SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getReadBufferForFileSegment( read_type = ReadType::CACHED; file_segment->resetDownloader(); - return getCacheReadBuffer(range.left, file_segment); + return getCacheReadBuffer(range.left); } if (download_offset < file_offset_of_buffer_end) @@ -328,7 +320,7 @@ SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getReadBufferForFileSegment( if (can_start_from_cache) { read_type = ReadType::CACHED; - return getCacheReadBuffer(range.left, file_segment); + return getCacheReadBuffer(range.left); } else { @@ -375,7 +367,7 @@ SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getImplementationBuffer(File if (file_size == 0 || range.left + file_size <= file_offset_of_buffer_end) throw Exception( - getErrorCode(file_segment), + ErrorCodes::LOGICAL_ERROR, "Unexpected state of cache file. Cache file size: {}, cache file offset: {}, " "expected file size to be non-zero and file downloaded size to exceed current file read offset (expected: {} > {})", file_size, @@ -757,10 +749,9 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() if (download_current_segment != file_segment->isDownloader()) { throw Exception( - getErrorCode(file_segment), + ErrorCodes::LOGICAL_ERROR, "Incorrect segment state. Having read type: {}, file segment info: {}", - toString(read_type), - file_segment->getInfoForLog()); + toString(read_type), file_segment->getInfoForLog()); } if (!result) @@ -771,7 +762,7 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() auto cache_file_size = cache_file_reader->size(); if (cache_file_size == 0) throw Exception( - getErrorCode(file_segment), "Attempt to read from an empty cache file: {} (just before actual read)", cache_file_size); + ErrorCodes::LOGICAL_ERROR, "Attempt to read from an empty cache file: {} (just before actual read)", cache_file_size); } #endif @@ -858,7 +849,7 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() cache_file_size = cache_file_reader->size(); throw Exception( - getErrorCode(file_segment), + ErrorCodes::LOGICAL_ERROR, "Having zero bytes, but range is not finished: file offset: {}, reading until: {}, read type: {}, cache file size: {}", file_offset_of_buffer_end, read_until_position, diff --git a/src/Disks/IO/CachedReadBufferFromRemoteFS.h b/src/Disks/IO/CachedReadBufferFromRemoteFS.h index 3c590aefd88..62bb070b437 100644 --- a/src/Disks/IO/CachedReadBufferFromRemoteFS.h +++ b/src/Disks/IO/CachedReadBufferFromRemoteFS.h @@ -54,7 +54,7 @@ private: SeekableReadBufferPtr getReadBufferForFileSegment(FileSegmentPtr & file_segment); - SeekableReadBufferPtr getCacheReadBuffer(size_t offset, const FileSegmentPtr & file_segment) const; + SeekableReadBufferPtr getCacheReadBuffer(size_t offset) const; std::optional getLastNonDownloadedOffset() const; diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index f5b558352d1..9471b5d319b 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -71,7 +71,6 @@ WriteBufferFromS3::WriteBufferFromS3( , s3_settings(s3_settings_) , schedule(std::move(schedule_)) , cache(cache_) - , cache_writer(cache_.get(), cache_->hash(key), /* max_file_segment_size */s3_settings.max_single_part_upload_size) { allocateBuffer(); } @@ -88,17 +87,39 @@ void WriteBufferFromS3::nextImpl() size_t size = offset(); temporary_buffer->write(working_buffer.begin(), size); - if (size && cacheEnabled()) - { - cache_writer.write(working_buffer.begin(), size, current_cache_write_offset); - ProfileEvents::increment(ProfileEvents::RemoteFSCacheDownloadBytes, size); - } - current_cache_write_offset += size; - ThreadGroupStatusPtr running_group = CurrentThread::isInitialized() && CurrentThread::get().getThreadGroup() ? CurrentThread::get().getThreadGroup() : MainThreadStatus::getInstance().getThreadGroup(); + if (cacheEnabled()) + { + auto cache_key = cache->hash(key); + + file_segments_holder.emplace(cache->setDownloading(cache_key, current_download_offset, size)); + current_download_offset += size; + + size_t remaining_size = size; + auto & file_segments = file_segments_holder->file_segments; + for (auto file_segment_it = file_segments.begin(); file_segment_it != file_segments.end(); ++file_segment_it) + { + auto & file_segment = *file_segment_it; + size_t current_size = std::min(file_segment->range().size(), remaining_size); + remaining_size -= current_size; + + if (file_segment->reserve(current_size)) + { + file_segment->writeInMemory(working_buffer.begin(), current_size); + } + else + { + for (auto reset_segment_it = file_segment_it; reset_segment_it != file_segments.end(); ++reset_segment_it) + (*reset_segment_it)->complete(FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); + file_segments.erase(file_segment_it, file_segments.end()); + break; + } + } + } + ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Bytes, offset()); last_part_size += offset(); @@ -112,6 +133,7 @@ void WriteBufferFromS3::nextImpl() writePart(); allocateBuffer(); + file_segments_holder.reset(); } waitForReadyBackGroundTasks(); @@ -170,9 +192,6 @@ void WriteBufferFromS3::finalizeImpl() if (!multipart_upload_id.empty()) completeMultipartUpload(); - - if (cacheEnabled()) - cache_writer.finalize(); } void WriteBufferFromS3::createMultipartUpload() @@ -235,6 +254,12 @@ void WriteBufferFromS3::writePart() fillUploadRequest(task->req, part_number); + if (file_segments_holder) + { + task->cache_files.emplace(std::move(*file_segments_holder)); + file_segments_holder.reset(); + } + schedule([this, task]() { try @@ -246,6 +271,15 @@ void WriteBufferFromS3::writePart() task->exception = std::current_exception(); } + try + { + finalizeCacheIfNeeded(task->cache_files); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + { std::lock_guard lock(bg_tasks_mutex); task->is_finised = true; @@ -262,8 +296,14 @@ void WriteBufferFromS3::writePart() { UploadPartTask task; fillUploadRequest(task.req, part_tags.size() + 1); + if (file_segments_holder) + { + task.cache_files.emplace(std::move(*file_segments_holder)); + file_segments_holder.reset(); + } processUploadRequest(task); part_tags.push_back(task.tag); + finalizeCacheIfNeeded(task.cache_files); } } @@ -352,6 +392,11 @@ void WriteBufferFromS3::makeSinglepartUpload() put_object_task = std::make_unique(); fillPutRequest(put_object_task->req); + if (file_segments_holder) + { + put_object_task->cache_files.emplace(std::move(*file_segments_holder)); + file_segments_holder.reset(); + } schedule([this]() { @@ -364,6 +409,15 @@ void WriteBufferFromS3::makeSinglepartUpload() put_object_task->exception = std::current_exception(); } + try + { + finalizeCacheIfNeeded(put_object_task->cache_files); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + { std::lock_guard lock(bg_tasks_mutex); put_object_task->is_finised = true; @@ -379,7 +433,13 @@ void WriteBufferFromS3::makeSinglepartUpload() { PutObjectTask task; fillPutRequest(task.req); + if (file_segments_holder) + { + task.cache_files.emplace(std::move(*file_segments_holder)); + file_segments_holder.reset(); + } processPutRequest(task); + finalizeCacheIfNeeded(task.cache_files); } } @@ -407,18 +467,25 @@ void WriteBufferFromS3::processPutRequest(PutObjectTask & task) throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); } -void WriteBufferFromS3::clearCache() +void WriteBufferFromS3::finalizeCacheIfNeeded(std::optional & file_segments_holder) { - if (!cacheEnabled()) + if (!file_segments_holder) return; - try + auto & file_segments = file_segments_holder->file_segments; + for (auto file_segment_it = file_segments.begin(); file_segment_it != file_segments.end();) { - cache_writer.clearDownloaded(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); + try + { + size_t size = (*file_segment_it)->finalizeWrite(); + file_segment_it = file_segments.erase(file_segment_it); + + ProfileEvents::increment(ProfileEvents::RemoteFSCacheDownloadBytes, size); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } } } @@ -438,7 +505,6 @@ void WriteBufferFromS3::waitForReadyBackGroundTasks() if (exception) { waitForAllBackGroundTasks(); - clearCache(); std::rethrow_exception(exception); } @@ -459,10 +525,7 @@ void WriteBufferFromS3::waitForAllBackGroundTasks() { auto & task = upload_object_tasks.front(); if (task.exception) - { - clearCache(); std::rethrow_exception(task.exception); - } part_tags.push_back(task.tag); @@ -473,10 +536,7 @@ void WriteBufferFromS3::waitForAllBackGroundTasks() { bg_tasks_condvar.wait(lock, [this]() { return put_object_task->is_finised; }); if (put_object_task->exception) - { - clearCache(); std::rethrow_exception(put_object_task->exception); - } } } } diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 0b22b234cc8..19faf0b1488 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -119,10 +119,9 @@ private: Poco::Logger * log = &Poco::Logger::get("WriteBufferFromS3"); FileCachePtr cache; - FileSegmentRangeWriter cache_writer; - void clearCache(); - - size_t current_cache_write_offset = 0; + size_t current_download_offset = 0; + std::optional file_segments_holder; + static void finalizeCacheIfNeeded(std::optional &); }; } diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index b52645c7854..0db6f353cf4 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -312,12 +312,12 @@ BlockIO InterpreterSystemQuery::execute() { auto caches = FileCacheFactory::instance().getAll(); for (const auto & [_, cache_data] : caches) - cache_data.cache->remove(query.force_removal); + cache_data.cache->remove(); } else { auto cache = FileCacheFactory::instance().get(query.filesystem_cache_path); - cache->remove(query.force_removal); + cache->remove(); } break; } diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp index 8a7891edfe1..a6ff52b74b7 100644 --- a/src/Parsers/ASTSystemQuery.cpp +++ b/src/Parsers/ASTSystemQuery.cpp @@ -200,8 +200,6 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &, { if (!filesystem_cache_path.empty()) settings.ostr << (settings.hilite ? hilite_none : "") << " " << filesystem_cache_path; - if (force_removal) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " FORCE"; } } diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index 99be7b709bc..eff71a3a9a0 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -91,9 +91,7 @@ public: String disk; UInt64 seconds{}; - /// Values for `drop filesystem cache` system query. String filesystem_cache_path; - bool force_removal = false; String getID(char) const override { return "SYSTEM query"; } diff --git a/tests/queries/0_stateless/02293_force_drop_cache.reference b/tests/queries/0_stateless/02293_force_drop_cache.reference deleted file mode 100644 index d00491fd7e5..00000000000 --- a/tests/queries/0_stateless/02293_force_drop_cache.reference +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/tests/queries/0_stateless/02293_force_drop_cache.sh b/tests/queries/0_stateless/02293_force_drop_cache.sh deleted file mode 100755 index 34416feec5e..00000000000 --- a/tests/queries/0_stateless/02293_force_drop_cache.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -# Tags: long, no-parallel, no-s3-storage - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CURDIR"/../shell_config.sh - -$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test" - -$CLICKHOUSE_CLIENT --query "CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache', min_bytes_for_wide_part = 0" - -$CLICKHOUSE_CLIENT --query "INSERT INTO test SELECT * FROM generateRandom('key UInt32, value String') LIMIT 1000000" - -function select_thread() -{ - $CLICKHOUSE_CLIENT --query "SELECT * FROM test FORMAT Null" -} - -function drop_thread() -{ - $CLICKHOUSE_CLIENT --query "SYSTEM DROP FILESYSTEM CACHE FORCE" -} - -export -f select_thread -export -f drop_thread - -TIMEOUT=30 - -clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & - -clickhouse_client_loop_timeout $TIMEOUT drop_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT drop_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT drop_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT drop_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT drop_thread 2> /dev/null & - -wait - -$CLICKHOUSE_CLIENT --query "SELECT 1" - -$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test" From 5385a4da8ab226f7563ba6b7b9b24432fc89a0df Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 11 May 2022 10:45:20 +0200 Subject: [PATCH 095/171] Fix review comments --- src/Common/ErrorCodes.cpp | 1 - src/Common/FileSegment.cpp | 25 +++++++++++++------------ src/Common/FileSegment.h | 16 ++++++++-------- src/Parsers/ParserSystemQuery.cpp | 2 -- 4 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index b4ebaaf17ff..eb84e24b713 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -624,7 +624,6 @@ M(653, CANNOT_PARSE_BACKUP_SETTINGS) \ M(654, WRONG_BACKUP_SETTINGS) \ M(655, FAILED_TO_RESTORE_METADATA_ON_OTHER_NODE) \ - M(656, CACHE_FILE_SEGMENT_IS_DETACHED) \ \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index 0fbfcd42e3b..bda81617808 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -18,7 +18,6 @@ namespace ErrorCodes { extern const int REMOTE_FS_OBJECT_CACHE_ERROR; extern const int LOGICAL_ERROR; - extern const int CACHE_FILE_SEGMENT_IS_DETACHED; } FileSegment::FileSegment( @@ -85,12 +84,6 @@ size_t FileSegment::getDownloadedSize() const return getDownloadedSize(segment_lock); } -size_t FileSegment::getAvailableSize() const -{ - std::lock_guard segment_lock(mutex); - return range().size() - downloaded_size; -} - size_t FileSegment::getDownloadedSize(std::lock_guard & /* segment_lock */) const { if (download_state == State::DOWNLOADED) @@ -389,7 +382,12 @@ bool FileSegment::reserve(size_t size) auto caller_id = getCallerId(); bool is_downloader = caller_id == downloader_id; if (!is_downloader) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Space can be reserved only by downloader (current: {}, expected: {})", caller_id, downloader_id); + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Space can be reserved only by downloader (current: {}, expected: {})", + caller_id, downloader_id); + } if (downloaded_size + size > range().size()) throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, @@ -672,13 +670,13 @@ void FileSegment::assertCorrectnessImpl(std::lock_guard & /* segment assert(download_state != FileSegment::State::DOWNLOADED || std::filesystem::file_size(cache->getPathInLocalCache(key(), offset())) > 0); } -void FileSegment::throwDetached() const +void FileSegment::throwIfDetached() const { std::lock_guard segment_lock(mutex); - throwDetachedUnlocked(segment_lock); + throwIfDetachedUnlocked(segment_lock); } -void FileSegment::throwDetachedUnlocked(std::lock_guard & segment_lock) const +void FileSegment::throwIfDetachedUnlocked(std::lock_guard & segment_lock) const { throw Exception( ErrorCodes::LOGICAL_ERROR, @@ -691,11 +689,14 @@ void FileSegment::throwDetachedUnlocked(std::lock_guard & segment_lo void FileSegment::assertNotDetached(std::lock_guard & segment_lock) const { if (is_detached) - throwDetachedUnlocked(segment_lock); + throwIfDetachedUnlocked(segment_lock); } void FileSegment::assertDetachedStatus(std::lock_guard & segment_lock) const { + /// Detached file segment is allowed to have only a certain subset of states. + /// It should be either EMPTY or one of the finalized states. + if (download_state != State::EMPTY && !hasFinalizedState()) { throw Exception( diff --git a/src/Common/FileSegment.h b/src/Common/FileSegment.h index b521209e671..38a12311bb7 100644 --- a/src/Common/FileSegment.h +++ b/src/Common/FileSegment.h @@ -137,8 +137,6 @@ public: size_t getDownloadedSize() const; - size_t getAvailableSize() const; - void completeBatchAndResetDownloader(); void complete(State state); @@ -153,13 +151,15 @@ public: void assertCorrectness() const; - static FileSegmentPtr getSnapshot(const FileSegmentPtr & file_segment, std::lock_guard & cache_lock); + static FileSegmentPtr getSnapshot( + const FileSegmentPtr & file_segment, + std::lock_guard & cache_lock); - void detach( - std::lock_guard & cache_lock, - std::lock_guard & segment_lock); + void detach( + std::lock_guard & cache_lock, + std::lock_guard & segment_lock); - [[noreturn]] void throwDetached() const; + [[noreturn]] void throwIfDetached() const; private: size_t availableSize() const { return reserved_size - downloaded_size; } @@ -171,7 +171,7 @@ private: bool isDetached(std::lock_guard & /* segment_lock */) const { return is_detached; } void markAsDetached(std::lock_guard & segment_lock); - [[noreturn]] void throwDetachedUnlocked(std::lock_guard & segment_lock) const; + [[noreturn]] void throwIfDetachedUnlocked(std::lock_guard & segment_lock) const; void assertDetachedStatus(std::lock_guard & segment_lock) const; void assertNotDetached(std::lock_guard & segment_lock) const; diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index 599de6ec828..e18b0aa5e10 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -360,8 +360,6 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & ASTPtr ast; if (path_parser.parse(pos, ast, expected)) res->filesystem_cache_path = ast->as()->value.safeGet(); - if (ParserKeyword{"FORCE"}.ignore(pos, expected)) - res->force_removal = true; break; } From 3dfede3165a240dbc1eec1cd6b64fca1e1615976 Mon Sep 17 00:00:00 2001 From: Vxider Date: Wed, 11 May 2022 09:00:49 +0000 Subject: [PATCH 096/171] support read for windowview --- src/Storages/WindowView/StorageWindowView.cpp | 72 +++++++++++++++++++ src/Storages/WindowView/StorageWindowView.h | 19 +++++ 2 files changed, 91 insertions(+) diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index 9462e115087..0f8d27772d4 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -37,6 +37,11 @@ #include #include #include +#include +#include +#include +#include +#include #include #include #include @@ -965,6 +970,73 @@ void StorageWindowView::threadFuncFireEvent() } } +Pipe StorageWindowView::read( + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + SelectQueryInfo & query_info, + ContextPtr local_context, + QueryProcessingStage::Enum processed_stage, + const size_t max_block_size, + const unsigned num_streams) +{ + QueryPlan plan; + read(plan, column_names, storage_snapshot, query_info, local_context, processed_stage, max_block_size, num_streams); + return plan.convertToPipe( + QueryPlanOptimizationSettings::fromContext(local_context), BuildQueryPipelineSettings::fromContext(local_context)); +} + +void StorageWindowView::read( + QueryPlan & query_plan, + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + SelectQueryInfo & query_info, + ContextPtr local_context, + QueryProcessingStage::Enum processed_stage, + const size_t max_block_size, + const unsigned num_streams) +{ + auto storage = getTargetStorage(); + auto lock = storage->lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); + auto target_metadata_snapshot = storage->getInMemoryMetadataPtr(); + auto target_storage_snapshot = storage->getStorageSnapshot(target_metadata_snapshot, local_context); + + if (query_info.order_optimizer) + query_info.input_order_info = query_info.order_optimizer->getInputOrder(target_metadata_snapshot, local_context); + + storage->read(query_plan, column_names, target_storage_snapshot, query_info, local_context, processed_stage, max_block_size, num_streams); + + if (query_plan.isInitialized()) + { + auto wv_header = getHeaderForProcessingStage(column_names, storage_snapshot, query_info, local_context, processed_stage); + auto target_header = query_plan.getCurrentDataStream().header; + + if (!blocksHaveEqualStructure(wv_header, target_header)) + { + auto converting_actions = ActionsDAG::makeConvertingActions( + target_header.getColumnsWithTypeAndName(), wv_header.getColumnsWithTypeAndName(), ActionsDAG::MatchColumnsMode::Name); + auto converting_step = std::make_unique(query_plan.getCurrentDataStream(), converting_actions); + converting_step->setStepDescription("Convert Target table structure to WindowView structure"); + query_plan.addStep(std::move(converting_step)); + } + + StreamLocalLimits limits; + SizeLimits leaf_limits; + + /// Add table lock for target table. + auto adding_limits_and_quota = std::make_unique( + query_plan.getCurrentDataStream(), + storage, + std::move(lock), + limits, + leaf_limits, + nullptr, + nullptr); + + adding_limits_and_quota->setStepDescription("Lock target table for WindowView"); + query_plan.addStep(std::move(adding_limits_and_quota)); + } +} + Pipe StorageWindowView::watch( const Names & /*column_names*/, const SelectQueryInfo & query_info, diff --git a/src/Storages/WindowView/StorageWindowView.h b/src/Storages/WindowView/StorageWindowView.h index 397e4d4946c..c09923f2859 100644 --- a/src/Storages/WindowView/StorageWindowView.h +++ b/src/Storages/WindowView/StorageWindowView.h @@ -137,6 +137,25 @@ public: void startup() override; void shutdown() override; + Pipe read( + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + SelectQueryInfo & query_info, + ContextPtr context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; + + void read( + QueryPlan & query_plan, + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + SelectQueryInfo & query_info, + ContextPtr context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; + Pipe watch( const Names & column_names, const SelectQueryInfo & query_info, From 88930ae42f3010f89a732c9131b59875c7a06d6e Mon Sep 17 00:00:00 2001 From: Vxider Date: Wed, 11 May 2022 09:01:12 +0000 Subject: [PATCH 097/171] add test --- .../01076_window_view_select.reference | 14 ++++++++ .../0_stateless/01076_window_view_select.sh | 35 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 tests/queries/0_stateless/01076_window_view_select.reference create mode 100755 tests/queries/0_stateless/01076_window_view_select.sh diff --git a/tests/queries/0_stateless/01076_window_view_select.reference b/tests/queries/0_stateless/01076_window_view_select.reference new file mode 100644 index 00000000000..febbcc49a76 --- /dev/null +++ b/tests/queries/0_stateless/01076_window_view_select.reference @@ -0,0 +1,14 @@ +1 1 1990-01-01 12:00:05 +1 2 1990-01-01 12:00:05 +1 3 1990-01-01 12:00:05 +1 4 1990-01-01 12:00:10 +1 5 1990-01-01 12:00:10 +1 6 1990-01-01 12:00:15 +1 7 1990-01-01 12:00:15 +1 +2 +3 +4 +5 +6 +7 diff --git a/tests/queries/0_stateless/01076_window_view_select.sh b/tests/queries/0_stateless/01076_window_view_select.sh new file mode 100755 index 00000000000..cbcc472c127 --- /dev/null +++ b/tests/queries/0_stateless/01076_window_view_select.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --multiquery < Date: Wed, 11 May 2022 17:16:23 +0800 Subject: [PATCH 098/171] fix ci fast test --- docker/test/fasttest/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index d9a5bb23a80..656789cd1a6 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -176,6 +176,7 @@ function clone_submodules contrib/NuRaft contrib/jemalloc contrib/replxx + contrib/eigen ) git submodule sync From 07fe7f5c09ac681f32dec396665fd0728ee37b44 Mon Sep 17 00:00:00 2001 From: Vxider Date: Wed, 11 May 2022 09:16:26 +0000 Subject: [PATCH 099/171] disable read when windowview has no target table --- src/Storages/WindowView/StorageWindowView.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index 0f8d27772d4..7ab3fae2675 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -995,6 +995,9 @@ void StorageWindowView::read( const size_t max_block_size, const unsigned num_streams) { + if (target_table_id.empty()) + return; + auto storage = getTargetStorage(); auto lock = storage->lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); auto target_metadata_snapshot = storage->getInMemoryMetadataPtr(); From a440e1376a594236040e228821fafe8a80f5f696 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 11 May 2022 12:05:50 +0200 Subject: [PATCH 100/171] Update FileSegment.cpp --- src/Common/FileSegment.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index bda81617808..386be0da75d 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -701,7 +701,7 @@ void FileSegment::assertDetachedStatus(std::lock_guard & segment_loc { throw Exception( ErrorCodes::LOGICAL_ERROR, - "Detached file segment has incorrent state: {}", + "Detached file segment has incorrect state: {}", getInfoForLogImpl(segment_lock)); } } From cc5eea486825f3d1f591179dd789f5365759c7ef Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 11 May 2022 12:08:41 +0200 Subject: [PATCH 101/171] Update 02240_system_remote_filesystem_cache_log.sql --- .../0_stateless/02240_system_remote_filesystem_cache_log.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02240_system_remote_filesystem_cache_log.sql b/tests/queries/0_stateless/02240_system_remote_filesystem_cache_log.sql index 4214aee6d93..871f9305c55 100644 --- a/tests/queries/0_stateless/02240_system_remote_filesystem_cache_log.sql +++ b/tests/queries/0_stateless/02240_system_remote_filesystem_cache_log.sql @@ -13,8 +13,8 @@ INSERT INTO test SELECT number, toString(number) FROM numbers(100000); SELECT 2240, * FROM test FORMAT Null; SYSTEM FLUSH LOGS; -SELECT file_segment_range, read_type FROM system.filesystem_cache_log WHERE read_type='READ_FROM_FS_AND_DOWNLOADED_TO_CACHE' AND query_id = (SELECT query_id from system.query_log where query LIKE '%SELECT 2240%' ORDER BY event_time desc LIMIT 1); +SELECT file_segment_range, read_type FROM system.filesystem_cache_log WHERE read_type='READ_FROM_FS_AND_DOWNLOADED_TO_CACHE' AND query_id = (SELECT query_id from system.query_log where query LIKE '%SELECT 2240%' AND current_database = currentDatabase() ORDER BY event_time desc LIMIT 1); SELECT 2241, * FROM test FORMAT Null; SYSTEM FLUSH LOGS; -SELECT file_segment_range, read_type FROM system.filesystem_cache_log WHERE read_type='READ_FROM_CACHE' AND query_id = (SELECT query_id from system.query_log where query LIKE '%SELECT 2241%' ORDER BY event_time desc LIMIT 1); +SELECT file_segment_range, read_type FROM system.filesystem_cache_log WHERE read_type='READ_FROM_CACHE' AND query_id = (SELECT query_id from system.query_log where query LIKE '%SELECT 2241%' AND current_database = currentDatabase() ORDER BY event_time desc LIMIT 1); From d4823eed12caced768aaefcbb8751e71431a2846 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 11 May 2022 12:09:16 +0200 Subject: [PATCH 102/171] Update 02240_system_remote_filesystem_cache_log.reference --- .../02240_system_remote_filesystem_cache_log.reference | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02240_system_remote_filesystem_cache_log.reference b/tests/queries/0_stateless/02240_system_remote_filesystem_cache_log.reference index 0525b8e3937..0a797d38d7e 100644 --- a/tests/queries/0_stateless/02240_system_remote_filesystem_cache_log.reference +++ b/tests/queries/0_stateless/02240_system_remote_filesystem_cache_log.reference @@ -9,9 +9,9 @@ CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SET INSERT INTO test SELECT number, toString(number) FROM numbers(100000); SELECT 2240, * FROM test FORMAT Null; SYSTEM FLUSH LOGS; -SELECT file_segment_range, read_type FROM system.filesystem_cache_log WHERE read_type='READ_FROM_FS_AND_DOWNLOADED_TO_CACHE' AND query_id = (SELECT query_id from system.query_log where query LIKE '%SELECT 2240%' ORDER BY event_time desc LIMIT 1); +SELECT file_segment_range, read_type FROM system.filesystem_cache_log WHERE read_type='READ_FROM_FS_AND_DOWNLOADED_TO_CACHE' AND query_id = (SELECT query_id from system.query_log where query LIKE '%SELECT 2240%' AND current_database = currentDatabase() ORDER BY event_time desc LIMIT 1); (0,808110) READ_FROM_FS_AND_DOWNLOADED_TO_CACHE SELECT 2241, * FROM test FORMAT Null; SYSTEM FLUSH LOGS; -SELECT file_segment_range, read_type FROM system.filesystem_cache_log WHERE read_type='READ_FROM_CACHE' AND query_id = (SELECT query_id from system.query_log where query LIKE '%SELECT 2241%' ORDER BY event_time desc LIMIT 1); +SELECT file_segment_range, read_type FROM system.filesystem_cache_log WHERE read_type='READ_FROM_CACHE' AND query_id = (SELECT query_id from system.query_log where query LIKE '%SELECT 2241%' AND current_database = currentDatabase() ORDER BY event_time desc LIMIT 1); (0,808110) READ_FROM_CACHE From 4b8a2e2d80b784a4f67a76b6510fda01a92485a6 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 11 May 2022 10:22:34 +0000 Subject: [PATCH 103/171] Fix fuzzed queries. --- src/Processors/QueryPlan/AggregatingStep.cpp | 10 ++--- .../01883_grouping_sets_crash.reference | 43 +++++++++++++++++++ .../0_stateless/01883_grouping_sets_crash.sql | 29 +++++++++++++ 3 files changed, 77 insertions(+), 5 deletions(-) diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index f090f78e41a..40316ae4adc 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -216,22 +216,22 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B grouping_node = &dag->materializeNode(*grouping_node); index.push_back(grouping_node); - size_t real_column_index = 0; size_t missign_column_index = 0; const auto & missing_columns = grouping_sets_params[set_counter].missing_keys; for (size_t i = 0; i < output_header.columns(); ++i) { + auto & col = output_header.getByPosition(i); if (missign_column_index < missing_columns.size() && missing_columns[missign_column_index] == i) { - auto missing = output_header.getByPosition(missing_columns[missign_column_index++]); - auto column = ColumnConst::create(missing.column->cloneResized(1), 0); - const auto * node = &dag->addColumn({ColumnPtr(std::move(column)), missing.type, missing.name}); + ++missign_column_index; + auto column = ColumnConst::create(col.column->cloneResized(1), 0); + const auto * node = &dag->addColumn({ColumnPtr(std::move(column)), col.type, col.name}); node = &dag->materializeNode(*node); index.push_back(node); } else - index.push_back(dag->getIndex()[real_column_index++]); + index.push_back(dag->getIndex()[header.getPositionByName(col.name)]); } dag->getIndex().swap(index); diff --git a/tests/queries/0_stateless/01883_grouping_sets_crash.reference b/tests/queries/0_stateless/01883_grouping_sets_crash.reference index fd5a9e61eb3..4d9e967b766 100644 --- a/tests/queries/0_stateless/01883_grouping_sets_crash.reference +++ b/tests/queries/0_stateless/01883_grouping_sets_crash.reference @@ -164,3 +164,46 @@ THIRD QUERY: 8 9 10 +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +w\0\0ldworldwo\0l\0world +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 diff --git a/tests/queries/0_stateless/01883_grouping_sets_crash.sql b/tests/queries/0_stateless/01883_grouping_sets_crash.sql index ffe77f59557..cf56c8546ce 100644 --- a/tests/queries/0_stateless/01883_grouping_sets_crash.sql +++ b/tests/queries/0_stateless/01883_grouping_sets_crash.sql @@ -67,4 +67,33 @@ GROUP BY GROUPING SETS ((fact_3_id, fact_4_id)) ORDER BY fact_3_id ASC; +-- Following two queries were fuzzed +SELECT 'w\0\0ldworldwo\0l\0world' +FROM grouping_sets +GROUP BY + GROUPING SETS ( + ( fact_4_id), + ( NULL), + ( fact_3_id, fact_4_id)) +ORDER BY + NULL ASC, + NULL DESC NULLS FIRST, + fact_3_id ASC, + fact_3_id ASC NULLS LAST, + 'wo\0ldworldwo\0ldworld' ASC NULLS LAST, + 'w\0\0ldworldwo\0l\0world' DESC NULLS FIRST, + 'wo\0ldworldwo\0ldworld' ASC, + NULL ASC NULLS FIRST, + fact_4_id DESC NULLS LAST; + +SELECT fact_3_id +FROM grouping_sets +GROUP BY + GROUPING SETS ( + ( 'wo\0ldworldwo\0ldworldwo\0ldworldwo\0ldworldwo\0ldworldwo\0ldworldwo\0ldworldwo\0ldworld'), + ( NULL), + ( fact_4_id), + ( fact_3_id, fact_4_id)) +ORDER BY fact_3_id ASC NULLS FIRST; + DROP TABLE IF EXISTS grouping_sets; \ No newline at end of file From 0adfee3ecae6f724be1d0389d2ceab87ca20cc1f Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 11 May 2022 12:26:16 +0200 Subject: [PATCH 104/171] Move assertion one layer higher --- src/Disks/IO/CachedReadBufferFromRemoteFS.cpp | 5 ++--- src/Disks/IO/CachedReadBufferFromRemoteFS.h | 1 + src/Disks/IO/ReadBufferFromRemoteFSGather.cpp | 7 +++++-- src/Disks/IO/ReadBufferFromRemoteFSGather.h | 4 ++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp index b1c95e2100c..d5ff0e3b50a 100644 --- a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp +++ b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp @@ -28,6 +28,7 @@ CachedReadBufferFromRemoteFS::CachedReadBufferFromRemoteFS( FileCachePtr cache_, RemoteFSFileReaderCreator remote_file_reader_creator_, const ReadSettings & settings_, + const String & query_id_, size_t read_until_position_) : SeekableReadBuffer(nullptr, 0) #ifndef NDEBUG @@ -41,11 +42,9 @@ CachedReadBufferFromRemoteFS::CachedReadBufferFromRemoteFS( , settings(settings_) , read_until_position(read_until_position_) , remote_file_reader_creator(remote_file_reader_creator_) - , query_id(CurrentThread::getQueryId()) + , query_id(query_id_) , enable_logging(!query_id.empty() && settings_.enable_filesystem_cache_log) { - assert(query_id.empty() - || CurrentThread::isInitialized() && CurrentThread::get().getQueryContext() != nullptr); } void CachedReadBufferFromRemoteFS::appendFilesystemCacheLog( diff --git a/src/Disks/IO/CachedReadBufferFromRemoteFS.h b/src/Disks/IO/CachedReadBufferFromRemoteFS.h index 62bb070b437..da270a1fdae 100644 --- a/src/Disks/IO/CachedReadBufferFromRemoteFS.h +++ b/src/Disks/IO/CachedReadBufferFromRemoteFS.h @@ -26,6 +26,7 @@ public: FileCachePtr cache_, RemoteFSFileReaderCreator remote_file_reader_creator_, const ReadSettings & settings_, + const String & query_id_, size_t read_until_position_); bool nextImpl() override; diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp index 5709822cf75..09271c4e7f8 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp @@ -62,7 +62,7 @@ SeekableReadBufferPtr ReadBufferFromS3Gather::createImplementationBufferImpl(con if (with_cache) { return std::make_shared( - remote_path, settings.remote_fs_cache, remote_file_reader_creator, settings, read_until_position ? read_until_position : file_size); + remote_path, settings.remote_fs_cache, remote_file_reader_creator, settings, query_id, read_until_position ? read_until_position : file_size); } return remote_file_reader_creator(); @@ -103,13 +103,16 @@ ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather( , common_path_prefix(common_path_prefix_) , blobs_to_read(blobs_to_read_) , settings(settings_) - , log(&Poco::Logger::get("ReadBufferFromRemoteFSGather")) , query_id(CurrentThread::getQueryId()) + , log(&Poco::Logger::get("ReadBufferFromRemoteFSGather")) , enable_cache_log(!query_id.empty() && settings.enable_filesystem_cache_log) { with_cache = settings.remote_fs_cache && settings.enable_filesystem_cache && (!IFileCache::isReadOnly() || settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache); + + assert(query_id.empty() + || CurrentThread::isInitialized() && CurrentThread::get().getQueryContext() != nullptr); } diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.h b/src/Disks/IO/ReadBufferFromRemoteFSGather.h index a0933e6ad5d..d6f12b2f45e 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.h +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.h @@ -72,6 +72,8 @@ protected: bool with_cache; + String query_id; + private: SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size); @@ -100,8 +102,6 @@ private: Poco::Logger * log; - String query_id; - size_t total_bytes_read_from_current_file = 0; bool enable_cache_log = false; From 28630b253589746841295be5f7a67364de5bec90 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 11 May 2022 14:13:21 +0200 Subject: [PATCH 105/171] Update test --- .../queries/0_stateless/02286_drop_filesystem_cache.reference | 4 +--- tests/queries/0_stateless/02286_drop_filesystem_cache.sql | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/02286_drop_filesystem_cache.reference b/tests/queries/0_stateless/02286_drop_filesystem_cache.reference index fb18ad12c4d..65de9ee04ab 100644 --- a/tests/queries/0_stateless/02286_drop_filesystem_cache.reference +++ b/tests/queries/0_stateless/02286_drop_filesystem_cache.reference @@ -27,6 +27,4 @@ SELECT count() FROM system.filesystem_cache; 3 SYSTEM DROP FILESYSTEM CACHE './s3_cache/'; SELECT count() FROM system.filesystem_cache; -2 -EXPLAIN SYNTAX SYSTEM DROP FILESYSTEM CACHE './s3_cache/' FORCE; -SYSTEM DROP FILESYSTEM CACHE ./s3_cache/ FORCE +2 \ No newline at end of file diff --git a/tests/queries/0_stateless/02286_drop_filesystem_cache.sql b/tests/queries/0_stateless/02286_drop_filesystem_cache.sql index ee93b165637..85bfb6ca1c7 100644 --- a/tests/queries/0_stateless/02286_drop_filesystem_cache.sql +++ b/tests/queries/0_stateless/02286_drop_filesystem_cache.sql @@ -31,6 +31,4 @@ SELECT * FROM test2 FORMAT Null; SELECT count() FROM system.filesystem_cache; SYSTEM DROP FILESYSTEM CACHE './s3_cache/'; -SELECT count() FROM system.filesystem_cache; - -EXPLAIN SYNTAX SYSTEM DROP FILESYSTEM CACHE './s3_cache/' FORCE; +SELECT count() FROM system.filesystem_cache; \ No newline at end of file From 2d99f0ce13eab84b03bef5ceedb7247de20d6386 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 11 May 2022 12:16:15 +0000 Subject: [PATCH 106/171] Simplify code a little bit. --- src/Interpreters/ExpressionAnalyzer.cpp | 6 - src/Interpreters/InterpreterSelectQuery.cpp | 22 ++-- src/Interpreters/InterpreterSelectQuery.h | 13 --- src/Processors/QueryPlan/AggregatingStep.cpp | 116 ++++++++++--------- src/QueryPipeline/Pipe.cpp | 19 ++- src/QueryPipeline/Pipe.h | 2 +- src/QueryPipeline/QueryPipelineBuilder.cpp | 4 +- src/QueryPipeline/QueryPipelineBuilder.h | 2 +- 8 files changed, 86 insertions(+), 98 deletions(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 25967919183..01769742071 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1560,12 +1560,6 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendProjectResult(ExpressionActio } } - if (getContext()->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY && useGroupingSetKey()) - { - result_columns.emplace_back("__grouping_set", "__grouping_set"); - step.addRequiredOutput("__grouping_set"); - } - auto actions = chain.getLastActions(); actions->project(result_columns); return actions; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index e8c1be5330e..6bf90a4359a 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2187,7 +2187,10 @@ void InterpreterSelectQuery::executeWhere(QueryPlan & query_plan, const ActionsD query_plan.addStep(std::move(where_step)); } -Aggregator::Params InterpreterSelectQuery::getAggregatorParams( +static Aggregator::Params getAggregatorParams( + const ASTPtr & query_ptr, + const SelectQueryExpressionAnalyzer & query_analyzer, + const Context & context, const Block & current_data_stream_header, const ColumnNumbers & keys, const AggregateDescriptions & aggregates, @@ -2212,8 +2215,8 @@ Aggregator::Params InterpreterSelectQuery::getAggregatorParams( settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set || (settings.empty_result_for_aggregation_by_constant_keys_on_empty_set && keys.empty() - && query_analyzer->hasConstAggregationKeys()), - context->getTemporaryVolume(), + && query_analyzer.hasConstAggregationKeys()), + context.getTemporaryVolume(), settings.max_threads, settings.min_free_disk_space_for_temporary_data, settings.compile_aggregate_expressions, @@ -2223,15 +2226,16 @@ Aggregator::Params InterpreterSelectQuery::getAggregatorParams( }; } -GroupingSetsParamsList InterpreterSelectQuery::getAggregatorGroupingSetsParams( +static GroupingSetsParamsList getAggregatorGroupingSetsParams( + const SelectQueryExpressionAnalyzer & query_analyzer, const Block & header_before_aggregation, const ColumnNumbers & all_keys ) { GroupingSetsParamsList result; - if (query_analyzer->useGroupingSetKey()) + if (query_analyzer.useGroupingSetKey()) { - auto const & aggregation_keys_list = query_analyzer->aggregationKeysList(); + auto const & aggregation_keys_list = query_analyzer.aggregationKeysList(); ColumnNumbersList grouping_sets_with_keys; ColumnNumbersList missing_columns_per_set; @@ -2281,10 +2285,10 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac for (const auto & key : query_analyzer->aggregationKeys()) keys.push_back(header_before_aggregation.getPositionByName(key.name)); - auto aggregator_params = getAggregatorParams(header_before_aggregation, keys, aggregates, overflow_row, settings, + auto aggregator_params = getAggregatorParams(query_ptr, *query_analyzer, *context, header_before_aggregation, keys, aggregates, overflow_row, settings, settings.group_by_two_level_threshold, settings.group_by_two_level_threshold_bytes); - auto grouping_sets_params = getAggregatorGroupingSetsParams(header_before_aggregation, keys); + auto grouping_sets_params = getAggregatorGroupingSetsParams(*query_analyzer, header_before_aggregation, keys); SortDescription group_by_sort_description; @@ -2374,7 +2378,7 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPlan & query_plan, Modific for (const auto & key : query_analyzer->aggregationKeys()) keys.push_back(header_before_transform.getPositionByName(key.name)); - auto params = getAggregatorParams(header_before_transform, keys, query_analyzer->aggregates(), false, settings, 0, 0); + auto params = getAggregatorParams(query_ptr, *query_analyzer, *context, header_before_transform, keys, query_analyzer->aggregates(), false, settings, 0, 0); auto transform_params = std::make_shared(std::move(params), true); QueryPlanStepPtr step; diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index ed6320963ed..3adbcad909c 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -3,7 +3,6 @@ #include #include -#include #include #include #include @@ -15,7 +14,6 @@ #include #include -#include namespace Poco { @@ -34,7 +32,6 @@ using GroupingSetsParamsList = std::vector; struct TreeRewriterResult; using TreeRewriterResultPtr = std::shared_ptr; -using AggregatorParamsPtr = std::unique_ptr; /** Interprets the SELECT query. Returns the stream of blocks with the results of the query before `to_stage` stage. @@ -146,16 +143,6 @@ private: void executeImpl(QueryPlan & query_plan, std::optional prepared_pipe); /// Different stages of query execution. - - Aggregator::Params getAggregatorParams( - const Block & current_data_stream_header, - const ColumnNumbers & keys, - const AggregateDescriptions & aggregates, - bool overflow_row, const Settings & settings, - size_t group_by_two_level_threshold, size_t group_by_two_level_threshold_bytes); - GroupingSetsParamsList getAggregatorGroupingSetsParams( - const Block & header_before_aggregation, - const ColumnNumbers & all_keys); void executeFetchColumns(QueryProcessingStage::Enum processing_stage, QueryPlan & query_plan); void executeWhere(QueryPlan & query_plan, const ActionsDAGPtr & expression, bool remove_filter); void executeAggregation( diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 40316ae4adc..d7d62d07d92 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -133,7 +133,7 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B pipeline.transform([&](OutputPortRawPtrs ports) { assert(streams * grouping_sets_size == ports.size()); - Processors aggregators; + Processors processors; for (size_t i = 0; i < grouping_sets_size; ++i) { Aggregator::Params params_for_set @@ -167,79 +167,85 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B // For each input stream we have `grouping_sets_size` copies, so port index // for transform #j should skip ports of first (j-1) streams. connect(*ports[i + grouping_sets_size * j], aggregation_for_set->getInputs().front()); - aggregators.push_back(aggregation_for_set); + ports[i + grouping_sets_size * j] = &aggregation_for_set->getOutputs().front(); + processors.push_back(aggregation_for_set); } } else { auto aggregation_for_set = std::make_shared(input_header, transform_params_for_set); connect(*ports[i], aggregation_for_set->getInputs().front()); - aggregators.push_back(aggregation_for_set); + ports[i] = &aggregation_for_set->getOutputs().front(); + processors.push_back(aggregation_for_set); } } - return aggregators; - }, false); - if (streams > 1) - { - pipeline.transform([&](OutputPortRawPtrs ports) + if (streams > 1) { - Processors resizes; + OutputPortRawPtrs new_ports; + new_ports.reserve(grouping_sets_size); + for (size_t i = 0; i < grouping_sets_size; ++i) { - auto output_it = ports.begin() + i * streams; - auto resize = std::make_shared((*output_it)->getHeader(), streams, 1); + size_t output_it = i; + auto resize = std::make_shared(ports[output_it]->getHeader(), streams, 1); auto & inputs = resize->getInputs(); - for (auto input_it = inputs.begin(); input_it != inputs.end(); ++output_it, ++input_it) - connect(**output_it, *input_it); - resizes.push_back(resize); + for (auto input_it = inputs.begin(); input_it != inputs.end(); output_it += grouping_sets_size, ++input_it) + connect(*ports[output_it], *input_it); + new_ports.push_back(&resize->getOutputs().front()); + processors.push_back(resize); } - return resizes; - }, false); - } - assert(pipeline.getNumStreams() == grouping_sets_size); - size_t set_counter = 0; - auto output_header = transform_params->getHeader(); - pipeline.addSimpleTransform([&](const Block & header) - { - /// Here we create a DAG which fills missing keys and adds `__grouping_set` column - auto dag = std::make_shared(header.getColumnsWithTypeAndName()); - ActionsDAG::NodeRawConstPtrs index; - index.reserve(output_header.columns() + 1); - - auto grouping_col = ColumnConst::create(ColumnUInt64::create(1, set_counter), 0); - const auto * grouping_node = &dag->addColumn( - {ColumnPtr(std::move(grouping_col)), std::make_shared(), "__grouping_set"}); - - grouping_node = &dag->materializeNode(*grouping_node); - index.push_back(grouping_node); - - size_t missign_column_index = 0; - const auto & missing_columns = grouping_sets_params[set_counter].missing_keys; - - for (size_t i = 0; i < output_header.columns(); ++i) - { - auto & col = output_header.getByPosition(i); - if (missign_column_index < missing_columns.size() && missing_columns[missign_column_index] == i) - { - ++missign_column_index; - auto column = ColumnConst::create(col.column->cloneResized(1), 0); - const auto * node = &dag->addColumn({ColumnPtr(std::move(column)), col.type, col.name}); - node = &dag->materializeNode(*node); - index.push_back(node); - } - else - index.push_back(dag->getIndex()[header.getPositionByName(col.name)]); + ports.swap(new_ports); } - dag->getIndex().swap(index); - auto expression = std::make_shared(dag, settings.getActionsSettings()); - auto transform = std::make_shared(header, expression); + assert(ports.size() == grouping_sets_size); + auto output_header = transform_params->getHeader(); - ++set_counter; - return transform; + for (size_t set_counter = 0; set_counter < grouping_sets_size; ++set_counter) + { + auto & header = ports[set_counter]->getHeader(); + + /// Here we create a DAG which fills missing keys and adds `__grouping_set` column + auto dag = std::make_shared(header.getColumnsWithTypeAndName()); + ActionsDAG::NodeRawConstPtrs index; + index.reserve(output_header.columns() + 1); + + auto grouping_col = ColumnConst::create(ColumnUInt64::create(1, set_counter), 0); + const auto * grouping_node = &dag->addColumn( + {ColumnPtr(std::move(grouping_col)), std::make_shared(), "__grouping_set"}); + + grouping_node = &dag->materializeNode(*grouping_node); + index.push_back(grouping_node); + + size_t missign_column_index = 0; + const auto & missing_columns = grouping_sets_params[set_counter].missing_keys; + + for (size_t i = 0; i < output_header.columns(); ++i) + { + auto & col = output_header.getByPosition(i); + if (missign_column_index < missing_columns.size() && missing_columns[missign_column_index] == i) + { + ++missign_column_index; + auto column = ColumnConst::create(col.column->cloneResized(1), 0); + const auto * node = &dag->addColumn({ColumnPtr(std::move(column)), col.type, col.name}); + node = &dag->materializeNode(*node); + index.push_back(node); + } + else + index.push_back(dag->getIndex()[header.getPositionByName(col.name)]); + } + + dag->getIndex().swap(index); + auto expression = std::make_shared(dag, settings.getActionsSettings()); + auto transform = std::make_shared(header, expression); + + connect(*ports[set_counter], transform->getInputPort()); + processors.emplace_back(std::move(transform)); + } + + return processors; }); aggregating = collector.detachProcessors(0); diff --git a/src/QueryPipeline/Pipe.cpp b/src/QueryPipeline/Pipe.cpp index 480f2046e23..19009d9692a 100644 --- a/src/QueryPipeline/Pipe.cpp +++ b/src/QueryPipeline/Pipe.cpp @@ -786,7 +786,7 @@ void Pipe::setSinks(const Pipe::ProcessorGetterWithStreamKind & getter) header.clear(); } -void Pipe::transform(const Transformer & transformer, bool check_block_structure) +void Pipe::transform(const Transformer & transformer) { if (output_ports.empty()) throw Exception("Cannot transform empty Pipe", ErrorCodes::LOGICAL_ERROR); @@ -852,18 +852,15 @@ void Pipe::transform(const Transformer & transformer, bool check_block_structure throw Exception( "Transformation of Pipe is not valid because processors don't have any disconnected output ports", ErrorCodes::LOGICAL_ERROR); - if (check_block_structure) - { - header = output_ports.front()->getHeader(); - for (size_t i = 1; i < output_ports.size(); ++i) - assertBlocksHaveEqualStructure(header, output_ports[i]->getHeader(), "Pipe"); + header = output_ports.front()->getHeader(); + for (size_t i = 1; i < output_ports.size(); ++i) + assertBlocksHaveEqualStructure(header, output_ports[i]->getHeader(), "Pipe"); - if (totals_port) - assertBlocksHaveEqualStructure(header, totals_port->getHeader(), "Pipes"); + if (totals_port) + assertBlocksHaveEqualStructure(header, totals_port->getHeader(), "Pipes"); - if (extremes_port) - assertBlocksHaveEqualStructure(header, extremes_port->getHeader(), "Pipes"); - } + if (extremes_port) + assertBlocksHaveEqualStructure(header, extremes_port->getHeader(), "Pipes"); if (collected_processors) { diff --git a/src/QueryPipeline/Pipe.h b/src/QueryPipeline/Pipe.h index 0fce25a3fae..6f85b7a6a88 100644 --- a/src/QueryPipeline/Pipe.h +++ b/src/QueryPipeline/Pipe.h @@ -89,7 +89,7 @@ public: using Transformer = std::function; /// Transform Pipe in general way. - void transform(const Transformer & transformer, bool check_block_structure = true); + void transform(const Transformer & transformer); /// Unite several pipes together. They should have same header. static Pipe unitePipes(Pipes pipes); diff --git a/src/QueryPipeline/QueryPipelineBuilder.cpp b/src/QueryPipeline/QueryPipelineBuilder.cpp index aaf8837a5d7..5e074861110 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.cpp +++ b/src/QueryPipeline/QueryPipelineBuilder.cpp @@ -159,10 +159,10 @@ void QueryPipelineBuilder::addChain(Chain chain) pipe.addChains(std::move(chains)); } -void QueryPipelineBuilder::transform(const Transformer & transformer, bool check_block_structure) +void QueryPipelineBuilder::transform(const Transformer & transformer) { checkInitializedAndNotCompleted(); - pipe.transform(transformer, check_block_structure); + pipe.transform(transformer); } void QueryPipelineBuilder::setSinks(const Pipe::ProcessorGetterWithStreamKind & getter) diff --git a/src/QueryPipeline/QueryPipelineBuilder.h b/src/QueryPipeline/QueryPipelineBuilder.h index 86b8a1323f1..ad25985ab48 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.h +++ b/src/QueryPipeline/QueryPipelineBuilder.h @@ -63,7 +63,7 @@ public: using Transformer = std::function; /// Transform pipeline in general way. - void transform(const Transformer & transformer, bool check_block_structure = true); + void transform(const Transformer & transformer); /// Add TotalsHavingTransform. Resize pipeline to single input. Adds totals port. void addTotalsHavingTransform(ProcessorPtr transform); From 7232f47c682dc7dc4f30cc3ac978ba5bc8d6c5c1 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 11 May 2022 13:54:26 +0200 Subject: [PATCH 107/171] Fix Bug 37114 - ilike on FixedString(N) columns produces wrong results The main fix is in MatchImpl.h where the "case_insensitive" parameter is added to Regexps::get(). Also made "case_insensitive" a non-default template parameter to reduce the risk of future bugs. The remainder of this commit are minor random code improvements. resoves #37114 --- src/Common/ObjectPool.h | 2 +- src/Functions/FunctionsStringArray.h | 4 +-- src/Functions/MatchImpl.h | 29 +++++++++---------- src/Functions/Regexps.h | 15 ++++------ src/Functions/extract.cpp | 2 +- src/Functions/extractAllGroups.h | 2 +- src/Functions/extractGroups.cpp | 2 +- .../02293_ilike_on_fixed_strings.reference | 2 ++ .../02293_ilike_on_fixed_strings.sql | 10 +++++++ 9 files changed, 38 insertions(+), 30 deletions(-) create mode 100644 tests/queries/0_stateless/02293_ilike_on_fixed_strings.reference create mode 100644 tests/queries/0_stateless/02293_ilike_on_fixed_strings.sql diff --git a/src/Common/ObjectPool.h b/src/Common/ObjectPool.h index ef07b8eed1b..801a37d0dfb 100644 --- a/src/Common/ObjectPool.h +++ b/src/Common/ObjectPool.h @@ -94,7 +94,7 @@ public: template Pointer get(const Key & key, Factory && f) { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); auto it = container.find(key); if (container.end() == it) diff --git a/src/Functions/FunctionsStringArray.h b/src/Functions/FunctionsStringArray.h index 6b3adf46ff5..2680816670f 100644 --- a/src/Functions/FunctionsStringArray.h +++ b/src/Functions/FunctionsStringArray.h @@ -477,7 +477,7 @@ public: ErrorCodes::ILLEGAL_COLUMN); if (!col->getValue().empty()) - re = Regexps::get(col->getValue()); + re = Regexps::get(col->getValue()); } @@ -560,7 +560,7 @@ public: + " of first argument of function " + getName() + ". Must be constant string.", ErrorCodes::ILLEGAL_COLUMN); - re = Regexps::get(col->getValue()); + re = Regexps::get(col->getValue()); capture = re->getNumberOfSubpatterns() > 0 ? 1 : 0; matches.resize(capture + 1); diff --git a/src/Functions/MatchImpl.h b/src/Functions/MatchImpl.h index 08ac47e692f..026b38b997b 100644 --- a/src/Functions/MatchImpl.h +++ b/src/Functions/MatchImpl.h @@ -24,12 +24,11 @@ namespace ErrorCodes /// Is the [I]LIKE expression reduced to finding a substring in a string? static inline bool likePatternIsStrstr(const String & pattern, String & res) { - res = ""; - if (pattern.size() < 2 || pattern.front() != '%' || pattern.back() != '%') return false; - res.reserve(pattern.size() * 2); + res = ""; + res.reserve(pattern.size() - 2); const char * pos = pattern.data(); const char * end = pos + pattern.size(); @@ -81,7 +80,7 @@ struct MatchImpl static void vectorConstant( const ColumnString::Chars & data, const ColumnString::Offsets & offsets, - const std::string & pattern, + const String & pattern, const ColumnPtr & start_pos, PaddedPODArray & res) { @@ -92,14 +91,13 @@ struct MatchImpl if (offsets.empty()) return; - String strstr_pattern; - /// A simple case where the [I]LIKE expression reduces to finding a substring in a string + String strstr_pattern; if (like && likePatternIsStrstr(pattern, strstr_pattern)) { - const UInt8 * begin = data.data(); + const UInt8 * const begin = data.data(); + const UInt8 * const end = data.data() + data.size(); const UInt8 * pos = begin; - const UInt8 * end = pos + data.size(); /// The current index in the array of strings. size_t i = 0; @@ -137,7 +135,7 @@ struct MatchImpl auto regexp = Regexps::get(pattern); - std::string required_substring; + String required_substring; bool is_trivial; bool required_substring_is_prefix; /// for `anchored` execution of the regexp. @@ -172,9 +170,9 @@ struct MatchImpl { /// NOTE This almost matches with the case of LikePatternIsStrstr. - const UInt8 * begin = data.data(); + const UInt8 * const begin = data.data(); + const UInt8 * const end = data.begin() + data.size(); const UInt8 * pos = begin; - const UInt8 * end = pos + data.size(); /// The current index in the array of strings. size_t i = 0; @@ -230,6 +228,7 @@ struct MatchImpl ++i; } + /// Tail, in which there can be no substring. if (i < res.size()) memset(&res[i], revert, (res.size() - i) * sizeof(res[0])); } @@ -238,14 +237,14 @@ struct MatchImpl /// Very carefully crafted copy-paste. static void vectorFixedConstant( - const ColumnString::Chars & data, size_t n, const std::string & pattern, + const ColumnString::Chars & data, size_t n, const String & pattern, PaddedPODArray & res) { if (data.empty()) return; - String strstr_pattern; /// A simple case where the LIKE expression reduces to finding a substring in a string + String strstr_pattern; if (like && likePatternIsStrstr(pattern, strstr_pattern)) { const UInt8 * begin = data.data(); @@ -291,9 +290,9 @@ struct MatchImpl { size_t size = data.size() / n; - auto regexp = Regexps::get(pattern); + auto regexp = Regexps::get(pattern); - std::string required_substring; + String required_substring; bool is_trivial; bool required_substring_is_prefix; /// for `anchored` execution of the regexp. diff --git a/src/Functions/Regexps.h b/src/Functions/Regexps.h index 8e6d30c8e14..9a1938a3f32 100644 --- a/src/Functions/Regexps.h +++ b/src/Functions/Regexps.h @@ -44,23 +44,20 @@ namespace Regexps template inline Regexp createRegexp(const std::string & pattern, int flags) { - return {pattern, flags}; - } - - template <> - inline Regexp createRegexp(const std::string & pattern, int flags) - { - return {likePatternToRegexp(pattern), flags}; + if constexpr (like) + return {likePatternToRegexp(pattern), flags}; + else + return {pattern, flags}; } /** Returns holder of an object from Pool. * You must hold the ownership while using the object. * In destructor, it returns the object back to the Pool for further reuse. */ - template + template inline Pool::Pointer get(const std::string & pattern) { - /// C++11 has thread-safe function-local static on most modern compilers. + /// the Singleton is thread-safe in C++11 static Pool known_regexps; /// Different variables for different pattern parameters. return known_regexps.get(pattern, [&pattern] diff --git a/src/Functions/extract.cpp b/src/Functions/extract.cpp index 0296602d205..5b138d19747 100644 --- a/src/Functions/extract.cpp +++ b/src/Functions/extract.cpp @@ -21,7 +21,7 @@ struct ExtractImpl res_data.reserve(data.size() / 5); res_offsets.resize(offsets.size()); - const auto & regexp = Regexps::get(pattern); + const auto & regexp = Regexps::get(pattern); unsigned capture = regexp->getNumberOfSubpatterns() > 0 ? 1 : 0; OptimizedRegularExpression::MatchVec matches; diff --git a/src/Functions/extractAllGroups.h b/src/Functions/extractAllGroups.h index 057dedab6e4..e6d31e00616 100644 --- a/src/Functions/extractAllGroups.h +++ b/src/Functions/extractAllGroups.h @@ -95,7 +95,7 @@ public: throw Exception("Length of 'needle' argument must be greater than 0.", ErrorCodes::BAD_ARGUMENTS); using StringPiece = typename Regexps::Regexp::StringPieceType; - auto holder = Regexps::get(needle); + auto holder = Regexps::get(needle); const auto & regexp = holder->getRE2(); if (!regexp) diff --git a/src/Functions/extractGroups.cpp b/src/Functions/extractGroups.cpp index 2286951bb8f..c5b958ec345 100644 --- a/src/Functions/extractGroups.cpp +++ b/src/Functions/extractGroups.cpp @@ -63,7 +63,7 @@ public: if (needle.empty()) throw Exception(getName() + " length of 'needle' argument must be greater than 0.", ErrorCodes::BAD_ARGUMENTS); - auto regexp = Regexps::get(needle); + auto regexp = Regexps::get(needle); const auto & re2 = regexp->getRE2(); if (!re2) diff --git a/tests/queries/0_stateless/02293_ilike_on_fixed_strings.reference b/tests/queries/0_stateless/02293_ilike_on_fixed_strings.reference new file mode 100644 index 00000000000..5489ab3d7ce --- /dev/null +++ b/tests/queries/0_stateless/02293_ilike_on_fixed_strings.reference @@ -0,0 +1,2 @@ +AA 0 1 +Aa 1 1 diff --git a/tests/queries/0_stateless/02293_ilike_on_fixed_strings.sql b/tests/queries/0_stateless/02293_ilike_on_fixed_strings.sql new file mode 100644 index 00000000000..3838e372e24 --- /dev/null +++ b/tests/queries/0_stateless/02293_ilike_on_fixed_strings.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab (col FixedString(2)) engine = MergeTree() ORDER BY col; + +INSERT INTO tab VALUES ('AA') ('Aa'); + +SELECT col, col LIKE '%a', col ILIKE '%a' FROM tab WHERE col = 'AA'; +SELECT col, col LIKE '%a', col ILIKE '%a' FROM tab WHERE col = 'Aa'; + +DROP TABLE IF EXISTS tab; From c84e0d13fa06d4e91c736c30d9ed78f3a6453fca Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 11 May 2022 14:21:44 +0200 Subject: [PATCH 108/171] Fix test --- src/IO/Progress.cpp | 7 ++----- src/Storages/StorageFile.cpp | 13 ++++++++++++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/IO/Progress.cpp b/src/IO/Progress.cpp index 29cf6c52d13..2993319fff0 100644 --- a/src/IO/Progress.cpp +++ b/src/IO/Progress.cpp @@ -69,11 +69,8 @@ bool Progress::incrementPiecewiseAtomically(const Progress & rhs) read_rows += rhs.read_rows; read_bytes += rhs.read_bytes; - if (rhs.total_rows_to_read) - total_rows_to_read.store(rhs.total_rows_to_read); - - if (rhs.total_bytes_to_read) - total_bytes_to_read.store(rhs.total_bytes_to_read); + total_rows_to_read += rhs.total_rows_to_read; + total_bytes_to_read += rhs.total_bytes_to_read; written_rows += rhs.written_rows; written_bytes += rhs.written_bytes; diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 7c71c831559..5b191b37f5e 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -582,7 +582,17 @@ public: total_rows_approx_accumulated += total_rows_approx; ++total_rows_count_times; total_rows_approx = total_rows_approx_accumulated / total_rows_count_times; - setTotalRowsApprox(total_rows_approx); + + /// We need to add diff, because total_rows_approx is incremental value. + /// It would be more correct to send total_rows_approx as is (not a diff), + /// but incrementation of total_rows_to_read does not allow that. + /// A new field can be introduces for that to be sent to client, but it does not worth it. + if (total_rows_approx > total_rows_approx_prev) + { + size_t diff = total_rows_approx - total_rows_approx_prev; + addTotalRowsApprox(diff); + total_rows_approx_prev = total_rows_approx; + } } return chunk; } @@ -622,6 +632,7 @@ private: UInt64 total_rows_approx_accumulated = 0; size_t total_rows_count_times = 0; + UInt64 total_rows_approx_prev = 0; }; From 118dd64c93a9119a539c706d59d0879dc830d12b Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Wed, 11 May 2022 14:45:10 +0200 Subject: [PATCH 109/171] Only enable MPL2-licensed code in eigen library --- contrib/eigen-cmake/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/eigen-cmake/CMakeLists.txt b/contrib/eigen-cmake/CMakeLists.txt index c8548b7fbc1..6bdf3ab7c35 100644 --- a/contrib/eigen-cmake/CMakeLists.txt +++ b/contrib/eigen-cmake/CMakeLists.txt @@ -16,5 +16,8 @@ if (ENABLE_MKL) endif() endif() +# Only include MPL2 code from Eigen library +target_compile_definitions(_eigen INTERFACE EIGEN_MPL2_ONLY) + target_include_directories (_eigen SYSTEM INTERFACE ${EIGEN_LIBRARY_DIR}) add_library(ch_contrib::eigen ALIAS _eigen) From 3533cd770d71ba2af1e4ba33c283af2838905e46 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Wed, 11 May 2022 14:46:06 +0200 Subject: [PATCH 110/171] Reserve result arrays --- src/Functions/array/arrayDistance.cpp | 2 ++ src/Functions/array/arrayNorm.cpp | 1 + 2 files changed, 3 insertions(+) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index 8996236058a..26cfd0c3244 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -23,6 +23,7 @@ struct LpDistance static void compute(const Eigen::MatrixX & left, const Eigen::MatrixX & right, PaddedPODArray & array) { auto & norms = (left - right).colwise().template lpNorm(); + array.reserve(norms.size()); // array.insert() failed to work with Eigen iterators for (auto n : norms) array.push_back(n); @@ -45,6 +46,7 @@ struct CosineDistance auto & ny = right.colwise().norm(); auto & nm = nx.cwiseProduct(ny).cwiseInverse(); auto & dist = 1.0 - prod.cwiseProduct(nm).array(); + array.reserve(dist.size()); for (auto d : dist) array.push_back(d); } diff --git a/src/Functions/array/arrayNorm.cpp b/src/Functions/array/arrayNorm.cpp index aefa219a720..b267b1d726e 100644 --- a/src/Functions/array/arrayNorm.cpp +++ b/src/Functions/array/arrayNorm.cpp @@ -21,6 +21,7 @@ struct LpNorm template static void compute(const std::vector> & vec, PaddedPODArray & array) { + array.reserve(vec.size()); for (const auto & v : vec) { array.push_back(v.template lpNorm()); From c5e84950d6b7b2a2a1e1f12d29e72363c1ee616d Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 11 May 2022 14:57:51 +0200 Subject: [PATCH 111/171] Update test, add descriptions to profile events --- src/Common/ProfileEvents.cpp | 14 +++++++------- .../0_stateless/02226_s3_with_cache.sql | 18 +++++++++--------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index e7f40bfe9f4..5a3982f1ec1 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -277,14 +277,14 @@ M(RemoteFSSeeksWithReset, "Number of seeks which lead to a new connection") \ M(RemoteFSBuffers, "Number of buffers created for asynchronous reading from remote filesystem") \ \ - M(ThreadpoolReaderTaskMicroseconds, "") \ - M(ThreadpoolReaderReadBytes, "") \ + M(ThreadpoolReaderTaskMicroseconds, "Time spent getting the data in asynchronous reading") \ + M(ThreadpoolReaderReadBytes, "Bytes read from a threadpool task in asynchronous reading") \ \ - M(FileSegmentWaitReadBufferMicroseconds, "") \ - M(FileSegmentReadMicroseconds, "") \ - M(FileSegmentCacheWriteMicroseconds, "") \ - M(FileSegmentPredownloadMicroseconds, "") \ - M(FileSegmentUsedBytes, "") \ + M(FileSegmentWaitReadBufferMicroseconds, "Metric per file segment. Time spend waiting for internal read buffer (includes cache waiting)") \ + M(FileSegmentReadMicroseconds, "Metric per file segment. Time spend reading from file") \ + M(FileSegmentCacheWriteMicroseconds, "Metric per file segment. Time spend writing data to cache") \ + M(FileSegmentPredownloadMicroseconds, "Metric per file segment. Time spent predownloading data to cache (predownloading - finishing file segment download (after someone who failed to do that) up to the point current thread was requested to do)") \ + M(FileSegmentUsedBytes, "Metric per file segment. How many bytes were actually used from current file segment") \ \ M(ReadBufferSeekCancelConnection, "Number of seeks which lead to new connection (s3, http)") \ \ diff --git a/tests/queries/0_stateless/02226_s3_with_cache.sql b/tests/queries/0_stateless/02226_s3_with_cache.sql index d470f2ef140..e62e63b7f97 100644 --- a/tests/queries/0_stateless/02226_s3_with_cache.sql +++ b/tests/queries/0_stateless/02226_s3_with_cache.sql @@ -13,9 +13,9 @@ SELECT 1, * FROM test LIMIT 10 FORMAT Null; SYSTEM FLUSH LOGS; SELECT query, - ProfileEvents['RemoteFSReadBytes'] > 0 as remote_fs_read, - ProfileEvents['RemoteFSCacheReadBytes'] > 0 as remote_fs_cache_read, - ProfileEvents['RemoteFSCacheDownloadBytes'] > 0 as remote_fs_read_and_download + ProfileEvents['CachedReadBufferReadFromSourceBytes'] > 0 as remote_fs_read, + ProfileEvents['CachedReadBufferReadFromCacheBytes'] > 0 as remote_fs_cache_read, + ProfileEvents['CachedReadBufferCacheWriteBytes'] > 0 as remote_fs_read_and_download FROM system.query_log WHERE query LIKE 'SELECT 1, * FROM test LIMIT%' AND type = 'QueryFinish' @@ -29,9 +29,9 @@ SELECT 2, * FROM test LIMIT 10 FORMAT Null; SYSTEM FLUSH LOGS; SELECT query, - ProfileEvents['RemoteFSReadBytes'] > 0 as remote_fs_read, - ProfileEvents['RemoteFSCacheReadBytes'] > 0 as remote_fs_cache_read, - ProfileEvents['RemoteFSCacheDownloadBytes'] > 0 as remote_fs_read_and_download + ProfileEvents['CachedReadBufferReadFromSourceBytes'] > 0 as remote_fs_read, + ProfileEvents['CachedReadBufferReadFromCacheBytes'] > 0 as remote_fs_cache_read, + ProfileEvents['CachedReadBufferCacheWriteBytes'] > 0 as remote_fs_read_and_download FROM system.query_log WHERE query LIKE 'SELECT 2, * FROM test LIMIT%' AND type = 'QueryFinish' @@ -56,9 +56,9 @@ SELECT 3, * FROM test LIMIT 10 FORMAT Null; SYSTEM FLUSH LOGS; SELECT query, - ProfileEvents['RemoteFSReadBytes'] > 0 as remote_fs_read, - ProfileEvents['RemoteFSCacheReadBytes'] > 0 as remote_fs_cache_read, - ProfileEvents['RemoteFSCacheDownloadBytes'] > 0 as remote_fs_read_and_download + ProfileEvents['CachedReadBufferReadFromSourceBytes'] > 0 as remote_fs_read, + ProfileEvents['CachedReadBufferReadFromCacheBytes'] > 0 as remote_fs_cache_read, + ProfileEvents['CachedReadBufferCacheWriteBytes'] > 0 as remote_fs_read_and_download FROM system.query_log WHERE query LIKE 'SELECT 3, * FROM test LIMIT%' AND type = 'QueryFinish' From 5e6ebe7bbea01c5d50a6076ca90822ae129b28a6 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 11 May 2022 15:20:08 +0200 Subject: [PATCH 112/171] Add exception --- src/Core/Block.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index f5764262c66..a21c96abfdb 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -498,6 +498,15 @@ Block Block::cloneWithColumns(MutableColumns && columns) const Block res; size_t num_columns = data.size(); + + if (num_columns != columns.size()) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Cannot clone block with columns because block has {} columns, but {} columns given", + num_columns, columns.size()); + } + res.reserve(num_columns); for (size_t i = 0; i < num_columns; ++i) @@ -514,8 +523,12 @@ Block Block::cloneWithColumns(const Columns & columns) const size_t num_columns = data.size(); if (num_columns != columns.size()) - throw Exception("Cannot clone block with columns because block has " + toString(num_columns) + " columns, " - "but " + toString(columns.size()) + " columns given.", ErrorCodes::LOGICAL_ERROR); + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Cannot clone block with columns because block has {} columns, but {} columns given", + num_columns, columns.size()); + } res.reserve(num_columns); From dc1edbe22f2d75f71535d5899c91566cf703f60e Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 11 May 2022 15:41:33 +0200 Subject: [PATCH 113/171] Fix --- src/Disks/IO/ReadBufferFromRemoteFSGather.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp index 09271c4e7f8..08707965e2f 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp @@ -112,7 +112,7 @@ ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather( && (!IFileCache::isReadOnly() || settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache); assert(query_id.empty() - || CurrentThread::isInitialized() && CurrentThread::get().getQueryContext() != nullptr); + || CurrentThread::isInitialized() && CurrentThread::get().getQueryContext().get() != nullptr); } From 567c9e6226b5e2932cca1e1d02469fd257065695 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 11 May 2022 15:55:12 +0200 Subject: [PATCH 114/171] Update 02286_drop_filesystem_cache.reference --- tests/queries/0_stateless/02286_drop_filesystem_cache.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02286_drop_filesystem_cache.reference b/tests/queries/0_stateless/02286_drop_filesystem_cache.reference index 65de9ee04ab..30026f943a1 100644 --- a/tests/queries/0_stateless/02286_drop_filesystem_cache.reference +++ b/tests/queries/0_stateless/02286_drop_filesystem_cache.reference @@ -27,4 +27,4 @@ SELECT count() FROM system.filesystem_cache; 3 SYSTEM DROP FILESYSTEM CACHE './s3_cache/'; SELECT count() FROM system.filesystem_cache; -2 \ No newline at end of file +2 From 17608b3d93b1ee3ff0565b597d5f89ff81cf0b2d Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 11 May 2022 16:18:41 +0000 Subject: [PATCH 115/171] Update documentation and defaults for memory overcommit --- .../settings.md | 10 +++++ .../operations/settings/memory-overcommit.md | 37 +++++++++++++++++++ docs/en/operations/settings/settings.md | 26 +++++++++++++ src/Core/Settings.h | 8 ++-- src/Interpreters/ProcessList.cpp | 4 +- src/Storages/MergeTree/MergeList.cpp | 2 +- .../test_global_overcommit_tracker/test.py | 4 +- .../0_stateless/02104_overcommit_memory.sh | 4 +- 8 files changed, 84 insertions(+), 11 deletions(-) create mode 100644 docs/en/operations/settings/memory-overcommit.md diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index f235fba84f7..fd5c2a187b5 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -1745,3 +1745,13 @@ Possible values: - Positive integer. Default value: `10000`. + +## global_memory_usage_overcommit_max_wait_microseconds {#global_memory_usage_overcommit_max_wait_microseconds} + +Sets maximum waiting time for global overcommit tracker. + +Possible values: + +- Positive integer. + +Default value: `200`. diff --git a/docs/en/operations/settings/memory-overcommit.md b/docs/en/operations/settings/memory-overcommit.md new file mode 100644 index 00000000000..b0c2e3bd35e --- /dev/null +++ b/docs/en/operations/settings/memory-overcommit.md @@ -0,0 +1,37 @@ +# Memory overcommit + +Memory overcommit is an experimental technique intended to allow to set more flexible memory limits for queries. + +The idea of this technique is to introduce settings which can represent guaranteed amount of memory a query can use. +When memory overcommit is enabled and the memory limit is reached ClickHouse will select the most overcommitted query and try to free memory by killing this query. + +When memory limit is reached any query will wait some time during atempt to allocate new memory. +If timeout is passed and memory is freed, the query continues execution. +Otherwise an exception will be thrown and the query is killed. + +Selection of query to stop or kill is performed by either global or user overcommit trackers depending on what memory limit is reached. +If overcommit tracker can't choose query to stop, MEMORY_LIMIT_EXCEEDED exception is thrown. + +## User overcommit tracker + +User overcommit tracker finds a query with the biggest overcommit ratio in the user's query list. +Overcommit ratio for a query is computed as number of allocated bytes divided by value of `memory_overcommit_ratio_denominator` setting. + +If `memory_overcommit_ratio_denominator` for the query is equals to zero, overcommit tracker won't choose this query. + +Waiting timeout is set by `memory_usage_overcommit_max_wait_microseconds` setting. + +**Example** + +```sql +SELECT number FROM numbers(1000) GROUP BY number SETTINGS max_guaranteed_memory_usage=4000, memory_usage_overcommit_max_wait_microseconds=500 +``` + +## Global overcommit tracker + +Global overcommit tracker finds a query with the biggest overcommit ratio in the list of all queries. +In this case overcommit ratio is computed as number of allocated bytes divided by value of `memory_overcommit_ratio_denominator_for_user` setting. + +If `memory_overcommit_ratio_denominator_for_user` for the query is equals to zero, overcommit tracker won't choose this query. + +Waiting timeout is set by `global_memory_usage_overcommit_max_wait_microseconds` parameter in the configuration file. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 8f2b9bc86fc..76fbc5f239d 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -4263,3 +4263,29 @@ Possible values: - 1 — Enabled. Default value: 1. + +## memory_overcommit_ratio_denominator + +It represents soft memory limit in case when hard limit is reached on user level. +This value is used to compute overcommit ratio for the query. +Zero means skip the query. +Read more about [memory overcommit](memory-overcommit.md). + +Default value: `1GiB`. + +## memory_usage_overcommit_max_wait_microseconds + +Maximum time thread will wait for memory to be freed in the case of memory overcommit on a user level. +If the timeout is reached and memory is not freed, an exception is thrown. +Read more about [memory overcommit](memory-overcommit.md). + +Default value: `200`. + +## memory_overcommit_ratio_denominator_for_user + +It represents soft memory limit in case when hard limit is reached on global level. +This value is used to compute overcommit ratio for the query. +Zero means skip the query. +Read more about [memory overcommit](memory-overcommit.md). + +Default value: `1GiB`. diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 76d2dca117b..993e7b759b0 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -22,7 +22,7 @@ namespace DB { class IColumn; -static constexpr UInt64 operator""_Gb(unsigned long long value) +static constexpr UInt64 operator""_GiB(unsigned long long value) { return value * 1024 * 1024 * 1024; } @@ -360,14 +360,14 @@ static constexpr UInt64 operator""_Gb(unsigned long long value) M(OverflowMode, distinct_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ \ M(UInt64, max_memory_usage, 0, "Maximum memory usage for processing of single query. Zero means unlimited.", 0) \ - M(UInt64, max_guaranteed_memory_usage, 10_Gb, "Maximum guaranteed memory usage for processing of single query. It represents soft limit. Zero means unlimited.", 0) \ + M(UInt64, memory_overcommit_ratio_denominator, 1_GiB, "It represents soft memory limit on the user level. This value is used to compute query overcommit ratio.", 0) \ M(UInt64, max_memory_usage_for_user, 0, "Maximum memory usage for processing all concurrently running queries for the user. Zero means unlimited.", 0) \ - M(UInt64, max_guaranteed_memory_usage_for_user, 10_Gb, "Maximum guaranteed memory usage for processing all concurrently running queries for the user. It represents soft limit. Zero means unlimited.", 0) \ + M(UInt64, memory_overcommit_ratio_denominator_for_user, 1_GiB, "It represents soft memory limit on the global level. This value is used to compute query overcommit ratio.", 0) \ M(UInt64, max_untracked_memory, (4 * 1024 * 1024), "Small allocations and deallocations are grouped in thread local variable and tracked or profiled only when amount (in absolute value) becomes larger than specified value. If the value is higher than 'memory_profiler_step' it will be effectively lowered to 'memory_profiler_step'.", 0) \ M(UInt64, memory_profiler_step, (4 * 1024 * 1024), "Whenever query memory usage becomes larger than every next step in number of bytes the memory profiler will collect the allocating stack trace. Zero means disabled memory profiler. Values lower than a few megabytes will slow down query processing.", 0) \ M(Float, memory_profiler_sample_probability, 0., "Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. The probability is for every alloc/free regardless to the size of the allocation. Note that sampling happens only when the amount of untracked memory exceeds 'max_untracked_memory'. You may want to set 'max_untracked_memory' to 0 for extra fine grained sampling.", 0) \ \ - M(UInt64, memory_usage_overcommit_max_wait_microseconds, 0, "Maximum time thread will wait for memory to be freed in the case of memory overcommit. If timeout is reached and memory is not freed, exception is thrown", 0) \ + M(UInt64, memory_usage_overcommit_max_wait_microseconds, 200, "Maximum time thread will wait for memory to be freed in the case of memory overcommit on user level. If timeout is reached and memory is not freed, exception is thrown.", 0) \ \ M(UInt64, max_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for a query. Zero means unlimited.", 0) \ M(UInt64, max_network_bytes, 0, "The maximum number of bytes (compressed) to receive or transmit over the network for execution of the query.", 0) \ diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index ac59d2c7235..6c101143234 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -212,7 +212,7 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as /// Set query-level memory trackers thread_group->memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage); - thread_group->memory_tracker.setSoftLimit(settings.max_guaranteed_memory_usage); + thread_group->memory_tracker.setSoftLimit(settings.memory_overcommit_ratio_denominator); if (query_context->hasTraceCollector()) { @@ -242,7 +242,7 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as /// Track memory usage for all simultaneously running queries from single user. user_process_list.user_memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage_for_user); - user_process_list.user_memory_tracker.setSoftLimit(settings.max_guaranteed_memory_usage_for_user); + user_process_list.user_memory_tracker.setSoftLimit(settings.memory_overcommit_ratio_denominator_for_user); user_process_list.user_memory_tracker.setDescription("(for user)"); user_process_list.user_overcommit_tracker.setMaxWaitTime(settings.memory_usage_overcommit_max_wait_microseconds); diff --git a/src/Storages/MergeTree/MergeList.cpp b/src/Storages/MergeTree/MergeList.cpp index 11d0bc8c565..8722ddc5a82 100644 --- a/src/Storages/MergeTree/MergeList.cpp +++ b/src/Storages/MergeTree/MergeList.cpp @@ -87,7 +87,7 @@ MergeListElement::MergeListElement( /// thread_group::memory_tracker, but MemoryTrackerThreadSwitcher will reset parent). memory_tracker.setProfilerStep(settings.memory_profiler_step); memory_tracker.setSampleProbability(settings.memory_profiler_sample_probability); - memory_tracker.setSoftLimit(settings.max_guaranteed_memory_usage); + memory_tracker.setSoftLimit(settings.memory_overcommit_ratio_denominator); if (settings.memory_tracker_fault_probability) memory_tracker.setFaultProbability(settings.memory_tracker_fault_probability); diff --git a/tests/integration/test_global_overcommit_tracker/test.py b/tests/integration/test_global_overcommit_tracker/test.py index cacc447be1a..d3d56e82f38 100644 --- a/tests/integration/test_global_overcommit_tracker/test.py +++ b/tests/integration/test_global_overcommit_tracker/test.py @@ -18,8 +18,8 @@ def start_cluster(): cluster.shutdown() -TEST_QUERY_A = "SELECT number FROM numbers(1000) GROUP BY number SETTINGS max_guaranteed_memory_usage_for_user=1" -TEST_QUERY_B = "SELECT number FROM numbers(1000) GROUP BY number SETTINGS max_guaranteed_memory_usage_for_user=2" +TEST_QUERY_A = "SELECT number FROM numbers(1000) GROUP BY number SETTINGS memory_overcommit_ratio_denominator_for_user=1" +TEST_QUERY_B = "SELECT number FROM numbers(1000) GROUP BY number SETTINGS memory_overcommit_ratio_denominator_for_user=2" def test_overcommited_is_killed(): diff --git a/tests/queries/0_stateless/02104_overcommit_memory.sh b/tests/queries/0_stateless/02104_overcommit_memory.sh index 7fdf74a30bf..f2016dbc0c1 100755 --- a/tests/queries/0_stateless/02104_overcommit_memory.sh +++ b/tests/queries/0_stateless/02104_overcommit_memory.sh @@ -11,13 +11,13 @@ $CLICKHOUSE_CLIENT -q 'GRANT ALL ON *.* TO u02104' function overcommited() { - $CLICKHOUSE_CLIENT -u u02104 -q 'SELECT number FROM numbers(130000) GROUP BY number SETTINGS max_guaranteed_memory_usage=1,memory_usage_overcommit_max_wait_microseconds=500' 2>&1 \ + $CLICKHOUSE_CLIENT -u u02104 -q 'SELECT number FROM numbers(130000) GROUP BY number SETTINGS memory_overcommit_ratio_denominator=1,memory_usage_overcommit_max_wait_microseconds=500' 2>&1 \ | grep -F -q "MEMORY_LIMIT_EXCEEDED" && echo "OVERCOMMITED WITH USER LIMIT IS KILLED" } function expect_execution() { - $CLICKHOUSE_CLIENT -u u02104 -q 'SELECT number FROM numbers(130000) GROUP BY number SETTINGS max_memory_usage_for_user=5000000,max_guaranteed_memory_usage=2,memory_usage_overcommit_max_wait_microseconds=500' >/dev/null 2>/dev/null + $CLICKHOUSE_CLIENT -u u02104 -q 'SELECT number FROM numbers(130000) GROUP BY number SETTINGS max_memory_usage_for_user=5000000,memory_overcommit_ratio_denominator=2,memory_usage_overcommit_max_wait_microseconds=500' >/dev/null 2>/dev/null } export -f overcommited From ec4955f157a9a6d1b77f7906d240d144459f932e Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 11 May 2022 18:32:31 +0200 Subject: [PATCH 116/171] Update ReadSettings.h --- src/IO/ReadSettings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/ReadSettings.h b/src/IO/ReadSettings.h index 92346615a7a..f4c0ee43d1c 100644 --- a/src/IO/ReadSettings.h +++ b/src/IO/ReadSettings.h @@ -57,7 +57,7 @@ struct ReadSettings /// Method to use reading from local filesystem. LocalFSReadMethod local_fs_method = LocalFSReadMethod::pread; /// Method to use reading from remote filesystem. - RemoteFSReadMethod remote_fs_method = RemoteFSReadMethod::read; + RemoteFSReadMethod remote_fs_method = RemoteFSReadMethod::threadpool; size_t local_fs_buffer_size = DBMS_DEFAULT_BUFFER_SIZE; size_t remote_fs_buffer_size = DBMS_DEFAULT_BUFFER_SIZE; From 89d6409ed10d043fe110f3f5c5753029c6c06e9e Mon Sep 17 00:00:00 2001 From: ni1l <94829417+ni1l@users.noreply.github.com> Date: Thu, 12 May 2022 00:33:31 +0800 Subject: [PATCH 117/171] Sync the arithmetic functions and array functions in the Chinese document with the English document --- .../functions/arithmetic-functions.md | 74 + .../functions/array-functions.md | 1291 +++++++++++++++-- 2 files changed, 1235 insertions(+), 130 deletions(-) diff --git a/docs/zh/sql-reference/functions/arithmetic-functions.md b/docs/zh/sql-reference/functions/arithmetic-functions.md index 7af0df084f3..15bec0d2107 100644 --- a/docs/zh/sql-reference/functions/arithmetic-functions.md +++ b/docs/zh/sql-reference/functions/arithmetic-functions.md @@ -80,4 +80,78 @@ SELECT toTypeName(0), toTypeName(0 + 0), toTypeName(0 + 0 + 0), toTypeName(0 + 0 返回数值的最小公倍数。 除以零或将最小负数除以-1时抛出异常。 +## max2 {#max2} + +比较两个值并返回最大值。返回值转换为[Float64](../../sql-reference/data-types/float.md)。 + +**语法** + +```sql +max2(value1, value2) +``` + +**参数** + +- `value1` — 第一个值,类型为[Int/UInt](../../sql-reference/data-types/int-uint.md)或[Float](../../sql-reference/data-types/float.md)。 +- `value2` — 第二个值,类型为[Int/UInt](../../sql-reference/data-types/int-uint.md)或[Float](../../sql-reference/data-types/float.md)。 + +**返回值** + +- 两个值中的最大值。 + +类型: [Float](../../sql-reference/data-types/float.md)。 + +**示例** + +查询语句: + +```sql +SELECT max2(-1, 2); +``` + +结果: + +```text +┌─max2(-1, 2)─┐ +│ 2 │ +└─────────────┘ +``` + +## min2 {#min2} + +比较两个值并返回最小值。返回值类型转换为[Float64](../../sql-reference/data-types/float.md)。 + +**语法** + +```sql +max2(value1, value2) +``` + +**参数** + +- `value1` — 第一个值,类型为[Int/UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md)。 +- `value2` — 第二个值,类型为[Int/UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md)。 + +**返回值** + +- 两个值中的最小值。 + +类型: [Float](../../sql-reference/data-types/float.md)。 + +**示例** + +查询语句: + +```sql +SELECT min2(-1, 2); +``` + +结果: + +```text +┌─min2(-1, 2)─┐ +│ -1 │ +└─────────────┘ +``` + [来源文章](https://clickhouse.com/docs/en/query_language/functions/arithmetic_functions/) diff --git a/docs/zh/sql-reference/functions/array-functions.md b/docs/zh/sql-reference/functions/array-functions.md index 6ec6d8fe352..0d4f51e1ddc 100644 --- a/docs/zh/sql-reference/functions/array-functions.md +++ b/docs/zh/sql-reference/functions/array-functions.md @@ -1,16 +1,92 @@ # 数组函数 {#shu-zu-han-shu} -## empty {#empty} +## empty {#empty函数} -对于空数组返回1,对于非空数组返回0。 -结果类型是UInt8。 -该函数也适用于字符串。 +检查输入的数组是否为空。 -## notEmpty {#notempty} +**语法** -对于空数组返回0,对于非空数组返回1。 -结果类型是UInt8。 -该函数也适用于字符串。 +``` sql +empty([x]) +``` + +如果一个数组中不包含任何元素,则此数组为空数组。 + +:::注意 +可以通过启用[optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns)配置进行优化。设置`optimize_functions_to_subcolumns = 1`后,函数通过读取[size0](../../sql-reference/data-types/array.md#array-size)子列获取结果,不在读取和处理整个数组列,查询语句`SELECT empty(arr) FROM TABLE;`将转化为`SELECT arr.size0 = 0 FROM TABLE;`。 +::: + +此函数也适用于[strings](string-functions.md#empty)或[UUID](uuid-functions.md#empty)。 + +**参数** + +- `[x]` — 输入的数组,类型为[数组](../data-types/array.md)。 + +**返回值** + +- 对于空数组返回`1`,对于非空数组返回`0`。 + +类型: [UInt8](../data-types/int-uint.md)。 + +**示例** + +查询语句: + +```sql +SELECT empty([]); +``` + +结果: + +```text +┌─empty(array())─┐ +│ 1 │ +└────────────────┘ +``` + +## notEmpty {#notempty函数} + +检测输入的数组是否非空。 + +**语法** + +``` sql +notEmpty([x]) +``` + +如果一个数组至少包含一个元素,则此数组为非空数组。 + +:::注意 +可以通过启用[optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns)配置进行优化。设置`optimize_functions_to_subcolumns = 1`后,函数通过读取[size0](../../sql-reference/data-types/array.md#array-size)子列获取结果,不在读取和处理整个数组列,查询语句`SELECT notEmpty(arr) FROM TABLE;`将转化为`SELECT arr.size0 != 0 FROM TABLE;`。 +::: + +此函数也适用于[strings](string-functions.md#empty)或[UUID](uuid-functions.md#empty)。 + +**参数** + +- `[x]` — 输入的数组,类型为[数组](../data-types/array.md)。 + +**返回值** + +- 对于空数组返回`0`,对于非空数组返回`1`。 + +类型: [UInt8](../data-types/int-uint.md). + +**示例** + +查询语句: + +```sql +SELECT notEmpty([1,2]); +``` + +结果: + +```text +┌─notEmpty([1, 2])─┐ +│ 1 │ +└──────────────────┘ +``` ## length {#array_functions-length} @@ -18,13 +94,15 @@ 结果类型是UInt64。 该函数也适用于字符串。 -## emptyArrayUInt8,emptyArrayUInt16,emptyArrayUInt32,emptyArrayUInt64 {#emptyarrayuint8-emptyarrayuint16-emptyarrayuint32-emptyarrayuint64} +可以通过启用[optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns)配置进行优化。设置`optimize_functions_to_subcolumns = 1`后,函数通过读取[size0](../../sql-reference/data-types/array.md#array-size)子列获取结果,不在读取和处理整个数组列,查询语句`SELECT length(arr) FROM table`将转化为`SELECT arr.size0 FROM TABLE`。 -## emptyArrayInt8,emptyArrayInt16,emptyArrayInt32,emptyArrayInt64 {#emptyarrayint8-emptyarrayint16-emptyarrayint32-emptyarrayint64} +## emptyArrayUInt8, emptyArrayUInt16, emptyArrayUInt32, emptyArrayUInt64 {#emptyarrayuint8-emptyarrayuint16-emptyarrayuint32-emptyarrayuint64} -## emptyArrayFloat32,emptyArrayFloat64 {#emptyarrayfloat32-emptyarrayfloat64} +## emptyArrayInt8, emptyArrayInt16, emptyArrayInt32, emptyArrayInt64 {#emptyarrayint8-emptyarrayint16-emptyarrayint32-emptyarrayint64} -## emptyArrayDate,emptyArrayDateTime {#emptyarraydate-emptyarraydatetime} +## emptyArrayFloat32, emptyArrayFloat64 {#emptyarrayfloat32-emptyarrayfloat64} + +## emptyArrayDate, emptyArrayDateTime {#emptyarraydate-emptyarraydatetime} ## emptyArrayString {#emptyarraystring} @@ -34,10 +112,43 @@ 接受一个空数组并返回一个仅包含一个默认值元素的数组。 -## range(N) {#rangen} +## range(end), range(\[start, \] end \[, step\]) {#range} -返回从0到N-1的数字数组。 -以防万一,如果在数据块中创建总长度超过100,000,000个元素的数组,则抛出异常。 +返回一个以`step`作为增量步长的从`start`到`end - 1`的`UInt`类型数字数组。 + +**语法** +``` sql +range([start, ] end [, step]) +``` + +**参数** + +- `start` — 数组的第一个元素。可选项,如果设置了`step`时同样需要`start`,默认值为:0,类型为[UInt](../data-types/int-uint.md)。 +- `end` — 计数到`end`结束,但不包括`end`,必填项,类型为[UInt](../data-types/int-uint.md)。 +- `step` — 确定数组中每个元素之间的增量步长。可选项,默认值为:1,类型为[UInt](../data-types/int-uint.md)。 + +**返回值** + +- 以`step`作为增量步长的从`start`到`end - 1`的`UInt`类型数字数组。 + +**注意事项** + +- 所有参数必须是正值:`start`、`end`、`step`,类型均为`UInt`,结果数组的元素与此相同。 +- 如果查询结果的数组总长度超过[function_range_max_elements_in_block](../../operations/settings/settings.md#settings-function_range_max_elements_in_block)指定的元素数,将会抛出异常。 + + +**示例** + +查询语句: +``` sql +SELECT range(5), range(1, 5), range(1, 5, 2); +``` +结果: +```txt +┌─range(5)────┬─range(1, 5)─┬─range(1, 5, 2)─┐ +│ [0,1,2,3,4] │ [1,2,3,4] │ [1,3] │ +└─────────────┴─────────────┴────────────────┘ +``` ## array(x1, …), operator \[x1, …\] {#arrayx1-operator-x1} @@ -49,7 +160,9 @@ 合并参数中传递的所有数组。 - arrayConcat(arrays) +``` sql +arrayConcat(arrays) +``` **参数** @@ -62,9 +175,11 @@ SELECT arrayConcat([1, 2], [3, 4], [5, 6]) AS res ``` - ┌─res───────────┐ - │ [1,2,3,4,5,6] │ - └───────────────┘ +``` text +┌─res───────────┐ +│ [1,2,3,4,5,6] │ +└───────────────┘ +``` ## arrayElement(arr,n),运算符arr\[n\] {#arrayelementarr-n-operator-arrn} @@ -156,22 +271,76 @@ SELECT arrayConcat([1, 2], [3, 4], [5, 6]) AS res `SELECT hasAll([[1, 2], [3, 4]], [[1, 2], [1, 2]])` 返回 `1`. +## hasSubstr {#hassubstr} + +检查 array2 的所有元素是否以相同的顺序出现在 array1 中。当且仅当 `array1 = prefix + array2 + suffix`时,该函数将返回 1。 + +``` sql +hasSubstr(array1, array2) +``` + +换句话说,函数将检查 `array2` 的所有元素是否都包含在 `array1` 中,就像 `hasAll` 函数一样。此外,它将检查元素在“array1”和“array2”中的出现顺序是否相同。 + +举例: +- `hasSubstr([1,2,3,4], [2,3])`返回`1`。然而`hasSubstr([1,2,3,4], [3,2])`将返回`0`。 +- `hasSubstr([1,2,3,4], [1,2,3])`返回`1`。然而`hasSubstr([1,2,3,4], [1,2,4])`将返回`0`。 + +**参数** + +- `array1` – 具有一组元素的任何类型数组。 +- `array2` – 具有一组元素的任何类型数组。 + +**返回值** + +- 如果`array1`中包含`array2`,返回`1`。 +- 其他情况返回`0`。 + +**特殊属性** + +- 当`array2`为空数组时,函数将返回`1`. +- `Null` 作为值处理。换句话说,`hasSubstr([1, 2, NULL, 3, 4], [2,3])` 将返回 `0`。但是,`hasSubstr([1, 2, NULL, 3, 4], [2,NULL,3])` 将返回 `1`。 +- 两个数组中**值的顺序**会影响函数结果。 + +**示例** + +`SELECT hasSubstr([], [])`返回1。 + +`SELECT hasSubstr([1, Null], [Null])`返回1。 + +`SELECT hasSubstr([1.0, 2, 3, 4], [1, 3])`返回0。 + +`SELECT hasSubstr(['a', 'b'], ['a'])`返回1。 + +`SELECT hasSubstr(['a', 'b' , 'c'], ['a', 'b'])`返回1。 + +`SELECT hasSubstr(['a', 'b' , 'c'], ['a', 'c'])`返回0。 + +`SELECT hasSubstr([[1, 2], [3, 4], [5, 6]], [[1, 2], [3, 4]])`返回1。 + ## indexOf(arr,x) {#indexofarr-x} 返回数组中第一个’x’元素的索引(从1开始),如果’x’元素不存在在数组中,则返回0。 示例: - :) SELECT indexOf([1,3,NULL,NULL],NULL) +``` sql +SELECT indexOf([1, 3, NULL, NULL], NULL) +``` - SELECT indexOf([1, 3, NULL, NULL], NULL) - - ┌─indexOf([1, 3, NULL, NULL], NULL)─┐ - │ 3 │ - └───────────────────────────────────┘ +``` text +┌─indexOf([1, 3, NULL, NULL], NULL)─┐ +│ 3 │ +└───────────────────────────────────┘ +``` 设置为«NULL»的元素将作为普通的元素值处理。 +## arrayCount(\[func,\] arr1, …) {#array-count} + +`func`将arr数组作为参数,其返回结果为非零值的数量。如果未指定“func”,则返回数组中非零元素的数量。 + +请注意,`arrayCount`是一个[高阶函数](../../sql-reference/functions/index.md#higher-order-functions)。您可以将 lambda 函数作为第一个参数传递给它。 + ## countEqual(arr,x) {#countequalarr-x} 返回数组中等于x的元素的个数。相当于arrayCount(elem - \> elem = x,arr)。 @@ -180,11 +349,15 @@ SELECT arrayConcat([1, 2], [3, 4], [5, 6]) AS res 示例: - SELECT countEqual([1, 2, NULL, NULL], NULL) +``` sql +SELECT countEqual([1, 2, NULL, NULL], NULL) +``` - ┌─countEqual([1, 2, NULL, NULL], NULL)─┐ - │ 2 │ - └──────────────────────────────────────┘ +``` text +┌─countEqual([1, 2, NULL, NULL], NULL)─┐ +│ 2 │ +└──────────────────────────────────────┘ +``` ## arrayEnumerate(arr) {#array_functions-arrayenumerate} @@ -204,9 +377,11 @@ WHERE CounterID = 160656 LIMIT 10 ``` - ┌─Reaches─┬──Hits─┐ - │ 95606 │ 31406 │ - └─────────┴───────┘ +``` text +┌─Reaches─┬──Hits─┐ +│ 95606 │ 31406 │ +└─────────┴───────┘ +``` 在此示例中,Reaches是转换次数(应用ARRAY JOIN后接收的字符串),Hits是浏览量(ARRAY JOIN之前的字符串)。在这种特殊情况下,您可以更轻松地获得相同的结果: @@ -218,9 +393,11 @@ FROM test.hits WHERE (CounterID = 160656) AND notEmpty(GoalsReached) ``` - ┌─Reaches─┬──Hits─┐ - │ 95606 │ 31406 │ - └─────────┴───────┘ +``` text +┌─Reaches─┬──Hits─┐ +│ 95606 │ 31406 │ +└─────────┴───────┘ +``` 此功能也可用于高阶函数。例如,您可以使用它来获取与条件匹配的元素的数组索引。 @@ -248,18 +425,20 @@ ORDER BY Reaches DESC LIMIT 10 ``` - ┌──GoalID─┬─Reaches─┬─Visits─┐ - │ 53225 │ 3214 │ 1097 │ - │ 2825062 │ 3188 │ 1097 │ - │ 56600 │ 2803 │ 488 │ - │ 1989037 │ 2401 │ 365 │ - │ 2830064 │ 2396 │ 910 │ - │ 1113562 │ 2372 │ 373 │ - │ 3270895 │ 2262 │ 812 │ - │ 1084657 │ 2262 │ 345 │ - │ 56599 │ 2260 │ 799 │ - │ 3271094 │ 2256 │ 812 │ - └─────────┴─────────┴────────┘ +``` text +┌──GoalID─┬─Reaches─┬─Visits─┐ +│ 53225 │ 3214 │ 1097 │ +│ 2825062 │ 3188 │ 1097 │ +│ 56600 │ 2803 │ 488 │ +│ 1989037 │ 2401 │ 365 │ +│ 2830064 │ 2396 │ 910 │ +│ 1113562 │ 2372 │ 373 │ +│ 3270895 │ 2262 │ 812 │ +│ 1084657 │ 2262 │ 345 │ +│ 56599 │ 2260 │ 799 │ +│ 3271094 │ 2256 │ 812 │ +└─────────┴─────────┴────────┘ +``` 在此示例中,每个GoalID都计算转换次数(目标嵌套数据结构中的每个元素都是达到的目标,我们称之为转换)和会话数。如果没有ARRAY JOIN,我们会将会话数计为总和(Sign)。但在这种特殊情况下,行乘以嵌套的Goals结构,因此为了在此之后计算每个会话一次,我们将一个条件应用于arrayEnumerateUniq(Goals.ID)函数的值。 @@ -269,9 +448,11 @@ arrayEnumerateUniq函数可以使用与参数大小相同的多个数组。在 SELECT arrayEnumerateUniq([1, 1, 1, 2, 2, 2], [1, 1, 2, 1, 1, 2]) AS res ``` - ┌─res───────────┐ - │ [1,2,1,1,2,1] │ - └───────────────┘ +``` text +┌─res───────────┐ +│ [1,2,1,1,2,1] │ +└───────────────┘ +``` 当使用带有嵌套数据结构的ARRAY JOIN并在此结构中跨多个元素进一步聚合时,这是必需的。 @@ -291,10 +472,11 @@ SELECT arrayEnumerateUniq([1, 1, 1, 2, 2, 2], [1, 1, 2, 1, 1, 2]) AS res SELECT arrayPopBack([1, 2, 3]) AS res ``` - ┌─res───┐ - │ [1,2] │ - └───────┘ - +``` text +┌─res───┐ +│ [1,2] │ +└───────┘ +``` ## arrayPopFront {#arraypopfront} 从数组中删除第一项。 @@ -311,15 +493,19 @@ SELECT arrayPopBack([1, 2, 3]) AS res SELECT arrayPopFront([1, 2, 3]) AS res ``` - ┌─res───┐ - │ [2,3] │ - └───────┘ +``` text +┌─res───┐ +│ [2,3] │ +└───────┘ +``` ## arrayPushBack {#arraypushback} 添加一个元素到数组的末尾。 - arrayPushBack(array, single_value) +``` sql +arrayPushBack(array, single_value) +``` **参数** @@ -332,15 +518,19 @@ SELECT arrayPopFront([1, 2, 3]) AS res SELECT arrayPushBack(['a'], 'b') AS res ``` - ┌─res───────┐ - │ ['a','b'] │ - └───────────┘ +``` text +┌─res───────┐ +│ ['a','b'] │ +└───────────┘ +``` ## arrayPushFront {#arraypushfront} 将一个元素添加到数组的开头。 - arrayPushFront(array, single_value) +``` sql +arrayPushFront(array, single_value) +``` **参数** @@ -353,15 +543,19 @@ SELECT arrayPushBack(['a'], 'b') AS res SELECT arrayPushFront(['b'], 'a') AS res ``` - ┌─res───────┐ - │ ['a','b'] │ - └───────────┘ +``` text +┌─res───────┐ +│ ['a','b'] │ +└───────────┘ +``` ## arrayResize {#arrayresize} 更改数组的长度。 - arrayResize(array, size[, extender]) +``` sql +arrayResize(array, size[, extender]) +``` **参数:** @@ -377,23 +571,33 @@ SELECT arrayPushFront(['b'], 'a') AS res **调用示例** - SELECT arrayResize([1], 3) +``` sql +SELECT arrayResize([1], 3); +``` - ┌─arrayResize([1], 3)─┐ - │ [1,0,0] │ - └─────────────────────┘ +``` text +┌─arrayResize([1], 3)─┐ +│ [1,0,0] │ +└─────────────────────┘ +``` - SELECT arrayResize([1], 3, NULL) +``` sql +SELECT arrayResize([1], 3, NULL); +``` - ┌─arrayResize([1], 3, NULL)─┐ - │ [1,NULL,NULL] │ - └───────────────────────────┘ +``` text +┌─arrayResize([1], 3, NULL)─┐ +│ [1,NULL,NULL] │ +└───────────────────────────┘ +``` ## arraySlice {#arrayslice} 返回一个子数组,包含从指定位置的指定长度的元素。 - arraySlice(array, offset[, length]) +``` sql +arraySlice(array, offset[, length]) +``` **参数** @@ -407,9 +611,11 @@ SELECT arrayPushFront(['b'], 'a') AS res SELECT arraySlice([1, 2, NULL, 4, 5], 2, 3) AS res ``` - ┌─res────────┐ - │ [2,NULL,4] │ - └────────────┘ +``` text +┌─res────────┐ +│ [2,NULL,4] │ +└────────────┘ +``` 设置为«NULL»的数组元素作为普通的数组元素值处理。 @@ -423,9 +629,11 @@ SELECT arraySlice([1, 2, NULL, 4, 5], 2, 3) AS res SELECT arraySort([1, 3, 3, 0]); ``` - ┌─arraySort([1, 3, 3, 0])─┐ - │ [0,1,3,3] │ - └─────────────────────────┘ +``` text +┌─arraySort([1, 3, 3, 0])─┐ +│ [0,1,3,3] │ +└─────────────────────────┘ +``` 字符串排序示例: @@ -433,9 +641,11 @@ SELECT arraySort([1, 3, 3, 0]); SELECT arraySort(['hello', 'world', '!']); ``` - ┌─arraySort(['hello', 'world', '!'])─┐ - │ ['!','hello','world'] │ - └────────────────────────────────────┘ +``` text +┌─arraySort(['hello', 'world', '!'])─┐ +│ ['!','hello','world'] │ +└────────────────────────────────────┘ +``` `NULL`,`NaN`和`Inf`的排序顺序: @@ -443,9 +653,11 @@ SELECT arraySort(['hello', 'world', '!']); SELECT arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]); ``` - ┌─arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf])─┐ - │ [-inf,-4,1,2,3,inf,nan,nan,NULL,NULL] │ - └───────────────────────────────────────────────────────────┘ +``` text +┌─arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf])─┐ +│ [-inf,-4,1,2,3,inf,nan,nan,NULL,NULL] │ +└───────────────────────────────────────────────────────────┘ +``` - `-Inf` 是数组中的第一个。 - `NULL` 是数组中的最后一个。 @@ -460,9 +672,11 @@ SELECT arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]); SELECT arraySort((x) -> -x, [1, 2, 3]) as res; ``` - ┌─res─────┐ - │ [3,2,1] │ - └─────────┘ +``` text +┌─res─────┐ +│ [3,2,1] │ +└─────────┘ +``` 对于源数组的每个元素,lambda函数返回排序键,即\[1 -\> -1, 2 -\> -2, 3 -\> -3\]。由于`arraySort`函数按升序对键进行排序,因此结果为\[3,2,1\]。因此,`(x) -> -x` lambda函数将排序设置为[降序](#array_functions-reverse-sort)。 @@ -472,9 +686,11 @@ lambda函数可以接受多个参数。在这种情况下,您需要为`arraySo SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]) as res; ``` - ┌─res────────────────┐ - │ ['world', 'hello'] │ - └────────────────────┘ +``` text +┌─res────────────────┐ +│ ['world', 'hello'] │ +└────────────────────┘ +``` 这里,在第二个数组(\[2, 1\])中定义了第一个数组(\[‘hello’,‘world’\])的相应元素的排序键,即\[‘hello’ -\> 2,‘world’ -\> 1\]。 由于lambda函数中没有使用`x`,因此源数组中的实际值不会影响结果的顺序。所以,’world’将是结果中的第一个元素,’hello’将是结果中的第二个元素。 @@ -484,7 +700,7 @@ SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]) as res; SELECT arraySort((x, y) -> y, [0, 1, 2], ['c', 'b', 'a']) as res; ``` -``` sql +``` text ┌─res─────┐ │ [2,1,0] │ └─────────┘ @@ -494,7 +710,7 @@ SELECT arraySort((x, y) -> y, [0, 1, 2], ['c', 'b', 'a']) as res; SELECT arraySort((x, y) -> -y, [0, 1, 2], [1, 2, 3]) as res; ``` -``` sql +``` text ┌─res─────┐ │ [2,1,0] │ └─────────┘ @@ -513,9 +729,11 @@ SELECT arraySort((x, y) -> -y, [0, 1, 2], [1, 2, 3]) as res; SELECT arrayReverseSort([1, 3, 3, 0]); ``` - ┌─arrayReverseSort([1, 3, 3, 0])─┐ - │ [3,3,1,0] │ - └────────────────────────────────┘ +``` text +┌─arrayReverseSort([1, 3, 3, 0])─┐ +│ [3,3,1,0] │ +└────────────────────────────────┘ +``` 字符串排序示例: @@ -523,9 +741,11 @@ SELECT arrayReverseSort([1, 3, 3, 0]); SELECT arrayReverseSort(['hello', 'world', '!']); ``` - ┌─arrayReverseSort(['hello', 'world', '!'])─┐ - │ ['world','hello','!'] │ - └───────────────────────────────────────────┘ +``` text +┌─arrayReverseSort(['hello', 'world', '!'])─┐ +│ ['world','hello','!'] │ +└───────────────────────────────────────────┘ +``` `NULL`,`NaN`和`Inf`的排序顺序: @@ -533,7 +753,7 @@ SELECT arrayReverseSort(['hello', 'world', '!']); SELECT arrayReverseSort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]) as res; ``` -``` sql +``` text ┌─res───────────────────────────────────┐ │ [inf,3,2,1,-4,-inf,nan,nan,NULL,NULL] │ └───────────────────────────────────────┘ @@ -550,11 +770,12 @@ SELECT arrayReverseSort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]) as res; SELECT arrayReverseSort((x) -> -x, [1, 2, 3]) as res; ``` - ┌─res─────┐ - │ [1,2,3] │ - └─────────┘ +``` text +┌─res─────┐ +│ [1,2,3] │ +└─────────┘ +``` -数组按以下方式排序: 数组按以下方式排序: 1. 首先,根据lambda函数的调用结果对源数组(\[1, 2, 3\])进行排序。 结果是\[3, 2, 1\]。 @@ -566,7 +787,7 @@ lambda函数可以接受多个参数。在这种情况下,您需要为`arrayRe SELECT arrayReverseSort((x, y) -> y, ['hello', 'world'], [2, 1]) as res; ``` -``` sql +``` text ┌─res───────────────┐ │ ['hello','world'] │ └───────────────────┘ @@ -583,7 +804,7 @@ SELECT arrayReverseSort((x, y) -> y, ['hello', 'world'], [2, 1]) as res; SELECT arrayReverseSort((x, y) -> y, [4, 3, 5], ['a', 'b', 'c']) AS res; ``` -``` sql +``` text ┌─res─────┐ │ [5,3,4] │ └─────────┘ @@ -593,7 +814,7 @@ SELECT arrayReverseSort((x, y) -> y, [4, 3, 5], ['a', 'b', 'c']) AS res; SELECT arrayReverseSort((x, y) -> -y, [4, 3, 5], [1, 2, 3]) AS res; ``` -``` sql +``` text ┌─res─────┐ │ [4,3,5] │ └─────────┘ @@ -610,35 +831,108 @@ SELECT arrayReverseSort((x, y) -> -y, [4, 3, 5], [1, 2, 3]) AS res; 一个特殊的功能。请参见[«ArrayJoin函数»](array-join.md#functions_arrayjoin)部分。 -## arrayDifference(arr) {#arraydifferencearr} +## arrayDifference {#arraydifference} -返回一个数组,其中包含所有相邻元素对之间的差值。例如: +计算相邻数组元素之间的差异。返回一个数组,其中第一个元素为 0,第二个是 `a[1] - a[0]` 之差等。结果数组中元素的类型由减法的类型推断规则确定(例如`UInt8` - `UInt8` = `Int16`)。 + +**语法** ``` sql -SELECT arrayDifference([1, 2, 3, 4]) +arrayDifference(array) ``` - ┌─arrayDifference([1, 2, 3, 4])─┐ - │ [0,1,1,1] │ - └───────────────────────────────┘ +**参数** -## arrayDistinct(arr) {#arraydistinctarr} +- `array` –类型为[数组](https://clickhouse.com/docs/en/data_types/array/)。 -返回一个包含所有数组中不同元素的数组。例如: +**返回值** + +返回相邻元素之间的差异数组。 + +类型: [UInt\*](https://clickhouse.com/docs/en/data_types/int_uint/#uint-ranges), [Int\*](https://clickhouse.com/docs/en/data_types/int_uint/#int-ranges), [Float\*](https://clickhouse.com/docs/en/data_types/float/)。 + +**示例** + +查询语句: ``` sql -SELECT arrayDistinct([1, 2, 2, 3, 1]) +SELECT arrayDifference([1, 2, 3, 4]); ``` - ┌─arrayDistinct([1, 2, 2, 3, 1])─┐ - │ [1,2,3] │ - └────────────────────────────────┘ +结果: + +``` text +┌─arrayDifference([1, 2, 3, 4])─┐ +│ [0,1,1,1] │ +└───────────────────────────────┘ +``` + +由于结果类型为Int64导致的溢出示例: + +查询语句: + +``` sql +SELECT arrayDifference([0, 10000000000000000000]); +``` + +结果: + +``` text +┌─arrayDifference([0, 10000000000000000000])─┐ +│ [0,-8446744073709551616] │ +└────────────────────────────────────────────┘ +``` +## arrayDistinct {#arraydistinctarr} + +返回一个包含所有数组中不同元素的数组。 + +**语法** + +``` sql +arrayDistinct(array) +``` + +**参数** + +- `array` –类型为[数组](https://clickhouse.com/docs/en/data_types/array/)。 + +**返回值** + +返回一个包含不同元素的数组。 + +**示例** + +查询语句: + +``` sql +SELECT arrayDistinct([1, 2, 2, 3, 1]); +``` + +结果: + +``` text +┌─arrayDistinct([1, 2, 2, 3, 1])─┐ +│ [1,2,3] │ +└────────────────────────────────┘ +``` ## arrayEnumerateDense(arr) {#arrayenumeratedensearr} 返回与源数组大小相同的数组,指示每个元素首次出现在源数组中的位置。例如:arrayEnumerateDense(\[10,20,10,30\])= \[1,2,1,3\]。 -## arrayIntersect(arr) {#arrayintersectarr} +示例: + +``` sql +SELECT arrayEnumerateDense([10, 20, 10, 30]) +``` + +``` text +┌─arrayEnumerateDense([10, 20, 10, 30])─┐ +│ [1,2,1,3] │ +└───────────────────────────────────────┘ +``` + +## arrayIntersect(arr) {#array-functions-arrayintersect} 返回所有数组元素的交集。例如: @@ -648,18 +942,755 @@ SELECT arrayIntersect([1, 2], [1, 3], [1, 4]) AS intersect ``` - ┌─no_intersect─┬─intersect─┐ - │ [] │ [1] │ - └──────────────┴───────────┘ +``` text +┌─no_intersect─┬─intersect─┐ +│ [] │ [1] │ +└──────────────┴───────────┘ +``` -## arrayReduce(agg_func, arr1, …) {#arrayreduceagg-func-arr1} +## arrayReduce {#arrayreduce} -将聚合函数应用于数组并返回其结果。如果聚合函数具有多个参数,则此函数可应用于相同大小的多个数组。 +将聚合函数应用于数组元素并返回其结果。聚合函数的名称以单引号 `'max'`、`'sum'` 中的字符串形式传递。使用参数聚合函数时,参数在括号中的函数名称后指示“uniqUpTo(6)”。 -arrayReduce(‘agg_func’,arr1,…) - 将聚合函数`agg_func`应用于数组`arr1 ...`。如果传递了多个数组,则相应位置上的元素将作为多个参数传递给聚合函数。例如:SELECT arrayReduce(‘max’,\[1,2,3\])= 3 +**语法** -## arrayReverse(arr) {#arrayreversearr} +``` sql +arrayReduce(agg_func, arr1, arr2, ..., arrN) +``` -返回与源数组大小相同的数组,包含反转源数组的所有元素的结果。 +**参数** + +- `agg_func` — 聚合函数的名称,应该是一个常量[string](../../sql-reference/data-types/string.md)。 +- `arr` — 任意数量的[数组](../../sql-reference/data-types/array.md)类型列作为聚合函数的参数。 + +**返回值** + +**示例** + +查询语句: + +``` sql +SELECT arrayReduce('max', [1, 2, 3]); +``` + +结果: + +``` text +┌─arrayReduce('max', [1, 2, 3])─┐ +│ 3 │ +└───────────────────────────────┘ +``` + +如果聚合函数采用多个参数,则该函数必须应用于多个相同大小的数组。 + +查询语句: + +``` sql +SELECT arrayReduce('maxIf', [3, 5], [1, 0]); +``` + +结果: + +``` text +┌─arrayReduce('maxIf', [3, 5], [1, 0])─┐ +│ 3 │ +└──────────────────────────────────────┘ +``` + +带有参数聚合函数的示例: + +查询语句: + +``` sql +SELECT arrayReduce('uniqUpTo(3)', [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); +``` + +结果: + +``` text +┌─arrayReduce('uniqUpTo(3)', [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])─┐ +│ 4 │ +└─────────────────────────────────────────────────────────────┘ +``` + +## arrayReduceInRanges {#arrayreduceinranges} + +将聚合函数应用于给定范围内的数组元素,并返回一个包含与每个范围对应的结果的数组。该函数将返回与多个 `arrayReduce(agg_func, arraySlice(arr1, index, length), ...)` 相同的结果。 + +**语法** + +``` sql +arrayReduceInRanges(agg_func, ranges, arr1, arr2, ..., arrN) +``` + +**参数** + +- `agg_func` — 聚合函数的名称,应该是一个常量[string](../../sql-reference/data-types/string.md)。 +- `ranges` — 要聚合的范围应该是[元组](../../sql-reference/data-types/tuple.md)为元素的[数组](../../sql-reference/data-types/array.md),其中包含每个索引和长度范围。 +- `arr` — 任意数量的[数组](../../sql-reference/data-types/array.md)类型列作为聚合函数的参数。 + +**返回值** + +- 包含指定范围内聚合函数结果的数组。 + +类型为: [数组](../../sql-reference/data-types/array.md). + +**示例** + +查询语句: + +``` sql +SELECT arrayReduceInRanges( + 'sum', + [(1, 5), (2, 3), (3, 4), (4, 4)], + [1000000, 200000, 30000, 4000, 500, 60, 7] +) AS res +``` + +结果: + +``` text +┌─res─────────────────────────┐ +│ [1234500,234000,34560,4567] │ +└─────────────────────────────┘ +``` + +## arrayReverse(arr) {#arrayreverse} + +返回一个与原始数组大小相同的数组,其中包含相反顺序的元素。 + +示例: + +``` sql +SELECT arrayReverse([1, 2, 3]) +``` + +``` text +┌─arrayReverse([1, 2, 3])─┐ +│ [3,2,1] │ +└─────────────────────────┘ +``` + +## reverse(arr) {#array-functions-reverse} + +与[“arrayReverse”](#arrayreverse)作用相同。 + +## arrayFlatten {#arrayflatten} + +将嵌套的数组展平。 + +函数: + +- 适用于任何深度的嵌套数组。 +- 不会更改已经展平的数组。 + +展平后的数组包含来自所有源数组的所有元素。 + +**语法** + +``` sql +flatten(array_of_arrays) +``` + +别名: `flatten`. + +**参数** + +- `array_of_arrays` — 嵌套[数组](../../sql-reference/data-types/array.md)。 例如:`[[1,2,3], [4,5]]`。 + +**示例** + +``` sql +SELECT flatten([[[1]], [[2], [3]]]); +``` + +``` text +┌─flatten(array(array([1]), array([2], [3])))─┐ +│ [1,2,3] │ +└─────────────────────────────────────────────┘ +``` + +## arrayCompact {#arraycompact} + +从数组中删除连续的重复元素。结果值的顺序由源数组中的顺序决定。 + +**语法** + +``` sql +arrayCompact(arr) +``` + +**参数** + +`arr` — 类型为[数组](../../sql-reference/data-types/array.md)。 + +**返回值** + +没有重复元素的数组。 + +类型为: `Array`。 + +**示例** + +查询语句: + +``` sql +SELECT arrayCompact([1, 1, nan, nan, 2, 3, 3, 3]); +``` + +结果: + +``` text +┌─arrayCompact([1, 1, nan, nan, 2, 3, 3, 3])─┐ +│ [1,nan,nan,2,3] │ +└────────────────────────────────────────────┘ +``` + +## arrayZip {#arrayzip} + +将多个数组组合成一个数组。结果数组包含按列出的参数顺序分组为元组的源数组的相应元素。 + +**语法** + +``` sql +arrayZip(arr1, arr2, ..., arrN) +``` + +**参数** + +- `arrN` — N个[数组](../../sql-reference/data-types/array.md)。 + +该函数可以采用任意数量的不同类型的数组。所有输入数组的大小必须相等。 + +**返回值** + +- 将源数组中的元素分组为[元组](../../sql-reference/data-types/tuple.md)的数组。元组中的数据类型与输入数组的类型相同,并且与传递数组的顺序相同。 + +类型为: [数组](../../sql-reference/data-types/array.md)。 + +**示例** + +查询语句: + +``` sql +SELECT arrayZip(['a', 'b', 'c'], [5, 2, 1]); +``` + +结果: + +``` text +┌─arrayZip(['a', 'b', 'c'], [5, 2, 1])─┐ +│ [('a',5),('b',2),('c',1)] │ +└──────────────────────────────────────┘ +``` + +## arrayAUC {#arrayauc} + +计算AUC (ROC曲线下的面积,这是机器学习中的一个概念,更多细节请查看:https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve)。 + +**语法** + +``` sql +arrayAUC(arr_scores, arr_labels) +``` + +**参数** + +- `arr_scores` — 分数预测模型给出。 +- `arr_labels` — 样本的标签,通常为 1 表示正样本,0 表示负样本。 + +**返回值** + +返回 Float64 类型的 AUC 值。 + +**示例** + +查询语句: + +``` sql +select arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]); +``` + +结果: + +``` text +┌─arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1])─┐ +│ 0.75 │ +└───────────────────────────────────────────────┘ +``` + +## arrayMap(func, arr1, …) {#array-map} + +将从 `func` 函数的原始应用中获得的数组返回给 `arr` 数组中的每个元素。 + +示例: + +``` sql +SELECT arrayMap(x -> (x + 2), [1, 2, 3]) as res; +``` + +``` text +┌─res─────┐ +│ [3,4,5] │ +└─────────┘ +``` + +下面的例子展示了如何从不同的数组创建一个元素的元组: + +``` sql +SELECT arrayMap((x, y) -> (x, y), [1, 2, 3], [4, 5, 6]) AS res +``` + +``` text +┌─res─────────────────┐ +│ [(1,4),(2,5),(3,6)] │ +└─────────────────────┘ +``` + +请注意,`arrayMap` 是一个[高阶函数](../../sql-reference/functions/index.md#higher-order-functions)。 您必须将 lambda 函数作为第一个参数传递给它,并且不能省略。 + +## arrayFilter(func, arr1, …) {#array-filter} + +返回一个仅包含 `arr1` 中的元素的数组,其中 `func` 返回的值不是 0。 + +示例: + +``` sql +SELECT arrayFilter(x -> x LIKE '%World%', ['Hello', 'abc World']) AS res +``` + +``` text +┌─res───────────┐ +│ ['abc World'] │ +└───────────────┘ +``` + +``` sql +SELECT + arrayFilter( + (i, x) -> x LIKE '%World%', + arrayEnumerate(arr), + ['Hello', 'abc World'] AS arr) + AS res +``` + +``` text +┌─res─┐ +│ [2] │ +└─────┘ +``` + +请注意,`arrayFilter`是一个[高阶函数](../../sql-reference/functions/index.md#higher-order-functions)。 您必须将 lambda 函数作为第一个参数传递给它,并且不能省略。 + +## arrayFill(func, arr1, …) {#array-fill} + +从第一个元素到最后一个元素扫描`arr1`,如果`func`返回0,则用`arr1[i - 1]`替换`arr1[i]`。`arr1`的第一个元素不会被替换。 + +示例: + +``` sql +SELECT arrayFill(x -> not isNull(x), [1, null, 3, 11, 12, null, null, 5, 6, 14, null, null]) AS res +``` + +``` text +┌─res──────────────────────────────┐ +│ [1,1,3,11,12,12,12,5,6,14,14,14] │ +└──────────────────────────────────┘ +``` + +请注意,`arrayFill` 是一个[高阶函数](../../sql-reference/functions/index.md#higher-order-functions)。 您必须将 lambda 函数作为第一个参数传递给它,并且不能省略。 + +## arrayReverseFill(func, arr1, …) {#array-reverse-fill} + +从最后一个元素到第一个元素扫描`arr1`,如果`func`返回0,则用`arr1[i + 1]`替换`arr1[i]`。`arr1`的最后一个元素不会被替换。 + +示例: + +``` sql +SELECT arrayReverseFill(x -> not isNull(x), [1, null, 3, 11, 12, null, null, 5, 6, 14, null, null]) AS res +``` + +``` text +┌─res────────────────────────────────┐ +│ [1,3,3,11,12,5,5,5,6,14,NULL,NULL] │ +└────────────────────────────────────┘ +``` + +请注意,`arrayReverseFill`是一个[高阶函数](../../sql-reference/functions/index.md#higher-order-functions)。 您必须将 lambda 函数作为第一个参数传递给它,并且不能省略。 + +## arraySplit(func, arr1, …) {#array-split} + +将 `arr1` 拆分为多个数组。当 `func` 返回 0 以外的值时,数组将在元素的左侧拆分。数组不会在第一个元素之前被拆分。 + +示例: + +``` sql +SELECT arraySplit((x, y) -> y, [1, 2, 3, 4, 5], [1, 0, 0, 1, 0]) AS res +``` + +``` text +┌─res─────────────┐ +│ [[1,2,3],[4,5]] │ +└─────────────────┘ +``` + +请注意,`arraySplit`是一个[高阶函数](../../sql-reference/functions/index.md#higher-order-functions)。 您必须将 lambda 函数作为第一个参数传递给它,并且不能省略。 + +## arrayReverseSplit(func, arr1, …) {#array-reverse-split} + +将 `arr1` 拆分为多个数组。当 `func` 返回 0 以外的值时,数组将在元素的右侧拆分。数组不会在最后一个元素之后被拆分。 + +示例: + +``` sql +SELECT arrayReverseSplit((x, y) -> y, [1, 2, 3, 4, 5], [1, 0, 0, 1, 0]) AS res +``` + +``` text +┌─res───────────────┐ +│ [[1],[2,3,4],[5]] │ +└───────────────────┘ +``` + +请注意,`arrayReverseSplit`是一个[高阶函数](../../sql-reference/functions/index.md#higher-order-functions)。 您必须将 lambda 函数作为第一个参数传递给它,并且不能省略。 + +## arrayExists(\[func,\] arr1, …) {#arrayexistsfunc-arr1} + +如果 `arr` 中至少有一个元素 `func` 返回 0 以外的值,则返回 1。否则,它返回 0。 + +请注意,`arrayExists`是一个[高阶函数](../../sql-reference/functions/index.md#higher-order-functions)。您可以将 lambda 函数作为第一个参数传递给它,并且不能省略。 + +## arrayAll(\[func,\] arr1, …) {#arrayallfunc-arr1} + +如果 `func` 为 `arr` 中的所有元素返回 0 以外的值,则返回 1。否则,它返回 0。 + +请注意,`arrayAll`是一个[高阶函数](../../sql-reference/functions/index.md#higher-order-functions)。您可以将 lambda 函数作为第一个参数传递给它,并且不能省略。 + +## arrayFirst(func, arr1, …) {#array-first} + +返回 `arr1` 数组中 `func` 返回非 0 的值的第一个元素。 + +请注意,`arrayFirst`是一个[高阶函数](../../sql-reference/functions/index.md#higher-order-functions)。您必须将 lambda 函数作为第一个参数传递给它,并且不能省略。 + +## arrayLast(func, arr1, …) {#array-last} + +返回 `arr1` 数组中的最后一个元素,其中 `func` 返回的值不是 0。 + +请注意,`arrayLast`是一个[高阶函数](../../sql-reference/functions/index.md#higher-order-functions)。您必须将 lambda 函数作为第一个参数传递给它,并且不能省略。 + +## arrayFirstIndex(func, arr1, …) {#array-first-index} + +返回 `arr1` 数组中第一个元素的索引,其中 `func` 返回的值不是 0。 + +请注意,`arrayFirstIndex`是一个[高阶函数](../../sql-reference/functions/index.md#higher-order-functions)。您必须将 lambda 函数作为第一个参数传递给它,并且不能省略。 + +## arrayLastIndex(func, arr1, …) {#array-last-index} + +返回 `arr1` 数组中最后一个元素的索引,其中 `func` 返回的值不是 0。 + +请注意,`arrayLastIndex`是一个[高阶函数](../../sql-reference/functions/index.md#higher-order-functions)。您必须将 lambda 函数作为第一个参数传递给它,并且不能省略。 + +## arrayMin {#array-min} + +返回源数组中的最小元素。 + +如果指定了 `func` 函数,则返回此函数转换的元素的最小值。 + +请注意,`arrayMin`是一个[高阶函数](../../sql-reference/functions/index.md#higher-order-functions)。您可以将 lambda 函数作为第一个参数传递给它,并且不能省略。 + +**语法** + +```sql +arrayMin([func,] arr) +``` + +**参数** + +- `func` — 类型为[表达式](../../sql-reference/data-types/special-data-types/expression.md)。 +- `arr` — 类型为[数组](../../sql-reference/data-types/array.md)。 + +**返回值** + +- 函数值的最小值(或数组最小值)。 + +类型:如果指定了`func`,则匹配`func`返回值类型,否则匹配数组元素类型。 + +**示例** + +查询语句: + +```sql +SELECT arrayMin([1, 2, 4]) AS res; +``` + +结果: + +```text +┌─res─┐ +│ 1 │ +└─────┘ +``` + +查询语句: + +```sql +SELECT arrayMin(x -> (-x), [1, 2, 4]) AS res; +``` + +结果: + +```text +┌─res─┐ +│ -4 │ +└─────┘ +``` + +## arrayMax {#array-max} + +返回源数组中元素的最大值。 + +如果指定了`func` 函数,则返回此函数转换的元素的最大值。 + +请注意,`arrayMax`是一个[高阶函数](../../sql-reference/functions/index.md#higher-order-functions). 您可以将 lambda 函数作为第一个参数传递给它,并且不能省略。 + +**语法** + +```sql +arrayMax([func,] arr) +``` + +**参数** + +- `func` — 类型为[表达式](../../sql-reference/data-types/special-data-types/expression.md)。 +- `arr` — 类型为[数组](../../sql-reference/data-types/array.md)。 + +**返回值** + +- 函数值的最大值(或数组最大值)。 + +类型:如果指定了`func`,则匹配`func`返回值类型,否则匹配数组元素类型。 + +**示例** + +查询语句: + +```sql +SELECT arrayMax([1, 2, 4]) AS res; +``` + +结果: + +```text +┌─res─┐ +│ 4 │ +└─────┘ +``` + +查询语句: + +```sql +SELECT arrayMax(x -> (-x), [1, 2, 4]) AS res; +``` + +结果: + +```text +┌─res─┐ +│ -1 │ +└─────┘ +``` + +## arraySum {#array-sum} + +返回源数组中元素的总和。 + +如果指定了 `func` 函数,则返回此函数转换的元素的总和。 + +请注意,`arraySum`是一个[高阶函数](../../sql-reference/functions/index.md#higher-order-functions). 您可以将 lambda 函数作为第一个参数传递给它,并且不能省略。 + +**语法** + +```sql +arraySum([func,] arr) +``` + +**参数** + +- `func` — 类型为[表达式](../../sql-reference/data-types/special-data-types/expression.md)。 +- `arr` — 类型为[数组](../../sql-reference/data-types/array.md)。 + +**返回值** + +- 函数值的总和(或数组总和)。 + +类型:源数组中的十进制数(或转换后的值,如果指定了`func`)-[Decimal128](../../sql-reference/data-types/decimal.md), 对于浮点数 — [Float64](../../sql-reference/data-types/float.md), 对于无符号数 — [UInt64](../../sql-reference/data-types/int-uint.md), 对于有符号数 — [Int64](../../sql-reference/data-types/int-uint.md)。 + +**示例** + +查询语句: + +```sql +SELECT arraySum([2, 3]) AS res; +``` + +结果: + +```text +┌─res─┐ +│ 5 │ +└─────┘ +``` + +查询语句: + +```sql +SELECT arraySum(x -> x*x, [2, 3]) AS res; +``` + +结果: + +```text +┌─res─┐ +│ 13 │ +└─────┘ +``` + +## arrayAvg {#array-avg} + +返回源数组中元素的平均值。 + +如果指定了 func 函数,则返回此函数转换的元素的平均值。 + +请注意,`arrayAvg`是一个[高阶函数](../../sql-reference/functions/index.md#higher-order-functions). 您可以将 lambda 函数作为第一个参数传递给它,并且不能省略。 + +**语法** + +```sql +arrayAvg([func,] arr) +``` + +**参数** + +- `func` — 类型为[表达式](../../sql-reference/data-types/special-data-types/expression.md)。 +- `arr` — 类型为[数组](../../sql-reference/data-types/array.md)。 + +**返回值** + +- 函数值的平均值(或数组平均值)。 + +类型为: [Float64](../../sql-reference/data-types/float.md). + +**示例** + +查询语句: + +```sql +SELECT arrayAvg([1, 2, 4]) AS res; +``` + +结果: + +```text +┌────────────────res─┐ +│ 2.3333333333333335 │ +└────────────────────┘ +``` + +查询语句: + +```sql +SELECT arrayAvg(x -> (x * x), [2, 4]) AS res; +``` + +结果: + +```text +┌─res─┐ +│ 10 │ +└─────┘ +``` + +## arrayCumSum(\[func,\] arr1, …) {#arraycumsumfunc-arr1} + +返回源数组中元素的部分和的数组(运行总和)。如果指定了 func 函数,则数组元素的值在求和之前由该函数转换。 + +示例: + +``` sql +SELECT arrayCumSum([1, 1, 1, 1]) AS res +``` + +``` text +┌─res──────────┐ +│ [1, 2, 3, 4] │ +└──────────────┘ +``` + +请注意,`arrayCumSum`是一个[高阶函数](../../sql-reference/functions/index.md#higher-order-functions). 您可以将 lambda 函数作为第一个参数传递给它,并且不能省略。 + +## arrayCumSumNonNegative(arr) {#arraycumsumnonnegativearr} + +与 `arrayCumSum` 相同,返回源数组中元素的部分和的数组(运行总和)。不同的`arrayCumSum`,当返回值包含小于零的值时,将该值替换为零,并以零参数执行后续计算。例如: + +``` sql +SELECT arrayCumSumNonNegative([1, 1, -4, 1]) AS res +``` + +``` text +┌─res───────┐ +│ [1,2,0,1] │ +└───────────┘ +``` +请注意`arraySumNonNegative`是一个[高阶函数](../../sql-reference/functions/index.md#higher-order-functions). 您可以将 lambda 函数作为第一个参数传递给它,并且不能省略。 + +## arrayProduct {#arrayproduct} + +将一个[数组](../../sql-reference/data-types/array.md)中的元素相乘。 + +**语法** + +``` sql +arrayProduct(arr) +``` + +**参数** + +- `arr` — 数值类型的[数组](../../sql-reference/data-types/array.md)。 + +**返回值** + +- 数组元素的乘积。 + +类型为: [Float64](../../sql-reference/data-types/float.md). + +**示例** + +查询语句: + +``` sql +SELECT arrayProduct([1,2,3,4,5,6]) as res; +``` + +结果: + +``` text +┌─res───┐ +│ 720 │ +└───────┘ +``` + +查询语句: + +``` sql +SELECT arrayProduct([toDecimal64(1,8), toDecimal64(2,8), toDecimal64(3,8)]) as res, toTypeName(res); +``` + +返回值类型总是[Float64](../../sql-reference/data-types/float.md). 结果: + +``` text +┌─res─┬─toTypeName(arrayProduct(array(toDecimal64(1, 8), toDecimal64(2, 8), toDecimal64(3, 8))))─┐ +│ 6 │ Float64 │ +└─────┴──────────────────────────────────────────────────────────────────────────────────────────┘ +``` [来源文章](https://clickhouse.com/docs/en/query_language/functions/array_functions/) From f9e635763e77b9b69723bb61fc70914f39f46d47 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 11 May 2022 18:54:20 +0200 Subject: [PATCH 118/171] Update docs/en/operations/settings/memory-overcommit.md Co-authored-by: Nikita Taranov --- docs/en/operations/settings/memory-overcommit.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/memory-overcommit.md b/docs/en/operations/settings/memory-overcommit.md index b0c2e3bd35e..5da6cd8eb74 100644 --- a/docs/en/operations/settings/memory-overcommit.md +++ b/docs/en/operations/settings/memory-overcommit.md @@ -5,7 +5,7 @@ Memory overcommit is an experimental technique intended to allow to set more fle The idea of this technique is to introduce settings which can represent guaranteed amount of memory a query can use. When memory overcommit is enabled and the memory limit is reached ClickHouse will select the most overcommitted query and try to free memory by killing this query. -When memory limit is reached any query will wait some time during atempt to allocate new memory. +When memory limit is reached any query will wait some time during attempt to allocate new memory. If timeout is passed and memory is freed, the query continues execution. Otherwise an exception will be thrown and the query is killed. From 3b733ec8eb58e9151faab35af2aebc7e0d3fd393 Mon Sep 17 00:00:00 2001 From: Marcelo Rodriguez Date: Wed, 11 May 2022 10:56:12 -0600 Subject: [PATCH 119/171] Update Exception Message for allowed auth types update error message per this commit: https://github.com/ClickHouse/ClickHouse/pull/31484/commits/cb66a63aa4a5b4b3a9d55b0d041b755330dc7a17 the xml tag changed from `` to `` will also submit a correction to the following doc page: https://clickhouse.com/docs/en/operations/external-authenticators/ssl-x509 --- src/Access/UsersConfigAccessStorage.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 240130bbf74..712e5393ce7 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -67,11 +67,11 @@ namespace size_t num_password_fields = has_no_password + has_password_plaintext + has_password_sha256_hex + has_password_double_sha1_hex + has_ldap + has_kerberos + has_certificates; if (num_password_fields > 1) - throw Exception("More than one field of 'password', 'password_sha256_hex', 'password_double_sha1_hex', 'no_password', 'ldap', 'kerberos', 'certificates' are used to specify authentication info for user " + user_name + ". Must be only one of them.", + throw Exception("More than one field of 'password', 'password_sha256_hex', 'password_double_sha1_hex', 'no_password', 'ldap', 'kerberos', 'ssl_certificates' are used to specify authentication info for user " + user_name + ". Must be only one of them.", ErrorCodes::BAD_ARGUMENTS); if (num_password_fields < 1) - throw Exception("Either 'password' or 'password_sha256_hex' or 'password_double_sha1_hex' or 'no_password' or 'ldap' or 'kerberos' or 'certificates' must be specified for user " + user_name + ".", ErrorCodes::BAD_ARGUMENTS); + throw Exception("Either 'password' or 'password_sha256_hex' or 'password_double_sha1_hex' or 'no_password' or 'ldap' or 'kerberos' or 'ssl_certificates' must be specified for user " + user_name + ".", ErrorCodes::BAD_ARGUMENTS); if (has_password_plaintext) { From d756a34d8958b4d0ed2e0c8670a8d7af2c726dbd Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 11 May 2022 19:01:39 +0200 Subject: [PATCH 120/171] Update docs/en/operations/settings/memory-overcommit.md --- docs/en/operations/settings/memory-overcommit.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/memory-overcommit.md b/docs/en/operations/settings/memory-overcommit.md index 5da6cd8eb74..74cbc4dbd03 100644 --- a/docs/en/operations/settings/memory-overcommit.md +++ b/docs/en/operations/settings/memory-overcommit.md @@ -24,7 +24,7 @@ Waiting timeout is set by `memory_usage_overcommit_max_wait_microseconds` settin **Example** ```sql -SELECT number FROM numbers(1000) GROUP BY number SETTINGS max_guaranteed_memory_usage=4000, memory_usage_overcommit_max_wait_microseconds=500 +SELECT number FROM numbers(1000) GROUP BY number SETTINGS memory_overcommit_ratio_denominator=4000, memory_usage_overcommit_max_wait_microseconds=500 ``` ## Global overcommit tracker From 2d0020e9f69b791901f565ea02db3c2fd5b16a61 Mon Sep 17 00:00:00 2001 From: Marcelo Rodriguez Date: Wed, 11 May 2022 11:36:53 -0600 Subject: [PATCH 121/171] updated config file and xml tag Updated with correct config file. updated xml tag per this commit: https://github.com/ClickHouse/ClickHouse/commit/cb66a63aa4a5b4b3a9d55b0d041b755330dc7a17 --- docs/en/operations/external-authenticators/ssl-x509.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/external-authenticators/ssl-x509.md b/docs/en/operations/external-authenticators/ssl-x509.md index dd4f35257bb..15b5990d00e 100644 --- a/docs/en/operations/external-authenticators/ssl-x509.md +++ b/docs/en/operations/external-authenticators/ssl-x509.md @@ -2,7 +2,7 @@ [SSL 'strict' option](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) enables mandatory certificate validation for the incoming connections. In this case, only connections with trusted certificates can be established. Connections with untrusted certificates will be rejected. Thus, certificate validation allows to uniquely authenticate an incoming connection. `Common Name` field of the certificate is used to identify connected user. This allows to associate multiple certificates with the same user. Additionally, reissuing and revoking of the certificates does not affect the ClickHouse configuration. -To enable SSL certificate authentication, a list of `Common Name`'s for each ClickHouse user must be sspecified in the settings file `config.xml `: +To enable SSL certificate authentication, a list of `Common Name`'s for each ClickHouse user must be specified in the settings file `users.xml `: **Example** ```xml @@ -10,11 +10,11 @@ To enable SSL certificate authentication, a list of `Common Name`'s for each Cli - + host.domain.com:example_user host.domain.com:example_user_dev - + From 044e76d91be90a5eb40a30e3de52a40748c7b29f Mon Sep 17 00:00:00 2001 From: rfraposa Date: Wed, 11 May 2022 12:10:16 -0600 Subject: [PATCH 122/171] Add BLAKE3 hash function --- .../sql-reference/functions/hash-functions.md | 276 ++++++++++-------- .../sql-reference/functions/hash-functions.md | 274 +++++++++-------- 2 files changed, 313 insertions(+), 237 deletions(-) diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index e4b1fdd3bbb..bd05f3b4ad2 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -13,7 +13,7 @@ Simhash is a hash function, which returns close hash values for close (similar) [Interprets](../../sql-reference/functions/type-conversion-functions.md#type_conversion_functions-reinterpretAsString) all the input parameters as strings and calculates the [MD5](https://en.wikipedia.org/wiki/MD5) hash value for each of them. Then combines hashes, takes the first 8 bytes of the hash of the resulting string, and interprets them as `UInt64` in big-endian byte order. -``` sql +```sql halfMD5(par1, ...) ``` @@ -30,11 +30,11 @@ A [UInt64](../../sql-reference/data-types/int-uint.md) data type hash value. **Example** -``` sql +```sql SELECT halfMD5(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS halfMD5hash, toTypeName(halfMD5hash) AS type; ``` -``` text +```response ┌────────halfMD5hash─┬─type───┐ │ 186182704141653334 │ UInt64 │ └────────────────────┴────────┘ @@ -54,7 +54,7 @@ If you want to get the same result as output by the md5sum utility, use lower(he Produces a 64-bit [SipHash](https://131002.net/siphash/) hash value. -``` sql +```sql sipHash64(par1,...) ``` @@ -77,11 +77,11 @@ A [UInt64](../../sql-reference/data-types/int-uint.md) data type hash value. **Example** -``` sql +```sql SELECT sipHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS SipHash, toTypeName(SipHash) AS type; ``` -``` text +```response ┌──────────────SipHash─┬─type───┐ │ 13726873534472839665 │ UInt64 │ └──────────────────────┴────────┘ @@ -93,7 +93,7 @@ Produces a 128-bit [SipHash](https://131002.net/siphash/) hash value. Differs fr **Syntax** -``` sql +```sql sipHash128(par1,...) ``` @@ -111,13 +111,13 @@ Type: [FixedString(16)](../../sql-reference/data-types/fixedstring.md). Query: -``` sql +```sql SELECT hex(sipHash128('foo', '\x01', 3)); ``` Result: -``` text +```response ┌─hex(sipHash128('foo', '', 3))────┐ │ 9DE516A64A414D4B1B609415E4523F24 │ └──────────────────────────────────┘ @@ -127,7 +127,7 @@ Result: Produces a 64-bit [CityHash](https://github.com/google/cityhash) hash value. -``` sql +```sql cityHash64(par1,...) ``` @@ -145,11 +145,11 @@ A [UInt64](../../sql-reference/data-types/int-uint.md) data type hash value. Call example: -``` sql +```sql SELECT cityHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS CityHash, toTypeName(CityHash) AS type; ``` -``` text +```response ┌─────────────CityHash─┬─type───┐ │ 12072650598913549138 │ UInt64 │ └──────────────────────┴────────┘ @@ -157,7 +157,7 @@ SELECT cityHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:0 The following example shows how to compute the checksum of the entire table with accuracy up to the row order: -``` sql +```sql SELECT groupBitXor(cityHash64(*)) FROM table ``` @@ -177,7 +177,7 @@ Calculates SHA-1, SHA-224, SHA-256, SHA-512 hash from a string and returns the r **Syntax** -``` sql +```sql SHA1('s') ... SHA512('s') @@ -203,24 +203,62 @@ Use the [hex](../functions/encoding-functions.md#hex) function to represent the Query: -``` sql +```sql SELECT hex(SHA1('abc')); ``` Result: -``` text +```response ┌─hex(SHA1('abc'))─────────────────────────┐ │ A9993E364706816ABA3E25717850C26C9CD0D89D │ └──────────────────────────────────────────┘ ``` +## BLAKE3 {#blake3} + +Calculates BLAKE3 hash string and returns the resulting set of bytes as [FixedString](../data-types/fixedstring.md). + +**Syntax** + +```sql +BLAKE3('s') +``` + +This cryptographic hash-function is integrated into ClickHouse with BLAKE3 Rust library. The function is rather fast and shows approximately two times faster performance compared to SHA-2, while generating hashes of the same length as SHA-256. + +**Arguments** + +- s - input string for BLAKE3 hash calculation. [String](../data-types/string.md). + +**Return value** + +- BLAKE3 hash as a byte array with type FixedString(32). + +Type: [FixedString](../data-types/fixedstring.md). + +**Example** + +Use function [hex](../functions/encoding-functions.md#hex) to represent the result as a hex-encoded string. + +Query: +```sql +SELECT hex(BLAKE3('ABC')) +``` + +Result: +```sql +┌─hex(BLAKE3('ABC'))───────────────────────────────────────────────┐ +│ D1717274597CF0289694F75D96D444B992A096F1AFD8E7BBFA6EBB1D360FEDFC │ +└──────────────────────────────────────────────────────────────────┘ +``` + ## URLHash(url\[, N\]) {#urlhashurl-n} A fast, decent-quality non-cryptographic hash function for a string obtained from a URL using some type of normalization. `URLHash(s)` – Calculates a hash from a string without one of the trailing symbols `/`,`?` or `#` at the end, if present. `URLHash(s, N)` – Calculates a hash from a string up to the N level in the URL hierarchy, without one of the trailing symbols `/`,`?` or `#` at the end, if present. -Levels are the same as in URLHierarchy. +Levels are the same as in URLHierarchy. ## farmFingerprint64 {#farmfingerprint64} @@ -228,7 +266,7 @@ Levels are the same as in URLHierarchy. Produces a 64-bit [FarmHash](https://github.com/google/farmhash) or Fingerprint value. `farmFingerprint64` is preferred for a stable and portable value. -``` sql +```sql farmFingerprint64(par1, ...) farmHash64(par1, ...) ``` @@ -245,11 +283,11 @@ A [UInt64](../../sql-reference/data-types/int-uint.md) data type hash value. **Example** -``` sql +```sql SELECT farmHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS FarmHash, toTypeName(FarmHash) AS type; ``` -``` text +```response ┌─────────────FarmHash─┬─type───┐ │ 17790458267262532859 │ UInt64 │ └──────────────────────┴────────┘ @@ -261,7 +299,7 @@ Calculates [JavaHash](http://hg.openjdk.java.net/jdk8u/jdk8u/jdk/file/478a4add97 **Syntax** -``` sql +```sql SELECT javaHash('') ``` @@ -273,13 +311,13 @@ A `Int32` data type hash value. Query: -``` sql +```sql SELECT javaHash('Hello, world!'); ``` Result: -``` text +```response ┌─javaHash('Hello, world!')─┐ │ -1880044555 │ └───────────────────────────┘ @@ -291,7 +329,7 @@ Calculates [JavaHash](http://hg.openjdk.java.net/jdk8u/jdk8u/jdk/file/478a4add97 **Syntax** -``` sql +```sql javaHashUTF16LE(stringUtf16le) ``` @@ -309,13 +347,13 @@ Correct query with UTF-16LE encoded string. Query: -``` sql +```sql SELECT javaHashUTF16LE(convertCharset('test', 'utf-8', 'utf-16le')); ``` Result: -``` text +```response ┌─javaHashUTF16LE(convertCharset('test', 'utf-8', 'utf-16le'))─┐ │ 3556498 │ └──────────────────────────────────────────────────────────────┘ @@ -325,7 +363,7 @@ Result: Calculates `HiveHash` from a string. -``` sql +```sql SELECT hiveHash('') ``` @@ -341,13 +379,13 @@ Type: `hiveHash`. Query: -``` sql +```sql SELECT hiveHash('Hello, world!'); ``` Result: -``` text +```response ┌─hiveHash('Hello, world!')─┐ │ 267439093 │ └───────────────────────────┘ @@ -357,7 +395,7 @@ Result: Produces a 64-bit [MetroHash](http://www.jandrewrogers.com/2015/05/27/metrohash/) hash value. -``` sql +```sql metroHash64(par1, ...) ``` @@ -371,11 +409,11 @@ A [UInt64](../../sql-reference/data-types/int-uint.md) data type hash value. **Example** -``` sql +```sql SELECT metroHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MetroHash, toTypeName(MetroHash) AS type; ``` -``` text +```response ┌────────────MetroHash─┬─type───┐ │ 14235658766382344533 │ UInt64 │ └──────────────────────┴────────┘ @@ -391,7 +429,7 @@ For more information, see the link: [JumpConsistentHash](https://arxiv.org/pdf/1 Produces a [MurmurHash2](https://github.com/aappleby/smhasher) hash value. -``` sql +```sql murmurHash2_32(par1, ...) murmurHash2_64(par1, ...) ``` @@ -407,11 +445,11 @@ Both functions take a variable number of input parameters. Arguments can be any **Example** -``` sql +```sql SELECT murmurHash2_64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MurmurHash2, toTypeName(MurmurHash2) AS type; ``` -``` text +```response ┌──────────MurmurHash2─┬─type───┐ │ 11832096901709403633 │ UInt64 │ └──────────────────────┴────────┘ @@ -423,7 +461,7 @@ Calculates a 64-bit [MurmurHash2](https://github.com/aappleby/smhasher) hash val **Syntax** -``` sql +```sql gccMurmurHash(par1, ...) ``` @@ -441,7 +479,7 @@ Type: [UInt64](../../sql-reference/data-types/int-uint.md). Query: -``` sql +```sql SELECT gccMurmurHash(1, 2, 3) AS res1, gccMurmurHash(('a', [1, 2, 3], 4, (4, ['foo', 'bar'], 1, (1, 2)))) AS res2 @@ -449,7 +487,7 @@ SELECT Result: -``` text +```response ┌─────────────────res1─┬────────────────res2─┐ │ 12384823029245979431 │ 1188926775431157506 │ └──────────────────────┴─────────────────────┘ @@ -459,7 +497,7 @@ Result: Produces a [MurmurHash3](https://github.com/aappleby/smhasher) hash value. -``` sql +```sql murmurHash3_32(par1, ...) murmurHash3_64(par1, ...) ``` @@ -475,11 +513,11 @@ Both functions take a variable number of input parameters. Arguments can be any **Example** -``` sql +```sql SELECT murmurHash3_32(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MurmurHash3, toTypeName(MurmurHash3) AS type; ``` -``` text +```response ┌─MurmurHash3─┬─type───┐ │ 2152717 │ UInt32 │ └─────────────┴────────┘ @@ -491,7 +529,7 @@ Produces a 128-bit [MurmurHash3](https://github.com/aappleby/smhasher) hash valu **Syntax** -``` sql +```sql murmurHash3_128(expr) ``` @@ -509,13 +547,13 @@ Type: [FixedString(16)](../../sql-reference/data-types/fixedstring.md). Query: -``` sql +```sql SELECT hex(murmurHash3_128('foo', 'foo', 'foo')); ``` Result: -``` text +```response ┌─hex(murmurHash3_128('foo', 'foo', 'foo'))─┐ │ F8F7AD9B6CD4CF117A71E277E2EC2931 │ └───────────────────────────────────────────┘ @@ -525,7 +563,7 @@ Result: Calculates `xxHash` from a string. It is proposed in two flavors, 32 and 64 bits. -``` sql +```sql SELECT xxHash32('') OR @@ -543,13 +581,13 @@ Type: `xxHash`. Query: -``` sql +```sql SELECT xxHash32('Hello, world!'); ``` Result: -``` text +```response ┌─xxHash32('Hello, world!')─┐ │ 834093149 │ └───────────────────────────┘ @@ -567,7 +605,7 @@ Can be used for detection of semi-duplicate strings with [bitHammingDistance](.. **Syntax** -``` sql +```sql ngramSimHash(string[, ngramsize]) ``` @@ -586,13 +624,13 @@ Type: [UInt64](../../sql-reference/data-types/int-uint.md). Query: -``` sql +```sql SELECT ngramSimHash('ClickHouse') AS Hash; ``` Result: -``` text +```response ┌───────Hash─┐ │ 1627567969 │ └────────────┘ @@ -606,7 +644,7 @@ Can be used for detection of semi-duplicate strings with [bitHammingDistance](.. **Syntax** -``` sql +```sql ngramSimHashCaseInsensitive(string[, ngramsize]) ``` @@ -625,13 +663,13 @@ Type: [UInt64](../../sql-reference/data-types/int-uint.md). Query: -``` sql +```sql SELECT ngramSimHashCaseInsensitive('ClickHouse') AS Hash; ``` Result: -``` text +```response ┌──────Hash─┐ │ 562180645 │ └───────────┘ @@ -645,7 +683,7 @@ Can be used for detection of semi-duplicate strings with [bitHammingDistance](.. **Syntax** -``` sql +```sql ngramSimHashUTF8(string[, ngramsize]) ``` @@ -664,13 +702,13 @@ Type: [UInt64](../../sql-reference/data-types/int-uint.md). Query: -``` sql +```sql SELECT ngramSimHashUTF8('ClickHouse') AS Hash; ``` Result: -``` text +```response ┌───────Hash─┐ │ 1628157797 │ └────────────┘ @@ -684,7 +722,7 @@ Can be used for detection of semi-duplicate strings with [bitHammingDistance](.. **Syntax** -``` sql +```sql ngramSimHashCaseInsensitiveUTF8(string[, ngramsize]) ``` @@ -703,13 +741,13 @@ Type: [UInt64](../../sql-reference/data-types/int-uint.md). Query: -``` sql +```sql SELECT ngramSimHashCaseInsensitiveUTF8('ClickHouse') AS Hash; ``` Result: -``` text +```response ┌───────Hash─┐ │ 1636742693 │ └────────────┘ @@ -723,7 +761,7 @@ Can be used for detection of semi-duplicate strings with [bitHammingDistance](.. **Syntax** -``` sql +```sql wordShingleSimHash(string[, shinglesize]) ``` @@ -742,13 +780,13 @@ Type: [UInt64](../../sql-reference/data-types/int-uint.md). Query: -``` sql +```sql SELECT wordShingleSimHash('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).') AS Hash; ``` Result: -``` text +```response ┌───────Hash─┐ │ 2328277067 │ └────────────┘ @@ -762,7 +800,7 @@ Can be used for detection of semi-duplicate strings with [bitHammingDistance](.. **Syntax** -``` sql +```sql wordShingleSimHashCaseInsensitive(string[, shinglesize]) ``` @@ -781,13 +819,13 @@ Type: [UInt64](../../sql-reference/data-types/int-uint.md). Query: -``` sql +```sql SELECT wordShingleSimHashCaseInsensitive('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).') AS Hash; ``` Result: -``` text +```response ┌───────Hash─┐ │ 2194812424 │ └────────────┘ @@ -801,7 +839,7 @@ Can be used for detection of semi-duplicate strings with [bitHammingDistance](.. **Syntax** -``` sql +```sql wordShingleSimHashUTF8(string[, shinglesize]) ``` @@ -820,13 +858,13 @@ Type: [UInt64](../../sql-reference/data-types/int-uint.md). Query: -``` sql +```sql SELECT wordShingleSimHashUTF8('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).') AS Hash; ``` Result: -``` text +```response ┌───────Hash─┐ │ 2328277067 │ └────────────┘ @@ -840,7 +878,7 @@ Can be used for detection of semi-duplicate strings with [bitHammingDistance](.. **Syntax** -``` sql +```sql wordShingleSimHashCaseInsensitiveUTF8(string[, shinglesize]) ``` @@ -859,13 +897,13 @@ Type: [UInt64](../../sql-reference/data-types/int-uint.md). Query: -``` sql +```sql SELECT wordShingleSimHashCaseInsensitiveUTF8('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).') AS Hash; ``` Result: -``` text +```response ┌───────Hash─┐ │ 2194812424 │ └────────────┘ @@ -879,7 +917,7 @@ Can be used for detection of semi-duplicate strings with [tupleHammingDistance]( **Syntax** -``` sql +```sql ngramMinHash(string[, ngramsize, hashnum]) ``` @@ -899,13 +937,13 @@ Type: [Tuple](../../sql-reference/data-types/tuple.md)([UInt64](../../sql-refere Query: -``` sql +```sql SELECT ngramMinHash('ClickHouse') AS Tuple; ``` Result: -``` text +```response ┌─Tuple──────────────────────────────────────┐ │ (18333312859352735453,9054248444481805918) │ └────────────────────────────────────────────┘ @@ -919,7 +957,7 @@ Can be used for detection of semi-duplicate strings with [tupleHammingDistance]( **Syntax** -``` sql +```sql ngramMinHashCaseInsensitive(string[, ngramsize, hashnum]) ``` @@ -939,13 +977,13 @@ Type: [Tuple](../../sql-reference/data-types/tuple.md)([UInt64](../../sql-refere Query: -``` sql +```sql SELECT ngramMinHashCaseInsensitive('ClickHouse') AS Tuple; ``` Result: -``` text +```response ┌─Tuple──────────────────────────────────────┐ │ (2106263556442004574,13203602793651726206) │ └────────────────────────────────────────────┘ @@ -959,7 +997,7 @@ Can be used for detection of semi-duplicate strings with [tupleHammingDistance]( **Syntax** -``` sql +```sql ngramMinHashUTF8(string[, ngramsize, hashnum]) ``` @@ -979,13 +1017,13 @@ Type: [Tuple](../../sql-reference/data-types/tuple.md)([UInt64](../../sql-refere Query: -``` sql +```sql SELECT ngramMinHashUTF8('ClickHouse') AS Tuple; ``` Result: -``` text +```response ┌─Tuple──────────────────────────────────────┐ │ (18333312859352735453,6742163577938632877) │ └────────────────────────────────────────────┘ @@ -999,7 +1037,7 @@ Can be used for detection of semi-duplicate strings with [tupleHammingDistance]( **Syntax** -``` sql +```sql ngramMinHashCaseInsensitiveUTF8(string [, ngramsize, hashnum]) ``` @@ -1019,13 +1057,13 @@ Type: [Tuple](../../sql-reference/data-types/tuple.md)([UInt64](../../sql-refere Query: -``` sql +```sql SELECT ngramMinHashCaseInsensitiveUTF8('ClickHouse') AS Tuple; ``` Result: -``` text +```response ┌─Tuple───────────────────────────────────────┐ │ (12493625717655877135,13203602793651726206) │ └─────────────────────────────────────────────┘ @@ -1037,7 +1075,7 @@ Splits a ASCII string into n-grams of `ngramsize` symbols and returns the n-gram **Syntax** -``` sql +```sql ngramMinHashArg(string[, ngramsize, hashnum]) ``` @@ -1057,13 +1095,13 @@ Type: [Tuple](../../sql-reference/data-types/tuple.md)([Tuple](../../sql-referen Query: -``` sql +```sql SELECT ngramMinHashArg('ClickHouse') AS Tuple; ``` Result: -``` text +```response ┌─Tuple─────────────────────────────────────────────────────────────────────────┐ │ (('ous','ick','lic','Hou','kHo','use'),('Hou','lic','ick','ous','ckH','Cli')) │ └───────────────────────────────────────────────────────────────────────────────┘ @@ -1075,7 +1113,7 @@ Splits a ASCII string into n-grams of `ngramsize` symbols and returns the n-gram **Syntax** -``` sql +```sql ngramMinHashArgCaseInsensitive(string[, ngramsize, hashnum]) ``` @@ -1095,13 +1133,13 @@ Type: [Tuple](../../sql-reference/data-types/tuple.md)([Tuple](../../sql-referen Query: -``` sql +```sql SELECT ngramMinHashArgCaseInsensitive('ClickHouse') AS Tuple; ``` Result: -``` text +```response ┌─Tuple─────────────────────────────────────────────────────────────────────────┐ │ (('ous','ick','lic','kHo','use','Cli'),('kHo','lic','ick','ous','ckH','Hou')) │ └───────────────────────────────────────────────────────────────────────────────┘ @@ -1113,7 +1151,7 @@ Splits a UTF-8 string into n-grams of `ngramsize` symbols and returns the n-gram **Syntax** -``` sql +```sql ngramMinHashArgUTF8(string[, ngramsize, hashnum]) ``` @@ -1133,13 +1171,13 @@ Type: [Tuple](../../sql-reference/data-types/tuple.md)([Tuple](../../sql-referen Query: -``` sql +```sql SELECT ngramMinHashArgUTF8('ClickHouse') AS Tuple; ``` Result: -``` text +```response ┌─Tuple─────────────────────────────────────────────────────────────────────────┐ │ (('ous','ick','lic','Hou','kHo','use'),('kHo','Hou','lic','ick','ous','ckH')) │ └───────────────────────────────────────────────────────────────────────────────┘ @@ -1151,7 +1189,7 @@ Splits a UTF-8 string into n-grams of `ngramsize` symbols and returns the n-gram **Syntax** -``` sql +```sql ngramMinHashArgCaseInsensitiveUTF8(string[, ngramsize, hashnum]) ``` @@ -1171,13 +1209,13 @@ Type: [Tuple](../../sql-reference/data-types/tuple.md)([Tuple](../../sql-referen Query: -``` sql +```sql SELECT ngramMinHashArgCaseInsensitiveUTF8('ClickHouse') AS Tuple; ``` Result: -``` text +```response ┌─Tuple─────────────────────────────────────────────────────────────────────────┐ │ (('ckH','ous','ick','lic','kHo','use'),('kHo','lic','ick','ous','ckH','Hou')) │ └───────────────────────────────────────────────────────────────────────────────┘ @@ -1191,7 +1229,7 @@ Can be used for detection of semi-duplicate strings with [tupleHammingDistance]( **Syntax** -``` sql +```sql wordShingleMinHash(string[, shinglesize, hashnum]) ``` @@ -1211,13 +1249,13 @@ Type: [Tuple](../../sql-reference/data-types/tuple.md)([UInt64](../../sql-refere Query: -``` sql +```sql SELECT wordShingleMinHash('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).') AS Tuple; ``` Result: -``` text +```response ┌─Tuple──────────────────────────────────────┐ │ (16452112859864147620,5844417301642981317) │ └────────────────────────────────────────────┘ @@ -1231,7 +1269,7 @@ Can be used for detection of semi-duplicate strings with [tupleHammingDistance]( **Syntax** -``` sql +```sql wordShingleMinHashCaseInsensitive(string[, shinglesize, hashnum]) ``` @@ -1251,13 +1289,13 @@ Type: [Tuple](../../sql-reference/data-types/tuple.md)([UInt64](../../sql-refere Query: -``` sql +```sql SELECT wordShingleMinHashCaseInsensitive('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).') AS Tuple; ``` Result: -``` text +```response ┌─Tuple─────────────────────────────────────┐ │ (3065874883688416519,1634050779997673240) │ └───────────────────────────────────────────┘ @@ -1271,7 +1309,7 @@ Can be used for detection of semi-duplicate strings with [tupleHammingDistance]( **Syntax** -``` sql +```sql wordShingleMinHashUTF8(string[, shinglesize, hashnum]) ``` @@ -1291,13 +1329,13 @@ Type: [Tuple](../../sql-reference/data-types/tuple.md)([UInt64](../../sql-refere Query: -``` sql +```sql SELECT wordShingleMinHashUTF8('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).') AS Tuple; ``` Result: -``` text +```response ┌─Tuple──────────────────────────────────────┐ │ (16452112859864147620,5844417301642981317) │ └────────────────────────────────────────────┘ @@ -1311,7 +1349,7 @@ Can be used for detection of semi-duplicate strings with [tupleHammingDistance]( **Syntax** -``` sql +```sql wordShingleMinHashCaseInsensitiveUTF8(string[, shinglesize, hashnum]) ``` @@ -1331,13 +1369,13 @@ Type: [Tuple](../../sql-reference/data-types/tuple.md)([UInt64](../../sql-refere Query: -``` sql +```sql SELECT wordShingleMinHashCaseInsensitiveUTF8('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).') AS Tuple; ``` Result: -``` text +```response ┌─Tuple─────────────────────────────────────┐ │ (3065874883688416519,1634050779997673240) │ └───────────────────────────────────────────┘ @@ -1349,7 +1387,7 @@ Splits a ASCII string into parts (shingles) of `shinglesize` words each and retu **Syntax** -``` sql +```sql wordShingleMinHashArg(string[, shinglesize, hashnum]) ``` @@ -1369,13 +1407,13 @@ Type: [Tuple](../../sql-reference/data-types/tuple.md)([Tuple](../../sql-referen Query: -``` sql +```sql SELECT wordShingleMinHashArg('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).', 1, 3) AS Tuple; ``` Result: -``` text +```response ┌─Tuple─────────────────────────────────────────────────────────────────┐ │ (('OLAP','database','analytical'),('online','oriented','processing')) │ └───────────────────────────────────────────────────────────────────────┘ @@ -1387,7 +1425,7 @@ Splits a ASCII string into parts (shingles) of `shinglesize` words each and retu **Syntax** -``` sql +```sql wordShingleMinHashArgCaseInsensitive(string[, shinglesize, hashnum]) ``` @@ -1407,13 +1445,13 @@ Type: [Tuple](../../sql-reference/data-types/tuple.md)([Tuple](../../sql-referen Query: -``` sql +```sql SELECT wordShingleMinHashArgCaseInsensitive('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).', 1, 3) AS Tuple; ``` Result: -``` text +```response ┌─Tuple──────────────────────────────────────────────────────────────────┐ │ (('queries','database','analytical'),('oriented','processing','DBMS')) │ └────────────────────────────────────────────────────────────────────────┘ @@ -1425,7 +1463,7 @@ Splits a UTF-8 string into parts (shingles) of `shinglesize` words each and retu **Syntax** -``` sql +```sql wordShingleMinHashArgUTF8(string[, shinglesize, hashnum]) ``` @@ -1445,13 +1483,13 @@ Type: [Tuple](../../sql-reference/data-types/tuple.md)([Tuple](../../sql-referen Query: -``` sql +```sql SELECT wordShingleMinHashArgUTF8('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).', 1, 3) AS Tuple; ``` Result: -``` text +```response ┌─Tuple─────────────────────────────────────────────────────────────────┐ │ (('OLAP','database','analytical'),('online','oriented','processing')) │ └───────────────────────────────────────────────────────────────────────┘ @@ -1463,7 +1501,7 @@ Splits a UTF-8 string into parts (shingles) of `shinglesize` words each and retu **Syntax** -``` sql +```sql wordShingleMinHashArgCaseInsensitiveUTF8(string[, shinglesize, hashnum]) ``` @@ -1483,13 +1521,13 @@ Type: [Tuple](../../sql-reference/data-types/tuple.md)([Tuple](../../sql-referen Query: -``` sql +```sql SELECT wordShingleMinHashArgCaseInsensitiveUTF8('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).', 1, 3) AS Tuple; ``` Result: -``` text +```response ┌─Tuple──────────────────────────────────────────────────────────────────┐ │ (('queries','database','analytical'),('oriented','processing','DBMS')) │ └────────────────────────────────────────────────────────────────────────┘ diff --git a/docs/ru/sql-reference/functions/hash-functions.md b/docs/ru/sql-reference/functions/hash-functions.md index 12091fcbc3f..0065275519b 100644 --- a/docs/ru/sql-reference/functions/hash-functions.md +++ b/docs/ru/sql-reference/functions/hash-functions.md @@ -13,7 +13,7 @@ Simhash – это хеш-функция, которая для близких [Интерпретирует](../../sql-reference/functions/hash-functions.md#type_conversion_functions-reinterpretAsString) все входные параметры как строки и вычисляет хэш [MD5](https://ru.wikipedia.org/wiki/MD5) для каждой из них. Затем объединяет хэши, берет первые 8 байт хэша результирующей строки и интерпретирует их как значение типа `UInt64` с big-endian порядком байтов. -``` sql +```sql halfMD5(par1, ...) ``` @@ -30,11 +30,11 @@ halfMD5(par1, ...) **Пример** -``` sql +```sql SELECT halfMD5(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS halfMD5hash, toTypeName(halfMD5hash) AS type; ``` -``` text +```response ┌────────halfMD5hash─┬─type───┐ │ 186182704141653334 │ UInt64 │ └────────────────────┴────────┘ @@ -54,7 +54,7 @@ SELECT halfMD5(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00') Генерирует 64-х битное значение [SipHash](https://131002.net/siphash/). -``` sql +```sql sipHash64(par1,...) ``` @@ -77,11 +77,11 @@ sipHash64(par1,...) **Пример** -``` sql +```sql SELECT sipHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS SipHash, toTypeName(SipHash) AS type; ``` -``` text +```response ┌──────────────SipHash─┬─type───┐ │ 13726873534472839665 │ UInt64 │ └──────────────────────┴────────┘ @@ -93,7 +93,7 @@ SELECT sipHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00 **Синтаксис** -``` sql +```sql sipHash128(par1,...) ``` @@ -111,13 +111,13 @@ sipHash128(par1,...) Запрос: -``` sql +```sql SELECT hex(sipHash128('foo', '\x01', 3)); ``` Результат: -``` text +```response ┌─hex(sipHash128('foo', '', 3))────┐ │ 9DE516A64A414D4B1B609415E4523F24 │ └──────────────────────────────────┘ @@ -127,7 +127,7 @@ SELECT hex(sipHash128('foo', '\x01', 3)); Генерирует 64-х битное значение [CityHash](https://github.com/google/cityhash). -``` sql +```sql cityHash64(par1,...) ``` @@ -145,11 +145,11 @@ cityHash64(par1,...) Пример вызова: -``` sql +```sql SELECT cityHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS CityHash, toTypeName(CityHash) AS type; ``` -``` text +```response ┌─────────────CityHash─┬─type───┐ │ 12072650598913549138 │ UInt64 │ └──────────────────────┴────────┘ @@ -157,7 +157,7 @@ SELECT cityHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:0 А вот так вы можете вычислить чексумму всей таблицы с точностью до порядка строк: -``` sql +```sql SELECT groupBitXor(cityHash64(*)) FROM table ``` @@ -177,7 +177,7 @@ SELECT groupBitXor(cityHash64(*)) FROM table **Синтаксис** -``` sql +```sql SHA1('s') ... SHA512('s') @@ -203,18 +203,56 @@ SHA512('s') Запрос: -``` sql +```sql SELECT hex(SHA1('abc')); ``` Результат: -``` text +```response ┌─hex(SHA1('abc'))─────────────────────────┐ │ A9993E364706816ABA3E25717850C26C9CD0D89D │ └──────────────────────────────────────────┘ ``` +## BLAKE3 {#blake3} + +Вычисляет BLAKE3 хеш строки и возвращает полученный набор байт в виде [FixedString](../data-types/fixedstring.md). + +**Синтаксис** + +```sql +BLAKE3('s') +``` + +Данная криптографическая функция интегрирована в ClickHouse из Rust-библиотеки. Функция работает сравнительно быстро, показывая в 2 раза более быстрые результаты по сравнению с SHA-2, генерируя хеши аналогичной SHA-256 длины. + +**Параметры** + +- s - входная строка для вычисления хеша BLAKE3. [String](../data-types/string.md). + +**Возвращаемое значение** + +- Хеш BLAKE3 в виде шестнадцатеричной строки, имеющей тип FixedString(32). + +Тип: [FixedString](../data-types/fixedstring.md). + +**Пример** + +Используйте функцию [hex](../functions/encoding-functions.md#hex) для представления результата в виде строки с шестнадцатеричной кодировкой. + +Запрос: +```sql +SELECT hex(BLAKE3('ABC')) +``` + +Результат: +```response +┌─hex(BLAKE3('ABC'))───────────────────────────────────────────────┐ +│ D1717274597CF0289694F75D96D444B992A096F1AFD8E7BBFA6EBB1D360FEDFC │ +└──────────────────────────────────────────────────────────────────┘ +``` + ## URLHash(url\[, N\]) {#urlhashurl-n} Быстрая не криптографическая хэш-функция неплохого качества для строки, полученной из URL путём некоторой нормализации. @@ -228,7 +266,7 @@ SELECT hex(SHA1('abc')); Создает 64-битное значение [FarmHash](https://github.com/google/farmhash), независимое от платформы (архитектуры сервера), что важно, если значения сохраняются или используются для разбиения данных на группы. -``` sql +```sql farmFingerprint64(par1, ...) farmHash64(par1, ...) ``` @@ -245,11 +283,11 @@ farmHash64(par1, ...) **Пример** -``` sql +```sql SELECT farmHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS FarmHash, toTypeName(FarmHash) AS type; ``` -``` text +```response ┌─────────────FarmHash─┬─type───┐ │ 17790458267262532859 │ UInt64 │ └──────────────────────┴────────┘ @@ -259,7 +297,7 @@ SELECT farmHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:0 Вычисляет [JavaHash](http://hg.openjdk.java.net/jdk8u/jdk8u/jdk/file/478a4add975b/src/share/classes/java/lang/String.java#l1452) от строки. `JavaHash` не отличается ни скоростью, ни качеством, поэтому эту функцию следует считать устаревшей. Используйте эту функцию, если вам необходимо получить значение хэша по такому же алгоритму. -``` sql +```sql SELECT javaHash('') ``` @@ -273,13 +311,13 @@ SELECT javaHash('') Запрос: -``` sql +```sql SELECT javaHash('Hello, world!'); ``` Результат: -``` text +```response ┌─javaHash('Hello, world!')─┐ │ -1880044555 │ └───────────────────────────┘ @@ -291,7 +329,7 @@ SELECT javaHash('Hello, world!'); **Синтаксис** -``` sql +```sql javaHashUTF16LE(stringUtf16le) ``` @@ -311,13 +349,13 @@ javaHashUTF16LE(stringUtf16le) Запрос: -``` sql +```sql SELECT javaHashUTF16LE(convertCharset('test', 'utf-8', 'utf-16le')); ``` Результат: -``` text +```response ┌─javaHashUTF16LE(convertCharset('test', 'utf-8', 'utf-16le'))─┐ │ 3556498 │ └──────────────────────────────────────────────────────────────┘ @@ -327,7 +365,7 @@ SELECT javaHashUTF16LE(convertCharset('test', 'utf-8', 'utf-16le')); Вычисляет `HiveHash` от строки. -``` sql +```sql SELECT hiveHash('') ``` @@ -343,13 +381,13 @@ SELECT hiveHash('') Запрос: -``` sql +```sql SELECT hiveHash('Hello, world!'); ``` Результат: -``` text +```response ┌─hiveHash('Hello, world!')─┐ │ 267439093 │ └───────────────────────────┘ @@ -359,7 +397,7 @@ SELECT hiveHash('Hello, world!'); Генерирует 64-х битное значение [MetroHash](http://www.jandrewrogers.com/2015/05/27/metrohash/). -``` sql +```sql metroHash64(par1, ...) ``` @@ -373,11 +411,11 @@ metroHash64(par1, ...) **Пример** -``` sql +```sql SELECT metroHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MetroHash, toTypeName(MetroHash) AS type; ``` -``` text +```response ┌────────────MetroHash─┬─type───┐ │ 14235658766382344533 │ UInt64 │ └──────────────────────┴────────┘ @@ -393,7 +431,7 @@ SELECT metroHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00: Генерирует значение [MurmurHash2](https://github.com/aappleby/smhasher). -``` sql +```sql murmurHash2_32(par1, ...) murmurHash2_64(par1, ...) ``` @@ -409,11 +447,11 @@ murmurHash2_64(par1, ...) **Пример** -``` sql +```sql SELECT murmurHash2_64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MurmurHash2, toTypeName(MurmurHash2) AS type; ``` -``` text +```response ┌──────────MurmurHash2─┬─type───┐ │ 11832096901709403633 │ UInt64 │ └──────────────────────┴────────┘ @@ -425,7 +463,7 @@ SELECT murmurHash2_64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23: **Синтаксис** -``` sql +```sql gccMurmurHash(par1, ...); ``` @@ -443,7 +481,7 @@ gccMurmurHash(par1, ...); Запрос: -``` sql +```sql SELECT gccMurmurHash(1, 2, 3) AS res1, gccMurmurHash(('a', [1, 2, 3], 4, (4, ['foo', 'bar'], 1, (1, 2)))) AS res2 @@ -451,7 +489,7 @@ SELECT Результат: -``` text +```response ┌─────────────────res1─┬────────────────res2─┐ │ 12384823029245979431 │ 1188926775431157506 │ └──────────────────────┴─────────────────────┘ @@ -461,7 +499,7 @@ SELECT Генерирует значение [MurmurHash3](https://github.com/aappleby/smhasher). -``` sql +```sql murmurHash3_32(par1, ...) murmurHash3_64(par1, ...) ``` @@ -477,11 +515,11 @@ murmurHash3_64(par1, ...) **Пример** -``` sql +```sql SELECT murmurHash3_32(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MurmurHash3, toTypeName(MurmurHash3) AS type; ``` -``` text +```response ┌─MurmurHash3─┬─type───┐ │ 2152717 │ UInt32 │ └─────────────┴────────┘ @@ -493,7 +531,7 @@ SELECT murmurHash3_32(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23: **Синтаксис** -``` sql +```sql murmurHash3_128(expr) ``` @@ -511,13 +549,13 @@ murmurHash3_128(expr) Запрос: -``` sql +```sql SELECT hex(murmurHash3_128('foo', 'foo', 'foo')); ``` Результат: -``` text +```response ┌─hex(murmurHash3_128('foo', 'foo', 'foo'))─┐ │ F8F7AD9B6CD4CF117A71E277E2EC2931 │ └───────────────────────────────────────────┘ @@ -527,7 +565,7 @@ SELECT hex(murmurHash3_128('foo', 'foo', 'foo')); Вычисляет `xxHash` от строки. Предлагается в двух вариантах: 32 и 64 бита. -``` sql +```sql SELECT xxHash32('') OR @@ -545,13 +583,13 @@ SELECT xxHash64('') Запрос: -``` sql +```sql SELECT xxHash32('Hello, world!'); ``` Результат: -``` text +```response ┌─xxHash32('Hello, world!')─┐ │ 834093149 │ └───────────────────────────┘ @@ -569,7 +607,7 @@ SELECT xxHash32('Hello, world!'); **Синтаксис** -``` sql +```sql ngramSimHash(string[, ngramsize]) ``` @@ -588,13 +626,13 @@ ngramSimHash(string[, ngramsize]) Запрос: -``` sql +```sql SELECT ngramSimHash('ClickHouse') AS Hash; ``` Результат: -``` text +```response ┌───────Hash─┐ │ 1627567969 │ └────────────┘ @@ -608,7 +646,7 @@ SELECT ngramSimHash('ClickHouse') AS Hash; **Синтаксис** -``` sql +```sql ngramSimHashCaseInsensitive(string[, ngramsize]) ``` @@ -627,13 +665,13 @@ ngramSimHashCaseInsensitive(string[, ngramsize]) Запрос: -``` sql +```sql SELECT ngramSimHashCaseInsensitive('ClickHouse') AS Hash; ``` Результат: -``` text +```response ┌──────Hash─┐ │ 562180645 │ └───────────┘ @@ -647,7 +685,7 @@ SELECT ngramSimHashCaseInsensitive('ClickHouse') AS Hash; **Синтаксис** -``` sql +```sql ngramSimHashUTF8(string[, ngramsize]) ``` @@ -666,13 +704,13 @@ ngramSimHashUTF8(string[, ngramsize]) Запрос: -``` sql +```sql SELECT ngramSimHashUTF8('ClickHouse') AS Hash; ``` Результат: -``` text +```response ┌───────Hash─┐ │ 1628157797 │ └────────────┘ @@ -686,7 +724,7 @@ SELECT ngramSimHashUTF8('ClickHouse') AS Hash; **Синтаксис** -``` sql +```sql ngramSimHashCaseInsensitiveUTF8(string[, ngramsize]) ``` @@ -705,13 +743,13 @@ ngramSimHashCaseInsensitiveUTF8(string[, ngramsize]) Запрос: -``` sql +```sql SELECT ngramSimHashCaseInsensitiveUTF8('ClickHouse') AS Hash; ``` Результат: -``` text +```response ┌───────Hash─┐ │ 1636742693 │ └────────────┘ @@ -725,7 +763,7 @@ SELECT ngramSimHashCaseInsensitiveUTF8('ClickHouse') AS Hash; **Синтаксис** -``` sql +```sql wordShingleSimHash(string[, shinglesize]) ``` @@ -744,13 +782,13 @@ wordShingleSimHash(string[, shinglesize]) Запрос: -``` sql +```sql SELECT wordShingleSimHash('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).') AS Hash; ``` Результат: -``` text +```response ┌───────Hash─┐ │ 2328277067 │ └────────────┘ @@ -764,7 +802,7 @@ SELECT wordShingleSimHash('ClickHouse® is a column-oriented database management **Синтаксис** -``` sql +```sql wordShingleSimHashCaseInsensitive(string[, shinglesize]) ``` @@ -783,13 +821,13 @@ wordShingleSimHashCaseInsensitive(string[, shinglesize]) Запрос: -``` sql +```sql SELECT wordShingleSimHashCaseInsensitive('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).') AS Hash; ``` Результат: -``` text +```response ┌───────Hash─┐ │ 2194812424 │ └────────────┘ @@ -803,7 +841,7 @@ SELECT wordShingleSimHashCaseInsensitive('ClickHouse® is a column-oriented data **Синтаксис** -``` sql +```sql wordShingleSimHashUTF8(string[, shinglesize]) ``` @@ -822,13 +860,13 @@ wordShingleSimHashUTF8(string[, shinglesize]) Запрос: -``` sql +```sql SELECT wordShingleSimHashUTF8('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).') AS Hash; ``` Результат: -``` text +```response ┌───────Hash─┐ │ 2328277067 │ └────────────┘ @@ -842,7 +880,7 @@ SELECT wordShingleSimHashUTF8('ClickHouse® is a column-oriented database manage **Синтаксис** -``` sql +```sql wordShingleSimHashCaseInsensitiveUTF8(string[, shinglesize]) ``` @@ -861,13 +899,13 @@ wordShingleSimHashCaseInsensitiveUTF8(string[, shinglesize]) Запрос: -``` sql +```sql SELECT wordShingleSimHashCaseInsensitiveUTF8('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).') AS Hash; ``` Результат: -``` text +```response ┌───────Hash─┐ │ 2194812424 │ └────────────┘ @@ -881,7 +919,7 @@ SELECT wordShingleSimHashCaseInsensitiveUTF8('ClickHouse® is a column-oriented **Синтаксис** -``` sql +```sql ngramMinHash(string[, ngramsize, hashnum]) ``` @@ -901,13 +939,13 @@ ngramMinHash(string[, ngramsize, hashnum]) Запрос: -``` sql +```sql SELECT ngramMinHash('ClickHouse') AS Tuple; ``` Результат: -``` text +```response ┌─Tuple──────────────────────────────────────┐ │ (18333312859352735453,9054248444481805918) │ └────────────────────────────────────────────┘ @@ -921,7 +959,7 @@ SELECT ngramMinHash('ClickHouse') AS Tuple; **Синтаксис** -``` sql +```sql ngramMinHashCaseInsensitive(string[, ngramsize, hashnum]) ``` @@ -941,13 +979,13 @@ ngramMinHashCaseInsensitive(string[, ngramsize, hashnum]) Запрос: -``` sql +```sql SELECT ngramMinHashCaseInsensitive('ClickHouse') AS Tuple; ``` Результат: -``` text +```response ┌─Tuple──────────────────────────────────────┐ │ (2106263556442004574,13203602793651726206) │ └────────────────────────────────────────────┘ @@ -960,7 +998,7 @@ SELECT ngramMinHashCaseInsensitive('ClickHouse') AS Tuple; Может быть использована для проверки двух строк на схожесть вместе с функцией [tupleHammingDistance](../../sql-reference/functions/tuple-functions.md#tuplehammingdistance). Если для двух строк минимальные или максимальные хеши одинаковы, мы считаем, что эти строки совпадают. **Синтаксис** -``` sql +```sql ngramMinHashUTF8(string[, ngramsize, hashnum]) ``` @@ -980,13 +1018,13 @@ ngramMinHashUTF8(string[, ngramsize, hashnum]) Запрос: -``` sql +```sql SELECT ngramMinHashUTF8('ClickHouse') AS Tuple; ``` Результат: -``` text +```response ┌─Tuple──────────────────────────────────────┐ │ (18333312859352735453,6742163577938632877) │ └────────────────────────────────────────────┘ @@ -1000,7 +1038,7 @@ SELECT ngramMinHashUTF8('ClickHouse') AS Tuple; **Синтаксис** -``` sql +```sql ngramMinHashCaseInsensitiveUTF8(string [, ngramsize, hashnum]) ``` @@ -1020,13 +1058,13 @@ ngramMinHashCaseInsensitiveUTF8(string [, ngramsize, hashnum]) Запрос: -``` sql +```sql SELECT ngramMinHashCaseInsensitiveUTF8('ClickHouse') AS Tuple; ``` Результат: -``` text +```response ┌─Tuple───────────────────────────────────────┐ │ (12493625717655877135,13203602793651726206) │ └─────────────────────────────────────────────┘ @@ -1038,7 +1076,7 @@ SELECT ngramMinHashCaseInsensitiveUTF8('ClickHouse') AS Tuple; **Синтаксис** -``` sql +```sql ngramMinHashArg(string[, ngramsize, hashnum]) ``` @@ -1058,13 +1096,13 @@ ngramMinHashArg(string[, ngramsize, hashnum]) Запрос: -``` sql +```sql SELECT ngramMinHashArg('ClickHouse') AS Tuple; ``` Результат: -``` text +```response ┌─Tuple─────────────────────────────────────────────────────────────────────────┐ │ (('ous','ick','lic','Hou','kHo','use'),('Hou','lic','ick','ous','ckH','Cli')) │ └───────────────────────────────────────────────────────────────────────────────┘ @@ -1076,7 +1114,7 @@ SELECT ngramMinHashArg('ClickHouse') AS Tuple; **Синтаксис** -``` sql +```sql ngramMinHashArgCaseInsensitive(string[, ngramsize, hashnum]) ``` @@ -1096,13 +1134,13 @@ ngramMinHashArgCaseInsensitive(string[, ngramsize, hashnum]) Запрос: -``` sql +```sql SELECT ngramMinHashArgCaseInsensitive('ClickHouse') AS Tuple; ``` Результат: -``` text +```response ┌─Tuple─────────────────────────────────────────────────────────────────────────┐ │ (('ous','ick','lic','kHo','use','Cli'),('kHo','lic','ick','ous','ckH','Hou')) │ └───────────────────────────────────────────────────────────────────────────────┘ @@ -1114,7 +1152,7 @@ SELECT ngramMinHashArgCaseInsensitive('ClickHouse') AS Tuple; **Синтаксис** -``` sql +```sql ngramMinHashArgUTF8(string[, ngramsize, hashnum]) ``` @@ -1134,13 +1172,13 @@ ngramMinHashArgUTF8(string[, ngramsize, hashnum]) Запрос: -``` sql +```sql SELECT ngramMinHashArgUTF8('ClickHouse') AS Tuple; ``` Результат: -``` text +```response ┌─Tuple─────────────────────────────────────────────────────────────────────────┐ │ (('ous','ick','lic','Hou','kHo','use'),('kHo','Hou','lic','ick','ous','ckH')) │ └───────────────────────────────────────────────────────────────────────────────┘ @@ -1152,7 +1190,7 @@ SELECT ngramMinHashArgUTF8('ClickHouse') AS Tuple; **Синтаксис** -``` sql +```sql ngramMinHashArgCaseInsensitiveUTF8(string[, ngramsize, hashnum]) ``` @@ -1172,13 +1210,13 @@ ngramMinHashArgCaseInsensitiveUTF8(string[, ngramsize, hashnum]) Запрос: -``` sql +```sql SELECT ngramMinHashArgCaseInsensitiveUTF8('ClickHouse') AS Tuple; ``` Результат: -``` text +```response ┌─Tuple─────────────────────────────────────────────────────────────────────────┐ │ (('ckH','ous','ick','lic','kHo','use'),('kHo','lic','ick','ous','ckH','Hou')) │ └───────────────────────────────────────────────────────────────────────────────┘ @@ -1192,7 +1230,7 @@ SELECT ngramMinHashArgCaseInsensitiveUTF8('ClickHouse') AS Tuple; **Синтаксис** -``` sql +```sql wordShingleMinHash(string[, shinglesize, hashnum]) ``` @@ -1212,13 +1250,13 @@ wordShingleMinHash(string[, shinglesize, hashnum]) Запрос: -``` sql +```sql SELECT wordShingleMinHash('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).') AS Tuple; ``` Результат: -``` text +```response ┌─Tuple──────────────────────────────────────┐ │ (16452112859864147620,5844417301642981317) │ └────────────────────────────────────────────┘ @@ -1232,7 +1270,7 @@ SELECT wordShingleMinHash('ClickHouse® is a column-oriented database management **Синтаксис** -``` sql +```sql wordShingleMinHashCaseInsensitive(string[, shinglesize, hashnum]) ``` @@ -1252,13 +1290,13 @@ wordShingleMinHashCaseInsensitive(string[, shinglesize, hashnum]) Запрос: -``` sql +```sql SELECT wordShingleMinHashCaseInsensitive('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).') AS Tuple; ``` Результат: -``` text +```response ┌─Tuple─────────────────────────────────────┐ │ (3065874883688416519,1634050779997673240) │ └───────────────────────────────────────────┘ @@ -1272,7 +1310,7 @@ SELECT wordShingleMinHashCaseInsensitive('ClickHouse® is a column-oriented data **Синтаксис** -``` sql +```sql wordShingleMinHashUTF8(string[, shinglesize, hashnum]) ``` @@ -1292,13 +1330,13 @@ wordShingleMinHashUTF8(string[, shinglesize, hashnum]) Запрос: -``` sql +```sql SELECT wordShingleMinHashUTF8('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).') AS Tuple; ``` Результат: -``` text +```response ┌─Tuple──────────────────────────────────────┐ │ (16452112859864147620,5844417301642981317) │ └────────────────────────────────────────────┘ @@ -1312,7 +1350,7 @@ SELECT wordShingleMinHashUTF8('ClickHouse® is a column-oriented database manage **Синтаксис** -``` sql +```sql wordShingleMinHashCaseInsensitiveUTF8(string[, shinglesize, hashnum]) ``` @@ -1332,13 +1370,13 @@ wordShingleMinHashCaseInsensitiveUTF8(string[, shinglesize, hashnum]) Запрос: -``` sql +```sql SELECT wordShingleMinHashCaseInsensitiveUTF8('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).') AS Tuple; ``` Результат: -``` text +```response ┌─Tuple─────────────────────────────────────┐ │ (3065874883688416519,1634050779997673240) │ └───────────────────────────────────────────┘ @@ -1350,7 +1388,7 @@ SELECT wordShingleMinHashCaseInsensitiveUTF8('ClickHouse® is a column-oriented **Синтаксис** -``` sql +```sql wordShingleMinHashArg(string[, shinglesize, hashnum]) ``` @@ -1370,13 +1408,13 @@ wordShingleMinHashArg(string[, shinglesize, hashnum]) Запрос: -``` sql +```sql SELECT wordShingleMinHashArg('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).', 1, 3) AS Tuple; ``` Результат: -``` text +```response ┌─Tuple─────────────────────────────────────────────────────────────────┐ │ (('OLAP','database','analytical'),('online','oriented','processing')) │ └───────────────────────────────────────────────────────────────────────┘ @@ -1388,7 +1426,7 @@ SELECT wordShingleMinHashArg('ClickHouse® is a column-oriented database managem **Синтаксис** -``` sql +```sql wordShingleMinHashArgCaseInsensitive(string[, shinglesize, hashnum]) ``` @@ -1408,13 +1446,13 @@ wordShingleMinHashArgCaseInsensitive(string[, shinglesize, hashnum]) Запрос: -``` sql +```sql SELECT wordShingleMinHashArgCaseInsensitive('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).', 1, 3) AS Tuple; ``` Результат: -``` text +```response ┌─Tuple──────────────────────────────────────────────────────────────────┐ │ (('queries','database','analytical'),('oriented','processing','DBMS')) │ └────────────────────────────────────────────────────────────────────────┘ @@ -1426,7 +1464,7 @@ SELECT wordShingleMinHashArgCaseInsensitive('ClickHouse® is a column-oriented d **Синтаксис** -``` sql +```sql wordShingleMinHashArgUTF8(string[, shinglesize, hashnum]) ``` @@ -1446,13 +1484,13 @@ wordShingleMinHashArgUTF8(string[, shinglesize, hashnum]) Запрос: -``` sql +```sql SELECT wordShingleMinHashArgUTF8('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).', 1, 3) AS Tuple; ``` Результат: -``` text +```response ┌─Tuple─────────────────────────────────────────────────────────────────┐ │ (('OLAP','database','analytical'),('online','oriented','processing')) │ └───────────────────────────────────────────────────────────────────────┘ @@ -1464,7 +1502,7 @@ SELECT wordShingleMinHashArgUTF8('ClickHouse® is a column-oriented database man **Синтаксис** -``` sql +```sql wordShingleMinHashArgCaseInsensitiveUTF8(string[, shinglesize, hashnum]) ``` @@ -1484,13 +1522,13 @@ wordShingleMinHashArgCaseInsensitiveUTF8(string[, shinglesize, hashnum]) Запрос: -``` sql +```sql SELECT wordShingleMinHashArgCaseInsensitiveUTF8('ClickHouse® is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).', 1, 3) AS Tuple; ``` Результат: -``` text +```response ┌─Tuple──────────────────────────────────────────────────────────────────┐ │ (('queries','database','analytical'),('oriented','processing','DBMS')) │ └────────────────────────────────────────────────────────────────────────┘ From 096b4626d652f426adaa09c5c41e5d1aa8815cf5 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Wed, 11 May 2022 21:18:41 +0200 Subject: [PATCH 123/171] Print more info in mismatching array sizes error message --- src/Functions/array/arrayDistance.cpp | 28 ++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index 26cfd0c3244..ae7d89f8db4 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -70,7 +70,7 @@ public: { const auto * array_type = checkAndGetDataType(argument.type.get()); if (!array_type) - throw Exception("Argument of function " + getName() + " must be array. ", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Argument of function {} must be array.", getName()); types.push_back(array_type->getNestedType()); } @@ -90,10 +90,10 @@ public: case TypeIndex::Float64: return std::make_shared(); default: - throw Exception( - "Arguments of function " + getName() + " has nested type " + common_type->getName() - + ". Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Arguments of function {} has nested type {}. " + "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", + getName(), common_type->getName()); } } @@ -139,8 +139,9 @@ private: if (mx.rows() && my.rows() && mx.rows() != my.rows()) { - throw Exception( - "Arguments of function " + getName() + " have different array sizes.", ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH); + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, + "Arguments of function {} have different array sizes: {} and {}", + getName(), mx.rows(), my.rows()); } auto & data = assert_cast &>(*column).getData(); Kernel::compute(mx, my, data); @@ -182,10 +183,10 @@ private: fillMatrix(mat, array); break; default: - throw Exception( - "Arguments of function " + getName() + " has nested type " + nested_type->getName() - + ". Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Arguments of function {} has nested type {}. " + "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", + getName(), nested_type->getName()); } } @@ -210,8 +211,9 @@ private: for (ColumnArray::Offset off : offsets) { if (off - prev != rows) - throw Exception( - "Arguments of function " + getName() + " have different array sizes.", ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH); + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, + "Arrays in a column passed to function {} have different sizes: {} and {}", + getName(), rows, off - prev); for (ColumnArray::Offset row = 0; row < off - prev; ++row) { From 8ceb63ee6c9de9e274182d1c682b7605a9377fc3 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 9 Feb 2022 16:32:52 +0000 Subject: [PATCH 124/171] Added JIT compilation of SortDescription --- src/Columns/ColumnNullable.cpp | 54 ++++++++ src/Columns/ColumnNullable.h | 12 ++ src/Columns/ColumnVector.cpp | 45 +++++++ src/Columns/ColumnVector.h | 10 ++ src/Columns/IColumn.h | 20 +++ src/Core/Settings.h | 2 + src/Core/SortCursor.h | 46 ++++++- src/Core/SortDescription.cpp | 100 ++++++++++++++ src/Core/SortDescription.h | 12 +- src/Interpreters/InterpreterSelectQuery.cpp | 6 +- src/Interpreters/JIT/compileFunction.cpp | 124 ++++++++++++++++++ src/Interpreters/JIT/compileFunction.h | 17 +++ src/Interpreters/MergeJoin.cpp | 2 +- src/Interpreters/SortedBlocksWriter.cpp | 5 + .../Algorithms/MergingSortedAlgorithm.cpp | 14 +- .../Algorithms/MergingSortedAlgorithm.h | 1 + .../Merges/MergingSortedTransform.cpp | 2 + .../Merges/MergingSortedTransform.h | 1 + .../QueryPlan/ReadFromMergeTree.cpp | 6 +- src/Processors/QueryPlan/SortingStep.cpp | 19 ++- src/Processors/QueryPlan/SortingStep.h | 4 + .../Transforms/FinishSortingTransform.cpp | 3 +- .../Transforms/FinishSortingTransform.h | 1 + .../Transforms/MergeSortingTransform.cpp | 4 +- .../Transforms/MergeSortingTransform.h | 1 + .../Transforms/SortingTransform.cpp | 17 ++- src/Processors/Transforms/SortingTransform.h | 5 +- .../gtest_blocks_size_merging_streams.cpp | 4 +- src/Storages/MergeTree/MergeTask.cpp | 3 +- 29 files changed, 520 insertions(+), 20 deletions(-) diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 214a9a807f5..a16307f6d6d 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -10,6 +10,12 @@ #include #include +#if USE_EMBEDDED_COMPILER +#include + +#include +#endif + namespace DB { @@ -241,6 +247,54 @@ ColumnPtr ColumnNullable::index(const IColumn & indexes, size_t limit) const return ColumnNullable::create(indexed_data, indexed_null_map); } +#if USE_EMBEDDED_COMPILER + +bool ColumnNullable::isComparatorCompilable() const +{ + return nested_column->isComparatorCompilable(); +} + +llvm::Value * ColumnNullable::compileComparator(llvm::IRBuilderBase & builder, llvm::Value * lhs, llvm::Value * rhs, llvm::Value * nan_direction_hint) const +{ + llvm::IRBuilder<> & b = static_cast &>(builder); + auto * head = b.GetInsertBlock(); + + llvm::Value * lhs_unwrapped_value = b.CreateExtractValue(lhs, {0}); + llvm::Value * lhs_is_null_value = b.CreateExtractValue(lhs, {1}); + + llvm::Value * rhs_unwrapped_value = b.CreateExtractValue(rhs, {0}); + llvm::Value * rhs_is_null_value = b.CreateExtractValue(rhs, {1}); + + llvm::Value * lhs_or_rhs_are_null = b.CreateOr(lhs_is_null_value, rhs_is_null_value); + + auto * lhs_or_rhs_are_null_block = llvm::BasicBlock::Create(head->getContext(), "lhs_or_rhs_are_null_block", head->getParent()); + auto * lhs_rhs_are_not_null_block = llvm::BasicBlock::Create(head->getContext(), "lhs_and_rhs_are_not_null_block", head->getParent()); + auto * join_block = llvm::BasicBlock::Create(head->getContext(), "join_block", head->getParent()); + + b.CreateCondBr(lhs_or_rhs_are_null, lhs_or_rhs_are_null_block, lhs_rhs_are_not_null_block); + + b.SetInsertPoint(lhs_or_rhs_are_null_block); + auto * lhs_equals_rhs_result = llvm::ConstantInt::getSigned(b.getInt8Ty(), 0); + llvm::Value * lhs_and_rhs_are_null = b.CreateOr(lhs_is_null_value, rhs_is_null_value); + llvm::Value * lhs_is_null_result = b.CreateSelect(lhs_is_null_value, nan_direction_hint, b.CreateNeg(nan_direction_hint)); + llvm::Value * lhs_or_rhs_are_null_block_result = b.CreateSelect(lhs_and_rhs_are_null, lhs_equals_rhs_result, lhs_is_null_result); + b.CreateBr(join_block); + + b.SetInsertPoint(lhs_rhs_are_not_null_block); + llvm::Value *lhs_rhs_are_not_null_block_result = nested_column->compileComparator(builder, lhs_unwrapped_value, rhs_unwrapped_value, nan_direction_hint); + b.CreateBr(join_block); + + b.SetInsertPoint(join_block); + + auto * result = b.CreatePHI(b.getInt8Ty(), 2); + result->addIncoming(lhs_or_rhs_are_null_block_result, lhs_or_rhs_are_null_block); + result->addIncoming(lhs_rhs_are_not_null_block_result, lhs_rhs_are_not_null_block); + + return result; +} + +#endif + int ColumnNullable::compareAtImpl(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint, const Collator * collator) const { /// NULL values share the properties of NaN values. diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h index a0c72182f63..60951dfcc2e 100644 --- a/src/Columns/ColumnNullable.h +++ b/src/Columns/ColumnNullable.h @@ -6,6 +6,9 @@ #include #include +#include "config_core.h" + + class Collator; namespace DB @@ -94,6 +97,15 @@ public: ColumnPtr permute(const Permutation & perm, size_t limit) const override; ColumnPtr index(const IColumn & indexes, size_t limit) const override; int compareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const override; + +#if USE_EMBEDDED_COMPILER + + bool isComparatorCompilable() const override; + + llvm::Value * compileComparator(llvm::IRBuilderBase & /*builder*/, llvm::Value * /*lhs*/, llvm::Value * /*rhs*/, llvm::Value * /*nan_direction_hint*/) const override; + +#endif + void compareColumn(const IColumn & rhs, size_t rhs_row_num, PaddedPODArray * row_indexes, PaddedPODArray & compare_results, int direction, int nan_direction_hint) const override; diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 90b821bfe06..2828690130c 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -25,6 +25,13 @@ # include #endif +#if USE_EMBEDDED_COMPILER +#include + +#include +#endif + + namespace DB { @@ -183,6 +190,44 @@ namespace }; } +#if USE_EMBEDDED_COMPILER + +template +bool ColumnVector::isComparatorCompilable() const +{ + /// TODO: for std::is_floating_point_v we need implement is_nan in LLVM IR. + return std::is_arithmetic_v; +} + +template +llvm::Value * ColumnVector::compileComparator(llvm::IRBuilderBase & builder, llvm::Value * lhs, llvm::Value * rhs, llvm::Value *) const +{ + llvm::IRBuilder<> & b = static_cast &>(builder); + + if constexpr (std::is_arithmetic_v) + { + // a > b ? 1 : (a < b ? -1 : 0); + + bool is_signed = std::is_signed_v; + + auto * lhs_greater_than_rhs_result = llvm::ConstantInt::getSigned(b.getInt8Ty(), 1); + auto * lhs_less_than_rhs_result = llvm::ConstantInt::getSigned(b.getInt8Ty(), -1); + auto * lhs_equals_rhs_result = llvm::ConstantInt::getSigned(b.getInt8Ty(), 0); + + auto * lhs_greater_than_rhs = is_signed ? b.CreateICmpSGT(lhs, rhs) : b.CreateICmpUGT(lhs, rhs); + auto * lhs_less_than_rhs = is_signed ? b.CreateICmpSLT(lhs, rhs) : b.CreateICmpULT(lhs, rhs); + auto * if_lhs_less_than_rhs_result = b.CreateSelect(lhs_less_than_rhs, lhs_less_than_rhs_result, lhs_equals_rhs_result); + + return b.CreateSelect(lhs_greater_than_rhs, lhs_greater_than_rhs_result, if_lhs_less_than_rhs_result); + } + else + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "Method compileComparator is not supported for type {}", TypeName); + return nullptr; + } +} + +#endif template void ColumnVector::getPermutation(IColumn::PermutationSortDirection direction, IColumn::PermutationSortStability stability, diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index ea9eae1881c..6ba9abaca32 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -10,6 +10,8 @@ #include #include +#include "config_core.h" + namespace DB { @@ -217,6 +219,14 @@ public: return CompareHelper::compare(data[n], assert_cast(rhs_).data[m], nan_direction_hint); } +#if USE_EMBEDDED_COMPILER + + bool isComparatorCompilable() const override; + + llvm::Value * compileComparator(llvm::IRBuilderBase & /*builder*/, llvm::Value * /*lhs*/, llvm::Value * /*rhs*/, llvm::Value * /*nan_direction_hint*/) const override; + +#endif + void compareColumn(const IColumn & rhs, size_t rhs_row_num, PaddedPODArray * row_indexes, PaddedPODArray & compare_results, int direction, int nan_direction_hint) const override diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index f4986799d47..65716d14ae2 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -7,10 +7,19 @@ #include #include +#include "config_core.h" + class SipHash; class Collator; +namespace llvm +{ + class LLVMContext; + class Value; + class IRBuilderBase; +} + namespace DB { @@ -281,6 +290,17 @@ public: */ [[nodiscard]] virtual int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const = 0; +#if USE_EMBEDDED_COMPILER + + virtual bool isComparatorCompilable() const { return false; } + + virtual llvm::Value * compileComparator(llvm::IRBuilderBase & /*builder*/, llvm::Value * /*lhs*/, llvm::Value * /*rhs*/, llvm::Value * /*nan_direction_hint*/) const + { + throw Exception("Method compileComparator is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED); + } + +#endif + /// Equivalent to compareAt, but collator is used to compare values. [[nodiscard]] virtual int compareAtWithCollation(size_t, size_t, const IColumn &, int, const Collator &) const { diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 30e01413e1b..4f633102847 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -120,6 +120,8 @@ static constexpr UInt64 operator""_Gb(unsigned long long value) M(UInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled", 0) \ M(Bool, compile_aggregate_expressions, true, "Compile aggregate functions to native code.", 0) \ M(UInt64, min_count_to_compile_aggregate_expression, 3, "The number of identical aggregate expressions before they are JIT-compiled", 0) \ + M(Bool, compile_sort_description, true, "Compile sort description to native code.", 0) \ + M(UInt64, min_count_to_compile_sort_description, 3, "The number of identical sort descriptions before they are JIT-compiled", 0) \ M(UInt64, group_by_two_level_threshold, 100000, "From what number of keys, a two-level aggregation starts. 0 - the threshold is not set.", 0) \ M(UInt64, group_by_two_level_threshold_bytes, 50000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. Two-level aggregation is used when at least one of the thresholds is triggered.", 0) \ M(Bool, distributed_aggregation_memory_efficient, true, "Is the memory-saving mode of distributed aggregation enabled.", 0) \ diff --git a/src/Core/SortCursor.h b/src/Core/SortCursor.h index a0f60fbccf8..52845fcb3f9 100644 --- a/src/Core/SortCursor.h +++ b/src/Core/SortCursor.h @@ -12,6 +12,11 @@ #include #include +#include "config_core.h" + +#if USE_EMBEDDED_COMPILER +#include +#endif namespace DB { @@ -49,6 +54,10 @@ struct SortCursorImpl */ IColumn::Permutation * permutation = nullptr; +#if USE_EMBEDDED_COMPILER + std::vector raw_sort_columns_data; +#endif + SortCursorImpl() = default; SortCursorImpl(const Block & block, const SortDescription & desc_, size_t order_ = 0, IColumn::Permutation * perm = nullptr) @@ -90,6 +99,10 @@ struct SortCursorImpl size_t column_number = block.getPositionByName(column_desc.column_name); sort_columns.push_back(columns[column_number].get()); +#if USE_EMBEDDED_COMPILER + if (desc.compiled_sort_description) + raw_sort_columns_data.emplace_back(getColumnData(sort_columns.back())); +#endif need_collation[j] = desc[j].collator != nullptr && sort_columns.back()->isCollationSupported(); has_collation |= need_collation[j]; } @@ -164,17 +177,34 @@ struct SortCursor : SortCursorHelper /// The specified row of this cursor is greater than the specified row of another cursor. bool ALWAYS_INLINE greaterAt(const SortCursor & rhs, size_t lhs_pos, size_t rhs_pos) const { +#if USE_EMBEDDED_COMPILER + if (impl->desc.compiled_sort_description) + { + auto sort_description_func_typed = reinterpret_cast(impl->desc.compiled_sort_description); + int res = sort_description_func_typed(lhs_pos, rhs_pos, impl->raw_sort_columns_data.data(), rhs.impl->raw_sort_columns_data.data()); /// NOLINT + + if (res > 0) + return true; + if (res < 0) + return false; + + return impl->order > rhs.impl->order; + } +#endif + for (size_t i = 0; i < impl->sort_columns_size; ++i) { const auto & desc = impl->desc[i]; int direction = desc.direction; int nulls_direction = desc.nulls_direction; int res = direction * impl->sort_columns[i]->compareAt(lhs_pos, rhs_pos, *(rhs.impl->sort_columns[i]), nulls_direction); + if (res > 0) return true; if (res < 0) return false; } + return impl->order > rhs.impl->order; } }; @@ -190,7 +220,21 @@ struct SimpleSortCursor : SortCursorHelper const auto & desc = impl->desc[0]; int direction = desc.direction; int nulls_direction = desc.nulls_direction; - int res = impl->sort_columns[0]->compareAt(lhs_pos, rhs_pos, *(rhs.impl->sort_columns[0]), nulls_direction); + + int res = 0; + +#if USE_EMBEDDED_COMPILER + if (impl->desc.compiled_sort_description && rhs.impl->desc.compiled_sort_description) + { + auto sort_description_func_typed = reinterpret_cast(impl->desc.compiled_sort_description); + res = sort_description_func_typed(lhs_pos, rhs_pos, impl->raw_sort_columns_data.data(), rhs.impl->raw_sort_columns_data.data()); /// NOLINT + } + else +#endif + { + res = impl->sort_columns[0]->compareAt(lhs_pos, rhs_pos, *(rhs.impl->sort_columns[0]), nulls_direction); + } + return res != 0 && ((res > 0) == (direction > 0)); } }; diff --git a/src/Core/SortDescription.cpp b/src/Core/SortDescription.cpp index 7994ada7b85..69e92425e9b 100644 --- a/src/Core/SortDescription.cpp +++ b/src/Core/SortDescription.cpp @@ -1,7 +1,13 @@ +#include +#include #include #include #include #include +#include + +#include +#include namespace DB { @@ -35,6 +41,100 @@ void SortColumnDescription::explain(JSONBuilder::JSONMap & map) const map.add("With Fill", with_fill); } +#if USE_EMBEDDED_COMPILER + +static CHJIT & getJITInstance() +{ + static CHJIT jit; + return jit; +} + +class CompiledSortDescriptionFunctionHolder final : public CompiledExpressionCacheEntry +{ +public: + explicit CompiledSortDescriptionFunctionHolder(CompiledSortDescriptionFunction compiled_function_) + : CompiledExpressionCacheEntry(compiled_function_.compiled_module.size) + , compiled_sort_description_function(compiled_function_) + {} + + ~CompiledSortDescriptionFunctionHolder() override + { + getJITInstance().deleteCompiledModule(compiled_sort_description_function.compiled_module); + } + + CompiledSortDescriptionFunction compiled_sort_description_function; +}; + +static std::unordered_map sort_description_to_compiled_function; + +static std::string getSortDescriptionDump(const SortDescription & description, const DataTypes & description_types) +{ + WriteBufferFromOwnString buffer; + + for (size_t i = 0; i < description.size(); ++i) + buffer << description_types[i]->getName() << ' ' << description[i].direction << ' ' << description[i].nulls_direction; + + return buffer.str(); +} + +static Poco::Logger * getLogger() +{ + static Poco::Logger & logger = Poco::Logger::get("SortDescription"); + return &logger; +} + +void compileSortDescriptionIfNeeded(SortDescription & description, const DataTypes & description_types) +{ + for (const auto & type : description_types) + { + if (!type->createColumn()->isComparatorCompilable()) + return; + } + + auto description_dump = getSortDescriptionDump(description, description_types); + + SipHash sort_description_dump_hash; + sort_description_dump_hash.update(description_dump); + + UInt128 aggregate_functions_description_hash_key; + sort_description_dump_hash.get128(aggregate_functions_description_hash_key); + + std::shared_ptr compiled_sort_description_holder; + + if (auto * compilation_cache = CompiledExpressionCacheFactory::instance().tryGetCache()) + { + auto [compiled_function_cache_entry, _] = compilation_cache->getOrSet(aggregate_functions_description_hash_key, [&] () + { + LOG_TRACE(getLogger(), "Compile sort description {}", description_dump); + + auto compiled_sort_description = compileSortDescription(getJITInstance(), description, description_types, description_dump); + return std::make_shared(std::move(compiled_sort_description)); + }); + + compiled_sort_description_holder = std::static_pointer_cast(compiled_function_cache_entry); + } + else + { + LOG_TRACE(getLogger(), "Compile sort description {}", description_dump); + auto compiled_sort_description = compileSortDescription(getJITInstance(), description, description_types, description_dump); + compiled_sort_description_holder = std::make_shared(std::move(compiled_sort_description)); + } + + auto comparator_function = compiled_sort_description_holder->compiled_sort_description_function.comparator_function; + description.compiled_sort_description = reinterpret_cast(comparator_function); + description.compiled_sort_description_holder = std::move(compiled_sort_description_holder); +} + +#else + +void compileSortDescriptionIfNeeded(SortDescription & description, const DataTypes & description_types) +{ + (void)(description); + (void)(description_types); +} + +#endif + std::string dumpSortDescription(const SortDescription & description) { WriteBufferFromOwnString wb; diff --git a/src/Core/SortDescription.h b/src/Core/SortDescription.h index b86706e665a..325b29fe96b 100644 --- a/src/Core/SortDescription.h +++ b/src/Core/SortDescription.h @@ -89,10 +89,20 @@ struct SortColumnDescriptionWithColumnIndex } }; +class CompiledSortDescriptionFunctionHolder; + /// Description of the sorting rule for several columns. -using SortDescription = std::vector; using SortDescriptionWithPositions = std::vector; +class SortDescription : public std::vector +{ +public: + void * compiled_sort_description = nullptr; + std::shared_ptr compiled_sort_description_holder; +}; + +void compileSortDescriptionIfNeeded(SortDescription & description, const DataTypes & description_types); + /// Outputs user-readable description into `out`. void dumpSortDescription(const SortDescription & description, WriteBuffer & out); diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 3b4f7eda24d..34ebb394ec6 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2468,10 +2468,10 @@ void InterpreterSelectQuery::executeWindow(QueryPlan & query_plan) // happens in case of `over ()`. if (!w.full_sort_description.empty() && (i == 0 || !sortIsPrefix(w, *windows_sorted[i - 1]))) { - auto sorting_step = std::make_unique( query_plan.getCurrentDataStream(), w.full_sort_description, + settings.compile_sort_description, settings.max_block_size, 0 /* LIMIT */, SizeLimits(settings.max_rows_to_sort, settings.max_bytes_to_sort, settings.sort_overflow_mode), @@ -2500,6 +2500,7 @@ void InterpreterSelectQuery::executeOrderOptimized(QueryPlan & query_plan, Input query_plan.getCurrentDataStream(), input_sorting_info->order_key_prefix_descr, output_order_descr, + settings.compile_sort_description, settings.max_block_size, limit); @@ -2530,6 +2531,7 @@ void InterpreterSelectQuery::executeOrder(QueryPlan & query_plan, InputOrderInfo auto sorting_step = std::make_unique( query_plan.getCurrentDataStream(), output_order_descr, + settings.compile_sort_description, settings.max_block_size, limit, SizeLimits(settings.max_rows_to_sort, settings.max_bytes_to_sort, settings.sort_overflow_mode), @@ -2558,7 +2560,7 @@ void InterpreterSelectQuery::executeMergeSorted(QueryPlan & query_plan, const So const Settings & settings = context->getSettingsRef(); auto merging_sorted - = std::make_unique(query_plan.getCurrentDataStream(), sort_description, settings.max_block_size, limit); + = std::make_unique(query_plan.getCurrentDataStream(), sort_description, settings.compile_sort_description, settings.max_block_size, limit); merging_sorted->setStepDescription("Merge sorted streams " + description); query_plan.addStep(std::move(merging_sorted)); diff --git a/src/Interpreters/JIT/compileFunction.cpp b/src/Interpreters/JIT/compileFunction.cpp index a820ead29b3..49d54d03ee5 100644 --- a/src/Interpreters/JIT/compileFunction.cpp +++ b/src/Interpreters/JIT/compileFunction.cpp @@ -775,6 +775,130 @@ CompiledAggregateFunctions compileAggregateFunctions(CHJIT & jit, const std::vec return compiled_aggregate_functions; } +CompiledSortDescriptionFunction compileSortDescription( + CHJIT & jit, + SortDescription & description, + const DataTypes & sort_description_types, + const std::string & sort_description_dump) +{ + Stopwatch watch; + + auto compiled_module = jit.compileModule([&](llvm::Module & module) + { + auto & context = module.getContext(); + llvm::IRBuilder<> b(context); + + auto * size_type = b.getIntNTy(sizeof(size_t) * 8); + + auto * column_data_type = llvm::StructType::get(b.getInt8PtrTy(), b.getInt8PtrTy()); + + std::vector types = { size_type, size_type, column_data_type->getPointerTo(), column_data_type->getPointerTo() }; + auto * comparator_func_declaration = llvm::FunctionType::get(b.getInt8Ty(), types, false); + auto * comparator_func = llvm::Function::Create(comparator_func_declaration, llvm::Function::ExternalLinkage, sort_description_dump, module); + + auto * arguments = comparator_func->args().begin(); + llvm::Value * lhs_index_arg = &*arguments++; + llvm::Value * rhs_index_arg = &*arguments++; + llvm::Value * columns_lhs_arg = &*arguments++; + llvm::Value * columns_rhs_arg = &*arguments++; + + size_t columns_size = description.size(); + + std::vector> comparator_steps_and_results; + for (size_t i = 0; i < columns_size; ++i) + { + auto * step = llvm::BasicBlock::Create(b.getContext(), "step_" + std::to_string(i), comparator_func); + llvm::Value * result_value = nullptr; + comparator_steps_and_results.emplace_back(step, result_value); + } + + auto * lhs_equals_rhs_result = llvm::ConstantInt::getSigned(b.getInt8Ty(), 0); + + auto * comparator_join = llvm::BasicBlock::Create(b.getContext(), "comparator_join", comparator_func); + + for (size_t i = 0; i < columns_size; ++i) + { + b.SetInsertPoint(comparator_steps_and_results[i].first); + + const auto & sort_description = description[i]; + const auto & column_type = sort_description_types[i]; + + auto dummy_column = column_type->createColumn(); + + auto * column_native_type_nullable = toNativeType(b, column_type); + auto * column_native_type = toNativeType(b, removeNullable(column_type)); + auto * column_native_type_pointer = column_native_type->getPointerTo(); + bool column_type_is_nullable = column_type->isNullable(); + + auto * nullable_unitilized = llvm::Constant::getNullValue(column_native_type_nullable); + + auto * lhs_column = b.CreateLoad(column_data_type, b.CreateConstInBoundsGEP1_64(column_data_type, columns_lhs_arg, i)); + auto * lhs_column_data = b.CreatePointerCast(b.CreateExtractValue(lhs_column, {0}), column_native_type_pointer); + auto * lhs_column_null_data = column_type_is_nullable ? b.CreateExtractValue(lhs_column, {1}) : nullptr; + + llvm::Value * lhs_value = b.CreateLoad(b.CreateInBoundsGEP(nullptr, lhs_column_data, lhs_index_arg)); + + if (lhs_column_null_data) + { + auto * is_null_value_pointer = b.CreateInBoundsGEP(nullptr, lhs_column_null_data, lhs_index_arg); + auto * is_null = b.CreateICmpNE(b.CreateLoad(b.getInt8Ty(), is_null_value_pointer), b.getInt8(0)); + auto * lhs_nullable_value = b.CreateInsertValue(b.CreateInsertValue(nullable_unitilized, lhs_value, {0}), is_null, {1}); + lhs_value = lhs_nullable_value; + } + + auto * rhs_column = b.CreateLoad(column_data_type, b.CreateConstInBoundsGEP1_64(column_data_type, columns_rhs_arg, i)); + auto * rhs_column_data = b.CreatePointerCast(b.CreateExtractValue(rhs_column, {0}), column_native_type_pointer); + auto * rhs_column_null_data = column_type_is_nullable ? b.CreateExtractValue(rhs_column, {1}) : nullptr; + + llvm::Value * rhs_value = b.CreateLoad(b.CreateInBoundsGEP(nullptr, rhs_column_data, rhs_index_arg)); + if (rhs_column_null_data) + { + auto * is_null_value_pointer = b.CreateInBoundsGEP(nullptr, rhs_column_null_data, rhs_index_arg); + auto * is_null = b.CreateICmpNE(b.CreateLoad(b.getInt8Ty(), is_null_value_pointer), b.getInt8(0)); + auto * rhs_nullable_value = b.CreateInsertValue(b.CreateInsertValue(nullable_unitilized, rhs_value, {0}), is_null, {1}); + rhs_value = rhs_nullable_value; + } + + llvm::Value * direction = llvm::ConstantInt::getSigned(b.getInt8Ty(), sort_description.direction); + llvm::Value * nan_direction_hint = llvm::ConstantInt::getSigned(b.getInt8Ty(), sort_description.nulls_direction); + llvm::Value * compare_result = dummy_column->compileComparator(b, lhs_value, rhs_value, nan_direction_hint); + llvm::Value * result = b.CreateMul(direction, compare_result); + + comparator_steps_and_results[i].first = b.GetInsertBlock(); + comparator_steps_and_results[i].second = result; + + if (i == columns_size - 1) + b.CreateBr(comparator_join); + else + b.CreateCondBr(b.CreateICmpEQ(result, lhs_equals_rhs_result), comparator_steps_and_results[i + 1].first, comparator_join); + } + + b.SetInsertPoint(comparator_join); + auto * phi = b.CreatePHI(b.getInt8Ty(), comparator_steps_and_results.size()); + + for (const auto & [block, result_value] : comparator_steps_and_results) + phi->addIncoming(result_value, block); + + b.CreateRet(phi); + }); + + ProfileEvents::increment(ProfileEvents::CompileExpressionsMicroseconds, watch.elapsedMicroseconds()); + ProfileEvents::increment(ProfileEvents::CompileExpressionsBytes, compiled_module.size); + ProfileEvents::increment(ProfileEvents::CompileFunction); + + auto comparator_function = reinterpret_cast(compiled_module.function_name_to_symbol[sort_description_dump]); + assert(comparator_function); + + CompiledSortDescriptionFunction compiled_sort_descriptor_function + { + .comparator_function = comparator_function, + + .compiled_module = std::move(compiled_module) + }; + + return compiled_sort_descriptor_function; +} + } #endif diff --git a/src/Interpreters/JIT/compileFunction.h b/src/Interpreters/JIT/compileFunction.h index f523bf5b970..bcd82ae8bab 100644 --- a/src/Interpreters/JIT/compileFunction.h +++ b/src/Interpreters/JIT/compileFunction.h @@ -4,10 +4,12 @@ #if USE_EMBEDDED_COMPILER +#include #include #include #include + namespace DB { @@ -83,6 +85,21 @@ struct CompiledAggregateFunctions */ CompiledAggregateFunctions compileAggregateFunctions(CHJIT & jit, const std::vector & functions, std::string functions_dump_name); + +using JITSortDescriptionFunc = int8_t (*)(size_t, size_t, ColumnData *, ColumnData *); + +struct CompiledSortDescriptionFunction +{ + JITSortDescriptionFunc comparator_function; + CHJIT::CompiledModule compiled_module; +}; + +CompiledSortDescriptionFunction compileSortDescription( + CHJIT & jit, + SortDescription & description, + const DataTypes & sort_description_types, + const std::string & sort_description_dump); + } #endif diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index 81603d6626d..0cbde659f25 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -590,7 +590,7 @@ void MergeJoin::mergeInMemoryRightBlocks() /// TODO: there should be no split keys by blocks for RIGHT|FULL JOIN builder.addTransform(std::make_shared( - builder.getHeader(), right_sort_description, max_rows_in_right_block, 0, 0, 0, 0, nullptr, 0)); + builder.getHeader(), right_sort_description, true, max_rows_in_right_block, 0, 0, 0, 0, nullptr, 0)); auto pipeline = QueryPipelineBuilder::getPipeline(std::move(builder)); PullingPipelineExecutor executor(pipeline); diff --git a/src/Interpreters/SortedBlocksWriter.cpp b/src/Interpreters/SortedBlocksWriter.cpp index 3caf144d9a8..7034e0f7b41 100644 --- a/src/Interpreters/SortedBlocksWriter.cpp +++ b/src/Interpreters/SortedBlocksWriter.cpp @@ -132,10 +132,12 @@ SortedBlocksWriter::TmpFilePtr SortedBlocksWriter::flush(const BlocksList & bloc if (pipeline.getNumStreams() > 1) { + auto transform = std::make_shared( pipeline.getHeader(), pipeline.getNumStreams(), sort_description, + true, rows_in_block); pipeline.addTransform(std::move(transform)); @@ -190,6 +192,7 @@ SortedBlocksWriter::PremergedFiles SortedBlocksWriter::premerge() pipeline.getHeader(), pipeline.getNumStreams(), sort_description, + true, rows_in_block); pipeline.addTransform(std::move(transform)); @@ -222,6 +225,7 @@ SortedBlocksWriter::SortedFiles SortedBlocksWriter::finishMerge(std::function(header, num_outputs, - sort_description, max_block_size); + sort_description, compile_sort_description, max_block_size); case MergeTreeData::MergingParams::Collapsing: return std::make_shared(header, num_outputs, @@ -774,7 +776,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( addMergingFinal( pipe, std::min(num_streams, settings.max_final_threads), - sort_description, data.merging_params, partition_key_columns, max_block_size); + sort_description, data.merging_params, partition_key_columns, settings.compile_sort_description, max_block_size); partition_pipes.emplace_back(std::move(pipe)); } diff --git a/src/Processors/QueryPlan/SortingStep.cpp b/src/Processors/QueryPlan/SortingStep.cpp index 255beebbdf1..c825efa09a0 100644 --- a/src/Processors/QueryPlan/SortingStep.cpp +++ b/src/Processors/QueryPlan/SortingStep.cpp @@ -31,6 +31,7 @@ static ITransformingStep::Traits getTraits(size_t limit) SortingStep::SortingStep( const DataStream & input_stream, const SortDescription & description_, + bool compile_sort_description_, size_t max_block_size_, UInt64 limit_, SizeLimits size_limits_, @@ -42,6 +43,7 @@ SortingStep::SortingStep( : ITransformingStep(input_stream, input_stream.header, getTraits(limit_)) , type(Type::Full) , result_description(description_) + , compile_sort_description(compile_sort_description_) , max_block_size(max_block_size_) , limit(limit_) , size_limits(size_limits_) @@ -59,12 +61,14 @@ SortingStep::SortingStep( const DataStream & input_stream_, SortDescription prefix_description_, SortDescription result_description_, + bool compile_sort_description_, size_t max_block_size_, UInt64 limit_) : ITransformingStep(input_stream_, input_stream_.header, getTraits(limit_)) , type(Type::FinishSorting) , prefix_description(std::move(prefix_description_)) , result_description(std::move(result_description_)) + , compile_sort_description(compile_sort_description_) , max_block_size(max_block_size_) , limit(limit_) { @@ -76,11 +80,13 @@ SortingStep::SortingStep( SortingStep::SortingStep( const DataStream & input_stream, SortDescription sort_description_, + bool compile_sort_description_, size_t max_block_size_, UInt64 limit_) : ITransformingStep(input_stream, input_stream.header, getTraits(limit_)) , type(Type::MergingSorted) , result_description(std::move(sort_description_)) + , compile_sort_description(compile_sort_description_) , max_block_size(max_block_size_) , limit(limit_) { @@ -124,6 +130,7 @@ void SortingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build pipeline.getHeader(), pipeline.getNumStreams(), prefix_description, + compile_sort_description, max_block_size, limit_for_merging); @@ -144,7 +151,7 @@ void SortingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build pipeline.addSimpleTransform([&](const Block & header) -> ProcessorPtr { return std::make_shared( - header, prefix_description, result_description, max_block_size, limit); + header, prefix_description, result_description, compile_sort_description, max_block_size, limit); }); } } @@ -177,7 +184,7 @@ void SortingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build return nullptr; return std::make_shared( - header, result_description, max_block_size, limit, + header, result_description, compile_sort_description, max_block_size, limit, max_bytes_before_remerge / pipeline.getNumStreams(), remerge_lowered_memory_bytes_ratio, max_bytes_before_external_sort, @@ -193,7 +200,9 @@ void SortingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build pipeline.getHeader(), pipeline.getNumStreams(), result_description, - max_block_size, limit); + compile_sort_description, + max_block_size, + limit); pipeline.addTransform(std::move(transform)); } @@ -207,7 +216,9 @@ void SortingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build pipeline.getHeader(), pipeline.getNumStreams(), result_description, - max_block_size, limit); + compile_sort_description, + max_block_size, + limit); pipeline.addTransform(std::move(transform)); } diff --git a/src/Processors/QueryPlan/SortingStep.h b/src/Processors/QueryPlan/SortingStep.h index 1738d8d4e45..f4ca45cc922 100644 --- a/src/Processors/QueryPlan/SortingStep.h +++ b/src/Processors/QueryPlan/SortingStep.h @@ -15,6 +15,7 @@ public: SortingStep( const DataStream & input_stream, const SortDescription & description_, + bool compile_sort_description_, size_t max_block_size_, UInt64 limit_, SizeLimits size_limits_, @@ -29,6 +30,7 @@ public: const DataStream & input_stream_, SortDescription prefix_description_, SortDescription result_description_, + bool compile_sort_description_, size_t max_block_size_, UInt64 limit_); @@ -36,6 +38,7 @@ public: SortingStep( const DataStream & input_stream, SortDescription sort_description_, + bool compile_sort_description_, size_t max_block_size_, UInt64 limit_ = 0); @@ -67,6 +70,7 @@ private: SortDescription prefix_description; SortDescription result_description; + bool compile_sort_description; size_t max_block_size; UInt64 limit; SizeLimits size_limits; diff --git a/src/Processors/Transforms/FinishSortingTransform.cpp b/src/Processors/Transforms/FinishSortingTransform.cpp index abded9bd2f0..fe8429a723a 100644 --- a/src/Processors/Transforms/FinishSortingTransform.cpp +++ b/src/Processors/Transforms/FinishSortingTransform.cpp @@ -24,9 +24,10 @@ FinishSortingTransform::FinishSortingTransform( const Block & header, const SortDescription & description_sorted_, const SortDescription & description_to_sort_, + bool compile_sort_description_, size_t max_merged_block_size_, UInt64 limit_) - : SortingTransform(header, description_to_sort_, max_merged_block_size_, limit_) + : SortingTransform(header, description_to_sort_, compile_sort_description_, max_merged_block_size_, limit_) { /// Check for sanity non-modified descriptions if (!isPrefix(description_sorted_, description_to_sort_)) diff --git a/src/Processors/Transforms/FinishSortingTransform.h b/src/Processors/Transforms/FinishSortingTransform.h index 3bebcc0a68f..ed89b1b5a4f 100644 --- a/src/Processors/Transforms/FinishSortingTransform.h +++ b/src/Processors/Transforms/FinishSortingTransform.h @@ -15,6 +15,7 @@ public: const Block & header, const SortDescription & description_sorted_, const SortDescription & description_to_sort_, + bool compile_sort_description, size_t max_merged_block_size_, UInt64 limit_); diff --git a/src/Processors/Transforms/MergeSortingTransform.cpp b/src/Processors/Transforms/MergeSortingTransform.cpp index 1fe945cbbc9..7bc834d2b7d 100644 --- a/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/src/Processors/Transforms/MergeSortingTransform.cpp @@ -90,6 +90,7 @@ private: MergeSortingTransform::MergeSortingTransform( const Block & header, const SortDescription & description_, + bool compile_sort_description_, size_t max_merged_block_size_, UInt64 limit_, size_t max_bytes_before_remerge_, @@ -97,7 +98,7 @@ MergeSortingTransform::MergeSortingTransform( size_t max_bytes_before_external_sort_, VolumePtr tmp_volume_, size_t min_free_disk_space_) - : SortingTransform(header, description_, max_merged_block_size_, limit_) + : SortingTransform(header, description_, compile_sort_description_, max_merged_block_size_, limit_) , max_bytes_before_remerge(max_bytes_before_remerge_) , remerge_lowered_memory_bytes_ratio(remerge_lowered_memory_bytes_ratio_) , max_bytes_before_external_sort(max_bytes_before_external_sort_) @@ -203,6 +204,7 @@ void MergeSortingTransform::consume(Chunk chunk) description, max_merged_block_size, limit, + compile_sort_description, nullptr, quiet, use_average_block_sizes, diff --git a/src/Processors/Transforms/MergeSortingTransform.h b/src/Processors/Transforms/MergeSortingTransform.h index 34a4842949b..7a067ecea51 100644 --- a/src/Processors/Transforms/MergeSortingTransform.h +++ b/src/Processors/Transforms/MergeSortingTransform.h @@ -21,6 +21,7 @@ public: MergeSortingTransform( const Block & header, const SortDescription & description_, + bool compile_sort_description_, size_t max_merged_block_size_, UInt64 limit_, size_t max_bytes_before_remerge_, diff --git a/src/Processors/Transforms/SortingTransform.cpp b/src/Processors/Transforms/SortingTransform.cpp index c0f700070fa..0d9336f96d7 100644 --- a/src/Processors/Transforms/SortingTransform.cpp +++ b/src/Processors/Transforms/SortingTransform.cpp @@ -127,14 +127,16 @@ Chunk MergeSorter::mergeImpl(TSortingHeap & queue) return Chunk(std::move(merged_columns), merged_rows); } - SortingTransform::SortingTransform( const Block & header, const SortDescription & description_, - size_t max_merged_block_size_, UInt64 limit_) + bool compile_sort_description_, + size_t max_merged_block_size_, + UInt64 limit_) : IProcessor({header}, {header}) , description(description_) , max_merged_block_size(max_merged_block_size_) + , compile_sort_description(compile_sort_description_) , limit(limit_) { const auto & sample = inputs.front().getHeader(); @@ -154,6 +156,9 @@ SortingTransform::SortingTransform( } } + DataTypes sort_description_types; + sort_description_types.reserve(description.size()); + /// Remove constants from column_description and remap positions. SortDescription description_without_constants; description_without_constants.reserve(description.size()); @@ -161,11 +166,19 @@ SortingTransform::SortingTransform( { auto old_pos = header.getPositionByName(column_description.column_name); auto new_pos = map[old_pos]; + if (new_pos < num_columns) + { + auto type = sample.safeGetByPosition(old_pos).type; + sort_description_types.emplace_back(type); description_without_constants.push_back(column_description); + } } description.swap(description_without_constants); + + if (compile_sort_description) + compileSortDescriptionIfNeeded(description, sort_description_types); } SortingTransform::~SortingTransform() = default; diff --git a/src/Processors/Transforms/SortingTransform.h b/src/Processors/Transforms/SortingTransform.h index 380ef4dff88..36d27420e18 100644 --- a/src/Processors/Transforms/SortingTransform.h +++ b/src/Processors/Transforms/SortingTransform.h @@ -68,7 +68,9 @@ public: /// limit - if not 0, allowed to return just first 'limit' rows in sorted order. SortingTransform(const Block & header, const SortDescription & description_, - size_t max_merged_block_size_, UInt64 limit_); + bool compile_sort_description_, + size_t max_merged_block_size_, + UInt64 limit_); ~SortingTransform() override; @@ -82,6 +84,7 @@ protected: SortDescription description; size_t max_merged_block_size; + bool compile_sort_description = false; UInt64 limit; /// Before operation, will remove constant columns from blocks. And after, place constant columns back. diff --git a/src/QueryPipeline/tests/gtest_blocks_size_merging_streams.cpp b/src/QueryPipeline/tests/gtest_blocks_size_merging_streams.cpp index e19d2c7114b..d51b9c96153 100644 --- a/src/QueryPipeline/tests/gtest_blocks_size_merging_streams.cpp +++ b/src/QueryPipeline/tests/gtest_blocks_size_merging_streams.cpp @@ -83,7 +83,7 @@ TEST(MergingSortedTest, SimpleBlockSizeTest) EXPECT_EQ(pipe.numOutputPorts(), 3); auto transform = std::make_shared(pipe.getHeader(), pipe.numOutputPorts(), sort_description, - DEFAULT_MERGE_BLOCK_SIZE, 0, nullptr, false, true); + DEFAULT_MERGE_BLOCK_SIZE, 0, false, nullptr, false, true); pipe.addTransform(std::move(transform)); @@ -130,7 +130,7 @@ TEST(MergingSortedTest, MoreInterestingBlockSizes) EXPECT_EQ(pipe.numOutputPorts(), 3); auto transform = std::make_shared(pipe.getHeader(), pipe.numOutputPorts(), sort_description, - DEFAULT_MERGE_BLOCK_SIZE, 0, nullptr, false, true); + DEFAULT_MERGE_BLOCK_SIZE, 0, false, nullptr, false, true); pipe.addTransform(std::move(transform)); diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index cb01fe3fcfc..aade89cac28 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -796,12 +796,13 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() ctx->blocks_are_granules_size = (global_ctx->chosen_merge_algorithm == MergeAlgorithm::Vertical); UInt64 merge_block_size = data_settings->merge_max_block_size; + bool compile_sort_description = global_ctx->data->getContext()->getSettingsRef().compile_sort_description; switch (ctx->merging_params.mode) { case MergeTreeData::MergingParams::Ordinary: merged_transform = std::make_shared( - header, pipes.size(), sort_description, merge_block_size, 0, ctx->rows_sources_write_buf.get(), true, ctx->blocks_are_granules_size); + header, pipes.size(), sort_description, merge_block_size, 0, compile_sort_description, ctx->rows_sources_write_buf.get(), true, ctx->blocks_are_granules_size); break; case MergeTreeData::MergingParams::Collapsing: From d85d72e5ad2198ee24aafcfc2c13603f90af4cf9 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 9 Feb 2022 19:40:38 +0000 Subject: [PATCH 125/171] Added performance tests --- tests/performance/jit_sort.xml | 126 +++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 tests/performance/jit_sort.xml diff --git a/tests/performance/jit_sort.xml b/tests/performance/jit_sort.xml new file mode 100644 index 00000000000..a3d8800f856 --- /dev/null +++ b/tests/performance/jit_sort.xml @@ -0,0 +1,126 @@ + + + hits_100m_single + + + + + + CREATE TABLE jit_test_memory ( + key UInt64, + value_1 UInt64, + value_2 UInt64, + value_3 UInt64, + value_4 UInt64, + value_5 UInt64 + ) Engine = Memory + + + + CREATE TABLE jit_test_merge_tree ( + key UInt64, + value_1 UInt64, + value_2 UInt64, + value_3 UInt64, + value_4 UInt64, + value_5 UInt64 + ) Engine = MergeTree + ORDER BY key + + + + CREATE TABLE jit_test_memory_nullable ( + key UInt64, + value_1 Nullable(UInt64), + value_2 Nullable(UInt64), + value_3 Nullable(UInt64), + value_4 Nullable(UInt64), + value_5 Nullable(UInt64) + ) Engine = Memory + + + + CREATE TABLE jit_test_merge_tree_nullable ( + key UInt64, + value_1 Nullable(UInt64), + value_2 Nullable(UInt64), + value_3 Nullable(UInt64), + value_4 Nullable(UInt64), + value_5 Nullable(UInt64) + ) Engine = MergeTree + ORDER BY key + + + + + table + + jit_test_memory + jit_test_merge_tree + jit_test_memory_nullable + jit_test_merge_tree_nullable + + + + + + INSERT INTO {table} + SELECT + number, + rand64(0), + rand64(1), + rand64(2), + rand64(3), + rand64(4) + FROM + system.numbers_mt + LIMIT 10000000 + + + + SELECT + * + FROM {table} + ORDER BY value_1 ASC + FORMAT Null + + + + SELECT + * + FROM {table} + ORDER BY value_1 ASC, value_2 DESC, value_3 ASC + FORMAT Null + + + + SELECT + * + FROM {table} + ORDER BY value_1 ASC, value_2 DESC, value_3 ASC, value_4 DESC, value_5 ASC + FORMAT Null + + + + SELECT + WatchID + FROM hits_100m_single + ORDER BY WatchID ASC, CounterID DESC, ClientIP ASC + LIMIT 2000000 + FORMAT Null + + + + SELECT + WatchID + FROM hits_100m_single + ORDER BY WatchID ASC, CounterID DESC, ClientIP ASC, IPNetworkID DESC, SearchEngineID ASC + LIMIT 2000000 + FORMAT Null + + + DROP TABLE IF EXISTS {table} + + \ No newline at end of file From b3b46d2e5c0c7c04bc464bb027098a395fe1f09b Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 13 Feb 2022 15:05:26 +0000 Subject: [PATCH 126/171] Fixed tests --- src/DataTypes/Native.h | 8 ++++---- src/Interpreters/JIT/compileFunction.cpp | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/DataTypes/Native.h b/src/DataTypes/Native.h index 3a635d2e240..26530164f75 100644 --- a/src/DataTypes/Native.h +++ b/src/DataTypes/Native.h @@ -47,7 +47,7 @@ static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder, const IDa return builder.getInt8Ty(); else if (data_type.isInt16() || data_type.isUInt16() || data_type.isDate()) return builder.getInt16Ty(); - else if (data_type.isInt32() || data_type.isUInt32() || data_type.isDateTime()) + else if (data_type.isInt32() || data_type.isUInt32() || data_type.isDate32() || data_type.isDateTime()) return builder.getInt32Ty(); else if (data_type.isInt64() || data_type.isUInt64()) return builder.getInt64Ty(); @@ -111,7 +111,7 @@ static inline bool canBeNativeType(const IDataType & type) return canBeNativeType(*data_type_nullable.getNestedType()); } - return data_type.isNativeInt() || data_type.isNativeUInt() || data_type.isFloat() || data_type.isDate() || data_type.isEnum(); + return data_type.isNativeInt() || data_type.isNativeUInt() || data_type.isFloat() || data_type.isDate() || data_type.isDateTime() || data_type.isEnum(); } static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder, const DataTypePtr & type) @@ -264,11 +264,11 @@ static inline llvm::Constant * getColumnNativeValue(llvm::IRBuilderBase & builde { return llvm::ConstantFP::get(type, assert_cast &>(column).getElement(index)); } - else if (column_data_type.isNativeUInt() || column_data_type.isDate() || column_data_type.isDateTime() || column_data_type.isDateTime64()) + else if (column_data_type.isNativeUInt() || column_data_type.isDate() || column_data_type.isDateTime()) { return llvm::ConstantInt::get(type, column.getUInt(index)); } - else if (column_data_type.isNativeInt() || column_data_type.isEnum()) + else if (column_data_type.isNativeInt() || column_data_type.isEnum() || column_data_type.isDate32()) { return llvm::ConstantInt::get(type, column.getInt(index)); } diff --git a/src/Interpreters/JIT/compileFunction.cpp b/src/Interpreters/JIT/compileFunction.cpp index 49d54d03ee5..9d2ab40bf76 100644 --- a/src/Interpreters/JIT/compileFunction.cpp +++ b/src/Interpreters/JIT/compileFunction.cpp @@ -827,6 +827,9 @@ CompiledSortDescriptionFunction compileSortDescription( auto * column_native_type_nullable = toNativeType(b, column_type); auto * column_native_type = toNativeType(b, removeNullable(column_type)); + if (!column_native_type) + throw Exception(ErrorCodes::LOGICAL_ERROR, "No native type for column type {}", column_type->getName()); + auto * column_native_type_pointer = column_native_type->getPointerTo(); bool column_type_is_nullable = column_type->isNullable(); From ce92f6aab1f37f2b37dc481d6a650e23edd18f38 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 16 Feb 2022 14:01:42 +0000 Subject: [PATCH 127/171] Fixed tests --- src/Interpreters/MergeJoin.cpp | 2 +- src/Interpreters/SortedBlocksWriter.cpp | 8 ++++---- .../tests/gtest_blocks_size_merging_streams.cpp | 4 ++-- src/Storages/MergeTree/MergeTask.cpp | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index 0cbde659f25..368a4c2ecf2 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -590,7 +590,7 @@ void MergeJoin::mergeInMemoryRightBlocks() /// TODO: there should be no split keys by blocks for RIGHT|FULL JOIN builder.addTransform(std::make_shared( - builder.getHeader(), right_sort_description, true, max_rows_in_right_block, 0, 0, 0, 0, nullptr, 0)); + builder.getHeader(), right_sort_description, false, max_rows_in_right_block, 0, 0, 0, 0, nullptr, 0)); auto pipeline = QueryPipelineBuilder::getPipeline(std::move(builder)); PullingPipelineExecutor executor(pipeline); diff --git a/src/Interpreters/SortedBlocksWriter.cpp b/src/Interpreters/SortedBlocksWriter.cpp index 7034e0f7b41..5b7c64943c7 100644 --- a/src/Interpreters/SortedBlocksWriter.cpp +++ b/src/Interpreters/SortedBlocksWriter.cpp @@ -137,7 +137,7 @@ SortedBlocksWriter::TmpFilePtr SortedBlocksWriter::flush(const BlocksList & bloc pipeline.getHeader(), pipeline.getNumStreams(), sort_description, - true, + false, rows_in_block); pipeline.addTransform(std::move(transform)); @@ -192,7 +192,7 @@ SortedBlocksWriter::PremergedFiles SortedBlocksWriter::premerge() pipeline.getHeader(), pipeline.getNumStreams(), sort_description, - true, + false, rows_in_block); pipeline.addTransform(std::move(transform)); @@ -225,7 +225,7 @@ SortedBlocksWriter::SortedFiles SortedBlocksWriter::finishMerge(std::function(pipe.getHeader(), pipe.numOutputPorts(), sort_description, - DEFAULT_MERGE_BLOCK_SIZE, 0, false, nullptr, false, true); + false, DEFAULT_MERGE_BLOCK_SIZE, false, nullptr, false, true); pipe.addTransform(std::move(transform)); @@ -130,7 +130,7 @@ TEST(MergingSortedTest, MoreInterestingBlockSizes) EXPECT_EQ(pipe.numOutputPorts(), 3); auto transform = std::make_shared(pipe.getHeader(), pipe.numOutputPorts(), sort_description, - DEFAULT_MERGE_BLOCK_SIZE, 0, false, nullptr, false, true); + false, DEFAULT_MERGE_BLOCK_SIZE, false, nullptr, false, true); pipe.addTransform(std::move(transform)); diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index aade89cac28..c516e6a39d5 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -802,7 +802,7 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() { case MergeTreeData::MergingParams::Ordinary: merged_transform = std::make_shared( - header, pipes.size(), sort_description, merge_block_size, 0, compile_sort_description, ctx->rows_sources_write_buf.get(), true, ctx->blocks_are_granules_size); + header, pipes.size(), sort_description, compile_sort_description, merge_block_size, 0, ctx->rows_sources_write_buf.get(), true, ctx->blocks_are_granules_size); break; case MergeTreeData::MergingParams::Collapsing: From cbfb773b506b1d80544faacbee87e81003a90dc2 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 17 Feb 2022 11:33:14 +0000 Subject: [PATCH 128/171] Fixed tests --- src/Core/SortCursor.h | 7 ++++++- src/Core/SortDescription.cpp | 6 +++--- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 2 +- src/Processors/Transforms/MergeSortingTransform.cpp | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/Core/SortCursor.h b/src/Core/SortCursor.h index 52845fcb3f9..1b212b511e4 100644 --- a/src/Core/SortCursor.h +++ b/src/Core/SortCursor.h @@ -87,6 +87,7 @@ struct SortCursorImpl { all_columns.clear(); sort_columns.clear(); + raw_sort_columns_data.clear(); size_t num_columns = columns.size(); @@ -178,8 +179,10 @@ struct SortCursor : SortCursorHelper bool ALWAYS_INLINE greaterAt(const SortCursor & rhs, size_t lhs_pos, size_t rhs_pos) const { #if USE_EMBEDDED_COMPILER - if (impl->desc.compiled_sort_description) + if (impl->desc.compiled_sort_description && rhs.impl->desc.compiled_sort_description) { + assert(impl->raw_sort_columns_data.size() == rhs.impl->raw_sort_columns_data.size()); + auto sort_description_func_typed = reinterpret_cast(impl->desc.compiled_sort_description); int res = sort_description_func_typed(lhs_pos, rhs_pos, impl->raw_sort_columns_data.data(), rhs.impl->raw_sort_columns_data.data()); /// NOLINT @@ -226,6 +229,8 @@ struct SimpleSortCursor : SortCursorHelper #if USE_EMBEDDED_COMPILER if (impl->desc.compiled_sort_description && rhs.impl->desc.compiled_sort_description) { + assert(impl->raw_sort_columns_data.size() == rhs.impl->raw_sort_columns_data.size()); + auto sort_description_func_typed = reinterpret_cast(impl->desc.compiled_sort_description); res = sort_description_func_typed(lhs_pos, rhs_pos, impl->raw_sort_columns_data.data(), rhs.impl->raw_sort_columns_data.data()); /// NOLINT } diff --git a/src/Core/SortDescription.cpp b/src/Core/SortDescription.cpp index 69e92425e9b..260e3ebd249 100644 --- a/src/Core/SortDescription.cpp +++ b/src/Core/SortDescription.cpp @@ -96,14 +96,14 @@ void compileSortDescriptionIfNeeded(SortDescription & description, const DataTyp SipHash sort_description_dump_hash; sort_description_dump_hash.update(description_dump); - UInt128 aggregate_functions_description_hash_key; - sort_description_dump_hash.get128(aggregate_functions_description_hash_key); + UInt128 sort_description_hash_key; + sort_description_dump_hash.get128(sort_description_hash_key); std::shared_ptr compiled_sort_description_holder; if (auto * compilation_cache = CompiledExpressionCacheFactory::instance().tryGetCache()) { - auto [compiled_function_cache_entry, _] = compilation_cache->getOrSet(aggregate_functions_description_hash_key, [&] () + auto [compiled_function_cache_entry, _] = compilation_cache->getOrSet(sort_description_hash_key, [&] () { LOG_TRACE(getLogger(), "Compile sort description {}", description_dump); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 605e3ebdd0d..27269a966cc 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -545,7 +545,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( pipe.getHeader(), pipe.numOutputPorts(), sort_description, - true, + settings.compile_sort_description, max_block_size); pipe.addTransform(std::move(transform)); diff --git a/src/Processors/Transforms/MergeSortingTransform.cpp b/src/Processors/Transforms/MergeSortingTransform.cpp index 7bc834d2b7d..2dd9152c506 100644 --- a/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/src/Processors/Transforms/MergeSortingTransform.cpp @@ -202,9 +202,9 @@ void MergeSortingTransform::consume(Chunk chunk) header_without_constants, 0, description, + compile_sort_description, max_merged_block_size, limit, - compile_sort_description, nullptr, quiet, use_average_block_sizes, From 512d88ddc6aef83013bc8b62c2ffd80ada48467b Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 17 Feb 2022 16:44:37 +0000 Subject: [PATCH 129/171] Fixed tests --- src/Core/SortCursor.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Core/SortCursor.h b/src/Core/SortCursor.h index 1b212b511e4..33a98ddc90a 100644 --- a/src/Core/SortCursor.h +++ b/src/Core/SortCursor.h @@ -87,7 +87,9 @@ struct SortCursorImpl { all_columns.clear(); sort_columns.clear(); +#if USE_EMBEDDED_COMPILER raw_sort_columns_data.clear(); +#endif size_t num_columns = columns.size(); From 8721e434e6a0a1094a92464808902aaa71cbf6ed Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 17 Feb 2022 19:29:21 +0000 Subject: [PATCH 130/171] Fixed tests --- src/Columns/ColumnVector.cpp | 2 +- src/Core/SortDescription.cpp | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 2828690130c..0cfc22736f8 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -196,7 +196,7 @@ template bool ColumnVector::isComparatorCompilable() const { /// TODO: for std::is_floating_point_v we need implement is_nan in LLVM IR. - return std::is_arithmetic_v; + return std::is_integral_v; } template diff --git a/src/Core/SortDescription.cpp b/src/Core/SortDescription.cpp index 260e3ebd249..7cd1727b027 100644 --- a/src/Core/SortDescription.cpp +++ b/src/Core/SortDescription.cpp @@ -85,6 +85,9 @@ static Poco::Logger * getLogger() void compileSortDescriptionIfNeeded(SortDescription & description, const DataTypes & description_types) { + if (description_types.empty()) + return; + for (const auto & type : description_types) { if (!type->createColumn()->isComparatorCompilable()) From c17a8b1a23cddfe86cd3ce445df90d6c10066dae Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 18 Feb 2022 11:58:32 +0000 Subject: [PATCH 131/171] Fixed tests --- src/Columns/ColumnNullable.cpp | 10 +++++++++- src/Core/SortCursor.h | 10 ++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index a16307f6d6d..81af3c816e5 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -273,13 +273,21 @@ llvm::Value * ColumnNullable::compileComparator(llvm::IRBuilderBase & builder, l b.CreateCondBr(lhs_or_rhs_are_null, lhs_or_rhs_are_null_block, lhs_rhs_are_not_null_block); + // if (unlikely(lval_is_null || rval_is_null)) + // { + // if (lval_is_null && rval_is_null) + // return 0; + // else + // return lval_is_null ? null_direction_hint : -null_direction_hint; + // } b.SetInsertPoint(lhs_or_rhs_are_null_block); auto * lhs_equals_rhs_result = llvm::ConstantInt::getSigned(b.getInt8Ty(), 0); - llvm::Value * lhs_and_rhs_are_null = b.CreateOr(lhs_is_null_value, rhs_is_null_value); + llvm::Value * lhs_and_rhs_are_null = b.CreateAnd(lhs_is_null_value, rhs_is_null_value); llvm::Value * lhs_is_null_result = b.CreateSelect(lhs_is_null_value, nan_direction_hint, b.CreateNeg(nan_direction_hint)); llvm::Value * lhs_or_rhs_are_null_block_result = b.CreateSelect(lhs_and_rhs_are_null, lhs_equals_rhs_result, lhs_is_null_result); b.CreateBr(join_block); + // getNestedColumn().compareAt(n, m, nested_rhs, null_direction_hint); b.SetInsertPoint(lhs_rhs_are_not_null_block); llvm::Value *lhs_rhs_are_not_null_block_result = nested_column->compileComparator(builder, lhs_unwrapped_value, rhs_unwrapped_value, nan_direction_hint); b.CreateBr(join_block); diff --git a/src/Core/SortCursor.h b/src/Core/SortCursor.h index 33a98ddc90a..584556045e8 100644 --- a/src/Core/SortCursor.h +++ b/src/Core/SortCursor.h @@ -226,7 +226,7 @@ struct SimpleSortCursor : SortCursorHelper int direction = desc.direction; int nulls_direction = desc.nulls_direction; - int res = 0; + bool result = false; #if USE_EMBEDDED_COMPILER if (impl->desc.compiled_sort_description && rhs.impl->desc.compiled_sort_description) @@ -234,15 +234,17 @@ struct SimpleSortCursor : SortCursorHelper assert(impl->raw_sort_columns_data.size() == rhs.impl->raw_sort_columns_data.size()); auto sort_description_func_typed = reinterpret_cast(impl->desc.compiled_sort_description); - res = sort_description_func_typed(lhs_pos, rhs_pos, impl->raw_sort_columns_data.data(), rhs.impl->raw_sort_columns_data.data()); /// NOLINT + int jit_result = sort_description_func_typed(lhs_pos, rhs_pos, impl->raw_sort_columns_data.data(), rhs.impl->raw_sort_columns_data.data()); /// NOLINT + result = jit_result > 0; } else #endif { - res = impl->sort_columns[0]->compareAt(lhs_pos, rhs_pos, *(rhs.impl->sort_columns[0]), nulls_direction); + int non_jit_result = impl->sort_columns[0]->compareAt(lhs_pos, rhs_pos, *(rhs.impl->sort_columns[0]), nulls_direction); + result = (non_jit_result != 0 && ((non_jit_result > 0) == (direction > 0))); } - return res != 0 && ((res > 0) == (direction > 0)); + return result; } }; From 4e7d10297b35d7f7493f04ba4ee3b031198e1ce0 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 28 Mar 2022 23:19:14 +0200 Subject: [PATCH 132/171] Fixed style --- src/Core/SortDescription.cpp | 2 +- src/Core/SortDescription.h | 2 ++ src/Interpreters/ExpressionAnalyzer.cpp | 17 +++++++++++-- src/Interpreters/InterpreterSelectQuery.cpp | 25 +++++++++---------- src/Interpreters/MergeJoin.cpp | 2 +- .../Algorithms/MergingSortedAlgorithm.cpp | 4 +-- .../Algorithms/MergingSortedAlgorithm.h | 1 - .../Merges/MergingSortedTransform.cpp | 2 -- .../Merges/MergingSortedTransform.h | 1 - src/Processors/QueryPlan/SortingStep.cpp | 13 ++-------- src/Processors/QueryPlan/SortingStep.h | 4 --- .../Transforms/FinishSortingTransform.h | 1 - .../Transforms/MergeSortingTransform.cpp | 3 +-- .../Transforms/SortingTransform.cpp | 5 +--- src/Processors/Transforms/SortingTransform.h | 2 -- .../gtest_blocks_size_merging_streams.cpp | 4 +-- src/Storages/MergeTree/MergeTask.cpp | 4 +-- 17 files changed, 40 insertions(+), 52 deletions(-) diff --git a/src/Core/SortDescription.cpp b/src/Core/SortDescription.cpp index 7cd1727b027..34e16f9caf2 100644 --- a/src/Core/SortDescription.cpp +++ b/src/Core/SortDescription.cpp @@ -85,7 +85,7 @@ static Poco::Logger * getLogger() void compileSortDescriptionIfNeeded(SortDescription & description, const DataTypes & description_types) { - if (description_types.empty()) + if (!description.compile_sort_description || description_types.empty()) return; for (const auto & type : description_types) diff --git a/src/Core/SortDescription.h b/src/Core/SortDescription.h index 325b29fe96b..2063f00f8d7 100644 --- a/src/Core/SortDescription.h +++ b/src/Core/SortDescription.h @@ -98,6 +98,8 @@ class SortDescription : public std::vector { public: void * compiled_sort_description = nullptr; + bool compile_sort_description = false; + size_t min_count_to_compile_sort_description = 3; std::shared_ptr compiled_sort_description_holder; }; diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 01769742071..8b751c3b353 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -793,6 +793,8 @@ void makeWindowDescriptionFromAST(const Context & context, void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr actions) { + auto current_context = getContext(); + // Window definitions from the WINDOW clause const auto * select_query = query->as(); if (select_query && select_query->window()) @@ -802,7 +804,7 @@ void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr actions) const auto & elem = ptr->as(); WindowDescription desc; desc.window_name = elem.name; - makeWindowDescriptionFromAST(*getContext(), window_descriptions, + makeWindowDescriptionFromAST(*current_context, window_descriptions, desc, elem.definition.get()); auto [it, inserted] = window_descriptions.insert( @@ -887,7 +889,7 @@ void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr actions) const ASTWindowDefinition &>(); WindowDescription desc; desc.window_name = definition.getDefaultWindowName(); - makeWindowDescriptionFromAST(*getContext(), window_descriptions, + makeWindowDescriptionFromAST(*current_context, window_descriptions, desc, &definition); auto [it, inserted] = window_descriptions.insert( @@ -902,6 +904,17 @@ void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr actions) it->second.window_functions.push_back(window_function); } } + + bool compile_sort_description = current_context->getSettingsRef().compile_sort_description; + size_t min_count_to_compile_sort_description = current_context->getSettingsRef().min_count_to_compile_sort_description; + + for (auto & [_, window_description] : window_descriptions) { + window_description.full_sort_description.compile_sort_description = compile_sort_description; + window_description.full_sort_description.min_count_to_compile_sort_description = min_count_to_compile_sort_description; + + window_description.partition_by.compile_sort_description = compile_sort_description; + window_description.partition_by.min_count_to_compile_sort_description = min_count_to_compile_sort_description; + } } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 34ebb394ec6..6bfadc66352 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -863,6 +863,9 @@ static SortDescription getSortDescription(const ASTSelectQuery & query, ContextP order_descr.emplace_back(name, order_by_elem.direction, order_by_elem.nulls_direction, collator); } + order_descr.compile_sort_description = context->getSettingsRef().compile_sort_description; + order_descr.min_count_to_compile_sort_description = context->getSettingsRef().min_count_to_compile_sort_description; + return order_descr; } @@ -2452,26 +2455,25 @@ void InterpreterSelectQuery::executeWindow(QueryPlan & query_plan) // Try to sort windows in such an order that the window with the longest // sort description goes first, and all window that use its prefixes follow. std::vector windows_sorted; - for (const auto & [_, w] : query_analyzer->windowDescriptions()) - windows_sorted.push_back(&w); + for (const auto & [_, window] : query_analyzer->windowDescriptions()) + windows_sorted.push_back(&window); ::sort(windows_sorted.begin(), windows_sorted.end(), windowDescriptionComparator); const Settings & settings = context->getSettingsRef(); for (size_t i = 0; i < windows_sorted.size(); ++i) { - const auto & w = *windows_sorted[i]; + const auto & window = *windows_sorted[i]; // We don't need to sort again if the input from previous window already // has suitable sorting. Also don't create sort steps when there are no // columns to sort by, because the sort nodes are confused by this. It // happens in case of `over ()`. - if (!w.full_sort_description.empty() && (i == 0 || !sortIsPrefix(w, *windows_sorted[i - 1]))) + if (!window.full_sort_description.empty() && (i == 0 || !sortIsPrefix(window, *windows_sorted[i - 1]))) { auto sorting_step = std::make_unique( query_plan.getCurrentDataStream(), - w.full_sort_description, - settings.compile_sort_description, + window.full_sort_description, settings.max_block_size, 0 /* LIMIT */, SizeLimits(settings.max_rows_to_sort, settings.max_bytes_to_sort, settings.sort_overflow_mode), @@ -2480,12 +2482,12 @@ void InterpreterSelectQuery::executeWindow(QueryPlan & query_plan) settings.max_bytes_before_external_sort, context->getTemporaryVolume(), settings.min_free_disk_space_for_temporary_data); - sorting_step->setStepDescription("Sorting for window '" + w.window_name + "'"); + sorting_step->setStepDescription("Sorting for window '" + window.window_name + "'"); query_plan.addStep(std::move(sorting_step)); } - auto window_step = std::make_unique(query_plan.getCurrentDataStream(), w, w.window_functions); - window_step->setStepDescription("Window step for window '" + w.window_name + "'"); + auto window_step = std::make_unique(query_plan.getCurrentDataStream(), window, window.window_functions); + window_step->setStepDescription("Window step for window '" + window.window_name + "'"); query_plan.addStep(std::move(window_step)); } @@ -2500,7 +2502,6 @@ void InterpreterSelectQuery::executeOrderOptimized(QueryPlan & query_plan, Input query_plan.getCurrentDataStream(), input_sorting_info->order_key_prefix_descr, output_order_descr, - settings.compile_sort_description, settings.max_block_size, limit); @@ -2531,7 +2532,6 @@ void InterpreterSelectQuery::executeOrder(QueryPlan & query_plan, InputOrderInfo auto sorting_step = std::make_unique( query_plan.getCurrentDataStream(), output_order_descr, - settings.compile_sort_description, settings.max_block_size, limit, SizeLimits(settings.max_rows_to_sort, settings.max_bytes_to_sort, settings.sort_overflow_mode), @@ -2559,8 +2559,7 @@ void InterpreterSelectQuery::executeMergeSorted(QueryPlan & query_plan, const So { const Settings & settings = context->getSettingsRef(); - auto merging_sorted - = std::make_unique(query_plan.getCurrentDataStream(), sort_description, settings.compile_sort_description, settings.max_block_size, limit); + auto merging_sorted = std::make_unique(query_plan.getCurrentDataStream(), sort_description, settings.max_block_size, limit); merging_sorted->setStepDescription("Merge sorted streams " + description); query_plan.addStep(std::move(merging_sorted)); diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index 368a4c2ecf2..81603d6626d 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -590,7 +590,7 @@ void MergeJoin::mergeInMemoryRightBlocks() /// TODO: there should be no split keys by blocks for RIGHT|FULL JOIN builder.addTransform(std::make_shared( - builder.getHeader(), right_sort_description, false, max_rows_in_right_block, 0, 0, 0, 0, nullptr, 0)); + builder.getHeader(), right_sort_description, max_rows_in_right_block, 0, 0, 0, 0, nullptr, 0)); auto pipeline = QueryPipelineBuilder::getPipeline(std::move(builder)); PullingPipelineExecutor executor(pipeline); diff --git a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp index 6738e94b887..435a376bdb8 100644 --- a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp @@ -14,7 +14,6 @@ MergingSortedAlgorithm::MergingSortedAlgorithm( Block header_, size_t num_inputs, SortDescription description_, - bool compile_sort_descriptor, size_t max_block_size, UInt64 limit_, WriteBuffer * out_row_sources_buf_, @@ -37,8 +36,7 @@ MergingSortedAlgorithm::MergingSortedAlgorithm( sort_description_types.emplace_back(header.getByName(column_description.column_name).type); } - if (compile_sort_descriptor) - compileSortDescriptionIfNeeded(description, sort_description_types); + compileSortDescriptionIfNeeded(description, sort_description_types); } void MergingSortedAlgorithm::addInput() diff --git a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h index 95d1df81772..cf3ec44f5fc 100644 --- a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h +++ b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h @@ -17,7 +17,6 @@ public: Block header_, size_t num_inputs, SortDescription description_, - bool compile_sort_descriptor, size_t max_block_size, UInt64 limit_ = 0, WriteBuffer * out_row_sources_buf_ = nullptr, diff --git a/src/Processors/Merges/MergingSortedTransform.cpp b/src/Processors/Merges/MergingSortedTransform.cpp index 99fb61fa393..c244388464f 100644 --- a/src/Processors/Merges/MergingSortedTransform.cpp +++ b/src/Processors/Merges/MergingSortedTransform.cpp @@ -11,7 +11,6 @@ MergingSortedTransform::MergingSortedTransform( const Block & header, size_t num_inputs, SortDescription description_, - bool compile_sort_description_, size_t max_block_size, UInt64 limit_, WriteBuffer * out_row_sources_buf_, @@ -23,7 +22,6 @@ MergingSortedTransform::MergingSortedTransform( header, num_inputs, std::move(description_), - compile_sort_description_, max_block_size, limit_, out_row_sources_buf_, diff --git a/src/Processors/Merges/MergingSortedTransform.h b/src/Processors/Merges/MergingSortedTransform.h index f62b76dfe52..93bd36d8aec 100644 --- a/src/Processors/Merges/MergingSortedTransform.h +++ b/src/Processors/Merges/MergingSortedTransform.h @@ -15,7 +15,6 @@ public: const Block & header, size_t num_inputs, SortDescription description, - bool compile_sort_description_, size_t max_block_size, UInt64 limit_ = 0, WriteBuffer * out_row_sources_buf_ = nullptr, diff --git a/src/Processors/QueryPlan/SortingStep.cpp b/src/Processors/QueryPlan/SortingStep.cpp index c825efa09a0..b675c4a8815 100644 --- a/src/Processors/QueryPlan/SortingStep.cpp +++ b/src/Processors/QueryPlan/SortingStep.cpp @@ -31,7 +31,6 @@ static ITransformingStep::Traits getTraits(size_t limit) SortingStep::SortingStep( const DataStream & input_stream, const SortDescription & description_, - bool compile_sort_description_, size_t max_block_size_, UInt64 limit_, SizeLimits size_limits_, @@ -43,7 +42,6 @@ SortingStep::SortingStep( : ITransformingStep(input_stream, input_stream.header, getTraits(limit_)) , type(Type::Full) , result_description(description_) - , compile_sort_description(compile_sort_description_) , max_block_size(max_block_size_) , limit(limit_) , size_limits(size_limits_) @@ -61,14 +59,12 @@ SortingStep::SortingStep( const DataStream & input_stream_, SortDescription prefix_description_, SortDescription result_description_, - bool compile_sort_description_, size_t max_block_size_, UInt64 limit_) : ITransformingStep(input_stream_, input_stream_.header, getTraits(limit_)) , type(Type::FinishSorting) , prefix_description(std::move(prefix_description_)) , result_description(std::move(result_description_)) - , compile_sort_description(compile_sort_description_) , max_block_size(max_block_size_) , limit(limit_) { @@ -80,13 +76,11 @@ SortingStep::SortingStep( SortingStep::SortingStep( const DataStream & input_stream, SortDescription sort_description_, - bool compile_sort_description_, size_t max_block_size_, UInt64 limit_) : ITransformingStep(input_stream, input_stream.header, getTraits(limit_)) , type(Type::MergingSorted) , result_description(std::move(sort_description_)) - , compile_sort_description(compile_sort_description_) , max_block_size(max_block_size_) , limit(limit_) { @@ -130,7 +124,6 @@ void SortingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build pipeline.getHeader(), pipeline.getNumStreams(), prefix_description, - compile_sort_description, max_block_size, limit_for_merging); @@ -151,7 +144,7 @@ void SortingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build pipeline.addSimpleTransform([&](const Block & header) -> ProcessorPtr { return std::make_shared( - header, prefix_description, result_description, compile_sort_description, max_block_size, limit); + header, prefix_description, result_description, max_block_size, limit); }); } } @@ -184,7 +177,7 @@ void SortingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build return nullptr; return std::make_shared( - header, result_description, compile_sort_description, max_block_size, limit, + header, result_description, max_block_size, limit, max_bytes_before_remerge / pipeline.getNumStreams(), remerge_lowered_memory_bytes_ratio, max_bytes_before_external_sort, @@ -200,7 +193,6 @@ void SortingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build pipeline.getHeader(), pipeline.getNumStreams(), result_description, - compile_sort_description, max_block_size, limit); @@ -216,7 +208,6 @@ void SortingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build pipeline.getHeader(), pipeline.getNumStreams(), result_description, - compile_sort_description, max_block_size, limit); diff --git a/src/Processors/QueryPlan/SortingStep.h b/src/Processors/QueryPlan/SortingStep.h index f4ca45cc922..1738d8d4e45 100644 --- a/src/Processors/QueryPlan/SortingStep.h +++ b/src/Processors/QueryPlan/SortingStep.h @@ -15,7 +15,6 @@ public: SortingStep( const DataStream & input_stream, const SortDescription & description_, - bool compile_sort_description_, size_t max_block_size_, UInt64 limit_, SizeLimits size_limits_, @@ -30,7 +29,6 @@ public: const DataStream & input_stream_, SortDescription prefix_description_, SortDescription result_description_, - bool compile_sort_description_, size_t max_block_size_, UInt64 limit_); @@ -38,7 +36,6 @@ public: SortingStep( const DataStream & input_stream, SortDescription sort_description_, - bool compile_sort_description_, size_t max_block_size_, UInt64 limit_ = 0); @@ -70,7 +67,6 @@ private: SortDescription prefix_description; SortDescription result_description; - bool compile_sort_description; size_t max_block_size; UInt64 limit; SizeLimits size_limits; diff --git a/src/Processors/Transforms/FinishSortingTransform.h b/src/Processors/Transforms/FinishSortingTransform.h index ed89b1b5a4f..3bebcc0a68f 100644 --- a/src/Processors/Transforms/FinishSortingTransform.h +++ b/src/Processors/Transforms/FinishSortingTransform.h @@ -15,7 +15,6 @@ public: const Block & header, const SortDescription & description_sorted_, const SortDescription & description_to_sort_, - bool compile_sort_description, size_t max_merged_block_size_, UInt64 limit_); diff --git a/src/Processors/Transforms/MergeSortingTransform.cpp b/src/Processors/Transforms/MergeSortingTransform.cpp index 2dd9152c506..c18e8c54d00 100644 --- a/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/src/Processors/Transforms/MergeSortingTransform.cpp @@ -98,7 +98,7 @@ MergeSortingTransform::MergeSortingTransform( size_t max_bytes_before_external_sort_, VolumePtr tmp_volume_, size_t min_free_disk_space_) - : SortingTransform(header, description_, compile_sort_description_, max_merged_block_size_, limit_) + : SortingTransform(header, description_, max_merged_block_size_, limit_) , max_bytes_before_remerge(max_bytes_before_remerge_) , remerge_lowered_memory_bytes_ratio(remerge_lowered_memory_bytes_ratio_) , max_bytes_before_external_sort(max_bytes_before_external_sort_) @@ -202,7 +202,6 @@ void MergeSortingTransform::consume(Chunk chunk) header_without_constants, 0, description, - compile_sort_description, max_merged_block_size, limit, nullptr, diff --git a/src/Processors/Transforms/SortingTransform.cpp b/src/Processors/Transforms/SortingTransform.cpp index 0d9336f96d7..ee197ae2de1 100644 --- a/src/Processors/Transforms/SortingTransform.cpp +++ b/src/Processors/Transforms/SortingTransform.cpp @@ -130,13 +130,11 @@ Chunk MergeSorter::mergeImpl(TSortingHeap & queue) SortingTransform::SortingTransform( const Block & header, const SortDescription & description_, - bool compile_sort_description_, size_t max_merged_block_size_, UInt64 limit_) : IProcessor({header}, {header}) , description(description_) , max_merged_block_size(max_merged_block_size_) - , compile_sort_description(compile_sort_description_) , limit(limit_) { const auto & sample = inputs.front().getHeader(); @@ -177,8 +175,7 @@ SortingTransform::SortingTransform( description.swap(description_without_constants); - if (compile_sort_description) - compileSortDescriptionIfNeeded(description, sort_description_types); + compileSortDescriptionIfNeeded(description, sort_description_types); } SortingTransform::~SortingTransform() = default; diff --git a/src/Processors/Transforms/SortingTransform.h b/src/Processors/Transforms/SortingTransform.h index 36d27420e18..e98a9f88b48 100644 --- a/src/Processors/Transforms/SortingTransform.h +++ b/src/Processors/Transforms/SortingTransform.h @@ -68,7 +68,6 @@ public: /// limit - if not 0, allowed to return just first 'limit' rows in sorted order. SortingTransform(const Block & header, const SortDescription & description_, - bool compile_sort_description_, size_t max_merged_block_size_, UInt64 limit_); @@ -84,7 +83,6 @@ protected: SortDescription description; size_t max_merged_block_size; - bool compile_sort_description = false; UInt64 limit; /// Before operation, will remove constant columns from blocks. And after, place constant columns back. diff --git a/src/QueryPipeline/tests/gtest_blocks_size_merging_streams.cpp b/src/QueryPipeline/tests/gtest_blocks_size_merging_streams.cpp index 61cd5df751e..feae7127349 100644 --- a/src/QueryPipeline/tests/gtest_blocks_size_merging_streams.cpp +++ b/src/QueryPipeline/tests/gtest_blocks_size_merging_streams.cpp @@ -83,7 +83,7 @@ TEST(MergingSortedTest, SimpleBlockSizeTest) EXPECT_EQ(pipe.numOutputPorts(), 3); auto transform = std::make_shared(pipe.getHeader(), pipe.numOutputPorts(), sort_description, - false, DEFAULT_MERGE_BLOCK_SIZE, false, nullptr, false, true); + DEFAULT_MERGE_BLOCK_SIZE, false, nullptr, false, true); pipe.addTransform(std::move(transform)); @@ -130,7 +130,7 @@ TEST(MergingSortedTest, MoreInterestingBlockSizes) EXPECT_EQ(pipe.numOutputPorts(), 3); auto transform = std::make_shared(pipe.getHeader(), pipe.numOutputPorts(), sort_description, - false, DEFAULT_MERGE_BLOCK_SIZE, false, nullptr, false, true); + DEFAULT_MERGE_BLOCK_SIZE, false, nullptr, false, true); pipe.addTransform(std::move(transform)); diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index c516e6a39d5..d17adb0c602 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -796,13 +796,13 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() ctx->blocks_are_granules_size = (global_ctx->chosen_merge_algorithm == MergeAlgorithm::Vertical); UInt64 merge_block_size = data_settings->merge_max_block_size; - bool compile_sort_description = global_ctx->data->getContext()->getSettingsRef().compile_sort_description; + sort_description.compile_sort_description = global_ctx->data->getContext()->getSettingsRef().compile_sort_description; switch (ctx->merging_params.mode) { case MergeTreeData::MergingParams::Ordinary: merged_transform = std::make_shared( - header, pipes.size(), sort_description, compile_sort_description, merge_block_size, 0, ctx->rows_sources_write_buf.get(), true, ctx->blocks_are_granules_size); + header, pipes.size(), sort_description, merge_block_size, 0, ctx->rows_sources_write_buf.get(), true, ctx->blocks_are_granules_size); break; case MergeTreeData::MergingParams::Collapsing: From ea8ce3140abf26be68d8a2ce341ae187b95aed7d Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 28 Mar 2022 23:42:16 +0200 Subject: [PATCH 133/171] Fixed tests --- src/Interpreters/SortedBlocksWriter.cpp | 5 ----- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 12 ++++++++---- tests/performance/jit_sort.xml | 5 +++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Interpreters/SortedBlocksWriter.cpp b/src/Interpreters/SortedBlocksWriter.cpp index 5b7c64943c7..3caf144d9a8 100644 --- a/src/Interpreters/SortedBlocksWriter.cpp +++ b/src/Interpreters/SortedBlocksWriter.cpp @@ -132,12 +132,10 @@ SortedBlocksWriter::TmpFilePtr SortedBlocksWriter::flush(const BlocksList & bloc if (pipeline.getNumStreams() > 1) { - auto transform = std::make_shared( pipeline.getHeader(), pipeline.getNumStreams(), sort_description, - false, rows_in_block); pipeline.addTransform(std::move(transform)); @@ -192,7 +190,6 @@ SortedBlocksWriter::PremergedFiles SortedBlocksWriter::premerge() pipeline.getHeader(), pipeline.getNumStreams(), sort_description, - false, rows_in_block); pipeline.addTransform(std::move(transform)); @@ -225,7 +222,6 @@ SortedBlocksWriter::SortedFiles SortedBlocksWriter::finishMerge(std::functiongetSortingKey().column_names; SortDescription sort_description; + sort_description.compile_sort_description = settings.compile_sort_description; + sort_description.min_count_to_compile_sort_description = settings.min_count_to_compile_sort_description; + for (size_t j = 0; j < prefix_size; ++j) sort_description.emplace_back(sorting_columns[j], input_order_info->direction); @@ -545,7 +548,6 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( pipe.getHeader(), pipe.numOutputPorts(), sort_description, - settings.compile_sort_description, max_block_size); pipe.addTransform(std::move(transform)); @@ -562,7 +564,6 @@ static void addMergingFinal( const SortDescription & sort_description, MergeTreeData::MergingParams merging_params, Names partition_key_columns, - bool compile_sort_description, size_t max_block_size) { const auto & header = pipe.getHeader(); @@ -576,7 +577,7 @@ static void addMergingFinal( { case MergeTreeData::MergingParams::Ordinary: return std::make_shared(header, num_outputs, - sort_description, compile_sort_description, max_block_size); + sort_description, max_block_size); case MergeTreeData::MergingParams::Collapsing: return std::make_shared(header, num_outputs, @@ -765,6 +766,9 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( Names sort_columns = metadata_for_reading->getSortingKeyColumns(); SortDescription sort_description; + sort_description.compile_sort_description = settings.compile_sort_description; + sort_description.min_count_to_compile_sort_description = settings.min_count_to_compile_sort_description; + size_t sort_columns_size = sort_columns.size(); sort_description.reserve(sort_columns_size); @@ -776,7 +780,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( addMergingFinal( pipe, std::min(num_streams, settings.max_final_threads), - sort_description, data.merging_params, partition_key_columns, settings.compile_sort_description, max_block_size); + sort_description, data.merging_params, partition_key_columns, max_block_size); partition_pipes.emplace_back(std::move(pipe)); } diff --git a/tests/performance/jit_sort.xml b/tests/performance/jit_sort.xml index a3d8800f856..6c21a13c651 100644 --- a/tests/performance/jit_sort.xml +++ b/tests/performance/jit_sort.xml @@ -3,9 +3,10 @@ hits_100m_single - + 0 + CREATE TABLE jit_test_memory ( From 06b21a5cc4b8fabd2debffff56c945a28dc471c1 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 1 May 2022 20:41:05 +0200 Subject: [PATCH 134/171] Fix build --- src/Core/SortDescription.cpp | 1 - src/Processors/Transforms/FinishSortingTransform.cpp | 3 +-- src/Processors/Transforms/MergeSortingTransform.cpp | 1 - src/Processors/Transforms/MergeSortingTransform.h | 1 - 4 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Core/SortDescription.cpp b/src/Core/SortDescription.cpp index 34e16f9caf2..aa7700fa9c5 100644 --- a/src/Core/SortDescription.cpp +++ b/src/Core/SortDescription.cpp @@ -1,4 +1,3 @@ -#include #include #include #include diff --git a/src/Processors/Transforms/FinishSortingTransform.cpp b/src/Processors/Transforms/FinishSortingTransform.cpp index fe8429a723a..abded9bd2f0 100644 --- a/src/Processors/Transforms/FinishSortingTransform.cpp +++ b/src/Processors/Transforms/FinishSortingTransform.cpp @@ -24,10 +24,9 @@ FinishSortingTransform::FinishSortingTransform( const Block & header, const SortDescription & description_sorted_, const SortDescription & description_to_sort_, - bool compile_sort_description_, size_t max_merged_block_size_, UInt64 limit_) - : SortingTransform(header, description_to_sort_, compile_sort_description_, max_merged_block_size_, limit_) + : SortingTransform(header, description_to_sort_, max_merged_block_size_, limit_) { /// Check for sanity non-modified descriptions if (!isPrefix(description_sorted_, description_to_sort_)) diff --git a/src/Processors/Transforms/MergeSortingTransform.cpp b/src/Processors/Transforms/MergeSortingTransform.cpp index c18e8c54d00..1fe945cbbc9 100644 --- a/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/src/Processors/Transforms/MergeSortingTransform.cpp @@ -90,7 +90,6 @@ private: MergeSortingTransform::MergeSortingTransform( const Block & header, const SortDescription & description_, - bool compile_sort_description_, size_t max_merged_block_size_, UInt64 limit_, size_t max_bytes_before_remerge_, diff --git a/src/Processors/Transforms/MergeSortingTransform.h b/src/Processors/Transforms/MergeSortingTransform.h index 7a067ecea51..34a4842949b 100644 --- a/src/Processors/Transforms/MergeSortingTransform.h +++ b/src/Processors/Transforms/MergeSortingTransform.h @@ -21,7 +21,6 @@ public: MergeSortingTransform( const Block & header, const SortDescription & description_, - bool compile_sort_description_, size_t max_merged_block_size_, UInt64 limit_, size_t max_bytes_before_remerge_, From 837f2e8b9c848f248f04f091b9a2f560a1502fd6 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 2 May 2022 12:06:10 +0200 Subject: [PATCH 135/171] Update performance test --- tests/performance/jit_sort.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/performance/jit_sort.xml b/tests/performance/jit_sort.xml index 6c21a13c651..d11b4351f56 100644 --- a/tests/performance/jit_sort.xml +++ b/tests/performance/jit_sort.xml @@ -3,11 +3,6 @@ hits_100m_single - - 1 - 0 - - CREATE TABLE jit_test_memory ( key UInt64, From fcc42fe3a82b07fe5c6cf02f4b4042ed157f8556 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 2 May 2022 12:07:13 +0200 Subject: [PATCH 136/171] Fix style check --- src/Core/SortDescription.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Core/SortDescription.cpp b/src/Core/SortDescription.cpp index aa7700fa9c5..c396538dd1f 100644 --- a/src/Core/SortDescription.cpp +++ b/src/Core/SortDescription.cpp @@ -1,4 +1,3 @@ -#include #include #include #include From 75555c436b4e1e4675f45c5990af10b68c9d10dc Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 11 May 2022 13:02:49 +0200 Subject: [PATCH 137/171] Fix usage of min_count_to_compile_sort_description setting --- src/Core/SortDescription.cpp | 29 ++++++++++---- src/Core/SortDescription.h | 11 +++-- src/Interpreters/MergeJoin.cpp | 2 +- .../Algorithms/MergingSortedAlgorithm.cpp | 4 +- src/Processors/QueryPlan/SortingStep.cpp | 40 ++++++++++++++----- .../Transforms/FinishSortingTransform.cpp | 5 ++- .../Transforms/FinishSortingTransform.h | 3 +- .../Transforms/MergeSortingTransform.cpp | 3 +- .../Transforms/MergeSortingTransform.h | 1 + .../Transforms/SortingTransform.cpp | 7 +++- src/Processors/Transforms/SortingTransform.h | 3 +- 11 files changed, 77 insertions(+), 31 deletions(-) diff --git a/src/Core/SortDescription.cpp b/src/Core/SortDescription.cpp index c396538dd1f..8d830ccb5da 100644 --- a/src/Core/SortDescription.cpp +++ b/src/Core/SortDescription.cpp @@ -65,12 +65,12 @@ public: static std::unordered_map sort_description_to_compiled_function; -static std::string getSortDescriptionDump(const SortDescription & description, const DataTypes & description_types) +static std::string getSortDescriptionDump(const SortDescription & description, const DataTypes & header_types) { WriteBufferFromOwnString buffer; for (size_t i = 0; i < description.size(); ++i) - buffer << description_types[i]->getName() << ' ' << description[i].direction << ' ' << description[i].nulls_direction; + buffer << header_types[i]->getName() << ' ' << description[i].direction << ' ' << description[i].nulls_direction; return buffer.str(); } @@ -81,18 +81,21 @@ static Poco::Logger * getLogger() return &logger; } -void compileSortDescriptionIfNeeded(SortDescription & description, const DataTypes & description_types) +void compileSortDescriptionIfNeeded(SortDescription & description, const DataTypes & sort_description_types, bool increase_compile_attemps) { - if (!description.compile_sort_description || description_types.empty()) + static std::unordered_map counter; + static std::mutex mutex; + + if (!description.compile_sort_description || sort_description_types.empty()) return; - for (const auto & type : description_types) + for (const auto & type : sort_description_types) { if (!type->createColumn()->isComparatorCompilable()) return; } - auto description_dump = getSortDescriptionDump(description, description_types); + auto description_dump = getSortDescriptionDump(description, sort_description_types); SipHash sort_description_dump_hash; sort_description_dump_hash.update(description_dump); @@ -100,6 +103,16 @@ void compileSortDescriptionIfNeeded(SortDescription & description, const DataTyp UInt128 sort_description_hash_key; sort_description_dump_hash.get128(sort_description_hash_key); + { + std::lock_guard lock(mutex); + UInt64 & current_counter = counter[sort_description_hash_key]; + if (current_counter < description.min_count_to_compile_sort_description) + { + current_counter += static_cast(increase_compile_attemps); + return; + } + } + std::shared_ptr compiled_sort_description_holder; if (auto * compilation_cache = CompiledExpressionCacheFactory::instance().tryGetCache()) @@ -108,7 +121,7 @@ void compileSortDescriptionIfNeeded(SortDescription & description, const DataTyp { LOG_TRACE(getLogger(), "Compile sort description {}", description_dump); - auto compiled_sort_description = compileSortDescription(getJITInstance(), description, description_types, description_dump); + auto compiled_sort_description = compileSortDescription(getJITInstance(), description, sort_description_types, description_dump); return std::make_shared(std::move(compiled_sort_description)); }); @@ -117,7 +130,7 @@ void compileSortDescriptionIfNeeded(SortDescription & description, const DataTyp else { LOG_TRACE(getLogger(), "Compile sort description {}", description_dump); - auto compiled_sort_description = compileSortDescription(getJITInstance(), description, description_types, description_dump); + auto compiled_sort_description = compileSortDescription(getJITInstance(), description, sort_description_types, description_dump); compiled_sort_description_holder = std::make_shared(std::move(compiled_sort_description)); } diff --git a/src/Core/SortDescription.h b/src/Core/SortDescription.h index 2063f00f8d7..ff053280a9e 100644 --- a/src/Core/SortDescription.h +++ b/src/Core/SortDescription.h @@ -97,13 +97,18 @@ using SortDescriptionWithPositions = std::vector { public: + /// Can be safely casted into JITSortDescriptionFunc void * compiled_sort_description = nullptr; - bool compile_sort_description = false; - size_t min_count_to_compile_sort_description = 3; std::shared_ptr compiled_sort_description_holder; + size_t min_count_to_compile_sort_description = 3; + bool compile_sort_description = false; }; -void compileSortDescriptionIfNeeded(SortDescription & description, const DataTypes & description_types); +/** Compile sort description for header_types. + * Description is compiled for header only if compile_sort_description = true and attemps to compile sort descriptions + * with unique combination of description and header will be greater or equal min_count_to_compile_sort_description. + */ +void compileSortDescriptionIfNeeded(SortDescription & description, const DataTypes & sort_description_types, bool increase_compile_attemps); /// Outputs user-readable description into `out`. void dumpSortDescription(const SortDescription & description, WriteBuffer & out); diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index 81603d6626d..d0bf4939c90 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -590,7 +590,7 @@ void MergeJoin::mergeInMemoryRightBlocks() /// TODO: there should be no split keys by blocks for RIGHT|FULL JOIN builder.addTransform(std::make_shared( - builder.getHeader(), right_sort_description, max_rows_in_right_block, 0, 0, 0, 0, nullptr, 0)); + builder.getHeader(), right_sort_description, max_rows_in_right_block, 0, false, 0, 0, 0, nullptr, 0)); auto pipeline = QueryPipelineBuilder::getPipeline(std::move(builder)); PullingPipelineExecutor executor(pipeline); diff --git a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp index 435a376bdb8..e437304f6d2 100644 --- a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp @@ -1,6 +1,8 @@ #include #include #include +#include +#include namespace DB { @@ -36,7 +38,7 @@ MergingSortedAlgorithm::MergingSortedAlgorithm( sort_description_types.emplace_back(header.getByName(column_description.column_name).type); } - compileSortDescriptionIfNeeded(description, sort_description_types); + compileSortDescriptionIfNeeded(description, sort_description_types, true /*increase_unique_count*/); } void MergingSortedAlgorithm::addInput() diff --git a/src/Processors/QueryPlan/SortingStep.cpp b/src/Processors/QueryPlan/SortingStep.cpp index b675c4a8815..4a430853b68 100644 --- a/src/Processors/QueryPlan/SortingStep.cpp +++ b/src/Processors/QueryPlan/SortingStep.cpp @@ -140,11 +140,21 @@ void SortingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build return std::make_shared(header, result_description, limit); }); + bool increase_sort_description_compile_attempts = true; + /// NOTE limits are not applied to the size of temporary sets in FinishSortingTransform - pipeline.addSimpleTransform([&](const Block & header) -> ProcessorPtr + pipeline.addSimpleTransform([&, increase_sort_description_compile_attempts](const Block & header) mutable -> ProcessorPtr { + /** For multiple FinishSortingTransform we need to count identical comparators only once per QueryPlan + * To property support min_count_to_compile_sort_description. + */ + bool increase_sort_description_compile_attempts_current = increase_sort_description_compile_attempts; + + if (increase_sort_description_compile_attempts) + increase_sort_description_compile_attempts = false; + return std::make_shared( - header, prefix_description, result_description, max_block_size, limit); + header, prefix_description, result_description, max_block_size, limit, increase_sort_description_compile_attempts_current); }); } } @@ -171,24 +181,33 @@ void SortingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build return transform; }); - pipeline.addSimpleTransform([&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr + bool increase_sort_description_compile_attempts = true; + + pipeline.addSimpleTransform([&, increase_sort_description_compile_attempts](const Block & header, QueryPipelineBuilder::StreamType stream_type) mutable -> ProcessorPtr { if (stream_type == QueryPipelineBuilder::StreamType::Totals) return nullptr; + /** For multiple FinishSortingTransform we need to count identical comparators only once per QueryPlan. + * To property support min_count_to_compile_sort_description. + */ + bool increase_sort_description_compile_attempts_current = increase_sort_description_compile_attempts; + + if (increase_sort_description_compile_attempts) + increase_sort_description_compile_attempts = false; + return std::make_shared( - header, result_description, max_block_size, limit, - max_bytes_before_remerge / pipeline.getNumStreams(), - remerge_lowered_memory_bytes_ratio, - max_bytes_before_external_sort, - tmp_volume, - min_free_disk_space); + header, result_description, max_block_size, limit, increase_sort_description_compile_attempts_current, + max_bytes_before_remerge / pipeline.getNumStreams(), + remerge_lowered_memory_bytes_ratio, + max_bytes_before_external_sort, + tmp_volume, + min_free_disk_space); }); /// If there are several streams, then we merge them into one if (pipeline.getNumStreams() > 1) { - auto transform = std::make_shared( pipeline.getHeader(), pipeline.getNumStreams(), @@ -203,7 +222,6 @@ void SortingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build { /// If there are several streams, then we merge them into one if (pipeline.getNumStreams() > 1) { - auto transform = std::make_shared( pipeline.getHeader(), pipeline.getNumStreams(), diff --git a/src/Processors/Transforms/FinishSortingTransform.cpp b/src/Processors/Transforms/FinishSortingTransform.cpp index abded9bd2f0..3842e034de7 100644 --- a/src/Processors/Transforms/FinishSortingTransform.cpp +++ b/src/Processors/Transforms/FinishSortingTransform.cpp @@ -25,8 +25,9 @@ FinishSortingTransform::FinishSortingTransform( const SortDescription & description_sorted_, const SortDescription & description_to_sort_, size_t max_merged_block_size_, - UInt64 limit_) - : SortingTransform(header, description_to_sort_, max_merged_block_size_, limit_) + UInt64 limit_, + bool increase_sort_description_compile_attempts) + : SortingTransform(header, description_to_sort_, max_merged_block_size_, limit_, increase_sort_description_compile_attempts) { /// Check for sanity non-modified descriptions if (!isPrefix(description_sorted_, description_to_sort_)) diff --git a/src/Processors/Transforms/FinishSortingTransform.h b/src/Processors/Transforms/FinishSortingTransform.h index 3bebcc0a68f..99a293fc1b0 100644 --- a/src/Processors/Transforms/FinishSortingTransform.h +++ b/src/Processors/Transforms/FinishSortingTransform.h @@ -16,7 +16,8 @@ public: const SortDescription & description_sorted_, const SortDescription & description_to_sort_, size_t max_merged_block_size_, - UInt64 limit_); + UInt64 limit_, + bool increase_sort_description_compile_attempts); String getName() const override { return "FinishSortingTransform"; } diff --git a/src/Processors/Transforms/MergeSortingTransform.cpp b/src/Processors/Transforms/MergeSortingTransform.cpp index 1fe945cbbc9..0c4615e9273 100644 --- a/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/src/Processors/Transforms/MergeSortingTransform.cpp @@ -92,12 +92,13 @@ MergeSortingTransform::MergeSortingTransform( const SortDescription & description_, size_t max_merged_block_size_, UInt64 limit_, + bool increase_sort_description_compile_attempts, size_t max_bytes_before_remerge_, double remerge_lowered_memory_bytes_ratio_, size_t max_bytes_before_external_sort_, VolumePtr tmp_volume_, size_t min_free_disk_space_) - : SortingTransform(header, description_, max_merged_block_size_, limit_) + : SortingTransform(header, description_, max_merged_block_size_, limit_, increase_sort_description_compile_attempts) , max_bytes_before_remerge(max_bytes_before_remerge_) , remerge_lowered_memory_bytes_ratio(remerge_lowered_memory_bytes_ratio_) , max_bytes_before_external_sort(max_bytes_before_external_sort_) diff --git a/src/Processors/Transforms/MergeSortingTransform.h b/src/Processors/Transforms/MergeSortingTransform.h index 34a4842949b..e118a2a655b 100644 --- a/src/Processors/Transforms/MergeSortingTransform.h +++ b/src/Processors/Transforms/MergeSortingTransform.h @@ -23,6 +23,7 @@ public: const SortDescription & description_, size_t max_merged_block_size_, UInt64 limit_, + bool increase_sort_description_compile_attempts, size_t max_bytes_before_remerge_, double remerge_lowered_memory_bytes_ratio_, size_t max_bytes_before_external_sort_, diff --git a/src/Processors/Transforms/SortingTransform.cpp b/src/Processors/Transforms/SortingTransform.cpp index ee197ae2de1..1ed79a44539 100644 --- a/src/Processors/Transforms/SortingTransform.cpp +++ b/src/Processors/Transforms/SortingTransform.cpp @@ -131,7 +131,8 @@ SortingTransform::SortingTransform( const Block & header, const SortDescription & description_, size_t max_merged_block_size_, - UInt64 limit_) + UInt64 limit_, + bool increase_sort_description_compile_attempts) : IProcessor({header}, {header}) , description(description_) , max_merged_block_size(max_merged_block_size_) @@ -175,7 +176,9 @@ SortingTransform::SortingTransform( description.swap(description_without_constants); - compileSortDescriptionIfNeeded(description, sort_description_types); + std::cout << "SortingTransform::SortingTransform increase compile attemps " << increase_sort_description_compile_attempts << std::endl; + compileSortDescriptionIfNeeded(description, sort_description_types, increase_sort_description_compile_attempts /*increase_compile_attemps*/); + std::cout << "SortingTransform::SortingTransform compiled sort description " << description.compiled_sort_description << std::endl; } SortingTransform::~SortingTransform() = default; diff --git a/src/Processors/Transforms/SortingTransform.h b/src/Processors/Transforms/SortingTransform.h index e98a9f88b48..938fd68811b 100644 --- a/src/Processors/Transforms/SortingTransform.h +++ b/src/Processors/Transforms/SortingTransform.h @@ -69,7 +69,8 @@ public: SortingTransform(const Block & header, const SortDescription & description_, size_t max_merged_block_size_, - UInt64 limit_); + UInt64 limit_, + bool increase_sort_description_compile_attempts); ~SortingTransform() override; From b1c154ccb4c6fcce79117b46ae3a996c5e59f595 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 11 May 2022 13:27:03 +0200 Subject: [PATCH 138/171] Fixed style check --- src/Interpreters/ExpressionAnalyzer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 8b751c3b353..37965f11c83 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -908,7 +908,8 @@ void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr actions) bool compile_sort_description = current_context->getSettingsRef().compile_sort_description; size_t min_count_to_compile_sort_description = current_context->getSettingsRef().min_count_to_compile_sort_description; - for (auto & [_, window_description] : window_descriptions) { + for (auto & [_, window_description] : window_descriptions) + { window_description.full_sort_description.compile_sort_description = compile_sort_description; window_description.full_sort_description.min_count_to_compile_sort_description = min_count_to_compile_sort_description; From 496302cb1dd218a37b95de0d6892b2b9d0f39f75 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 11 May 2022 13:45:24 +0200 Subject: [PATCH 139/171] Fix tests --- src/Core/SortDescription.cpp | 4 ++-- src/Processors/Transforms/SortingTransform.cpp | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Core/SortDescription.cpp b/src/Core/SortDescription.cpp index 8d830ccb5da..86946b8b922 100644 --- a/src/Core/SortDescription.cpp +++ b/src/Core/SortDescription.cpp @@ -3,7 +3,7 @@ #include #include #include - +#include #include #include @@ -91,7 +91,7 @@ void compileSortDescriptionIfNeeded(SortDescription & description, const DataTyp for (const auto & type : sort_description_types) { - if (!type->createColumn()->isComparatorCompilable()) + if (!type->createColumn()->isComparatorCompilable() || !canBeNativeType(*type)) return; } diff --git a/src/Processors/Transforms/SortingTransform.cpp b/src/Processors/Transforms/SortingTransform.cpp index 1ed79a44539..05a5c966c6f 100644 --- a/src/Processors/Transforms/SortingTransform.cpp +++ b/src/Processors/Transforms/SortingTransform.cpp @@ -176,9 +176,7 @@ SortingTransform::SortingTransform( description.swap(description_without_constants); - std::cout << "SortingTransform::SortingTransform increase compile attemps " << increase_sort_description_compile_attempts << std::endl; compileSortDescriptionIfNeeded(description, sort_description_types, increase_sort_description_compile_attempts /*increase_compile_attemps*/); - std::cout << "SortingTransform::SortingTransform compiled sort description " << description.compiled_sort_description << std::endl; } SortingTransform::~SortingTransform() = default; From 437d70d4da1234afd986fcaee24083d72acb56a5 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 11 May 2022 15:35:37 +0200 Subject: [PATCH 140/171] Fixed tests --- src/Columns/ColumnNullable.cpp | 9 ++++++--- src/Columns/ColumnVector.cpp | 4 +--- src/Columns/IColumn.h | 4 ++-- src/Core/SortDescription.cpp | 5 +++-- src/Core/SortDescription.h | 3 +-- src/DataTypes/Native.h | 3 ++- .../Merges/Algorithms/MergingSortedAlgorithm.cpp | 2 +- src/Processors/QueryPlan/SortingStep.cpp | 12 ++++++------ src/Processors/Transforms/SortingTransform.cpp | 3 +-- src/Storages/MergeTree/MergeTask.cpp | 4 +++- tests/performance/jit_sort.xml | 2 +- 11 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 81af3c816e5..435e6bf1fbc 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -12,7 +12,6 @@ #if USE_EMBEDDED_COMPILER #include - #include #endif @@ -254,7 +253,8 @@ bool ColumnNullable::isComparatorCompilable() const return nested_column->isComparatorCompilable(); } -llvm::Value * ColumnNullable::compileComparator(llvm::IRBuilderBase & builder, llvm::Value * lhs, llvm::Value * rhs, llvm::Value * nan_direction_hint) const +llvm::Value * ColumnNullable::compileComparator(llvm::IRBuilderBase & builder, llvm::Value * lhs, llvm::Value * rhs, + llvm::Value * nan_direction_hint) const { llvm::IRBuilder<> & b = static_cast &>(builder); auto * head = b.GetInsertBlock(); @@ -280,6 +280,7 @@ llvm::Value * ColumnNullable::compileComparator(llvm::IRBuilderBase & builder, l // else // return lval_is_null ? null_direction_hint : -null_direction_hint; // } + b.SetInsertPoint(lhs_or_rhs_are_null_block); auto * lhs_equals_rhs_result = llvm::ConstantInt::getSigned(b.getInt8Ty(), 0); llvm::Value * lhs_and_rhs_are_null = b.CreateAnd(lhs_is_null_value, rhs_is_null_value); @@ -288,8 +289,10 @@ llvm::Value * ColumnNullable::compileComparator(llvm::IRBuilderBase & builder, l b.CreateBr(join_block); // getNestedColumn().compareAt(n, m, nested_rhs, null_direction_hint); + b.SetInsertPoint(lhs_rhs_are_not_null_block); - llvm::Value *lhs_rhs_are_not_null_block_result = nested_column->compileComparator(builder, lhs_unwrapped_value, rhs_unwrapped_value, nan_direction_hint); + llvm::Value * lhs_rhs_are_not_null_block_result + = nested_column->compileComparator(builder, lhs_unwrapped_value, rhs_unwrapped_value, nan_direction_hint); b.CreateBr(join_block); b.SetInsertPoint(join_block); diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 0cfc22736f8..04f2efab0d7 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -27,7 +27,6 @@ #if USE_EMBEDDED_COMPILER #include - #include #endif @@ -204,7 +203,7 @@ llvm::Value * ColumnVector::compileComparator(llvm::IRBuilderBase & builder, { llvm::IRBuilder<> & b = static_cast &>(builder); - if constexpr (std::is_arithmetic_v) + if constexpr (std::is_integral_v) { // a > b ? 1 : (a < b ? -1 : 0); @@ -223,7 +222,6 @@ llvm::Value * ColumnVector::compileComparator(llvm::IRBuilderBase & builder, else { throw Exception(ErrorCodes::LOGICAL_ERROR, "Method compileComparator is not supported for type {}", TypeName); - return nullptr; } } diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 65716d14ae2..f62f6c444b3 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -292,9 +292,9 @@ public: #if USE_EMBEDDED_COMPILER - virtual bool isComparatorCompilable() const { return false; } + [[nodiscard]] virtual bool isComparatorCompilable() const { return false; } - virtual llvm::Value * compileComparator(llvm::IRBuilderBase & /*builder*/, llvm::Value * /*lhs*/, llvm::Value * /*rhs*/, llvm::Value * /*nan_direction_hint*/) const + [[nodiscard]] virtual llvm::Value * compileComparator(llvm::IRBuilderBase & /*builder*/, llvm::Value * /*lhs*/, llvm::Value * /*rhs*/, llvm::Value * /*nan_direction_hint*/) const { throw Exception("Method compileComparator is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED); } diff --git a/src/Core/SortDescription.cpp b/src/Core/SortDescription.cpp index 86946b8b922..2edb5ea4022 100644 --- a/src/Core/SortDescription.cpp +++ b/src/Core/SortDescription.cpp @@ -3,9 +3,12 @@ #include #include #include + +#if USE_EMBEDDED_COMPILER #include #include #include +#endif namespace DB { @@ -63,8 +66,6 @@ public: CompiledSortDescriptionFunction compiled_sort_description_function; }; -static std::unordered_map sort_description_to_compiled_function; - static std::string getSortDescriptionDump(const SortDescription & description, const DataTypes & header_types) { WriteBufferFromOwnString buffer; diff --git a/src/Core/SortDescription.h b/src/Core/SortDescription.h index ff053280a9e..3d4e3b665ee 100644 --- a/src/Core/SortDescription.h +++ b/src/Core/SortDescription.h @@ -105,8 +105,7 @@ public: }; /** Compile sort description for header_types. - * Description is compiled for header only if compile_sort_description = true and attemps to compile sort descriptions - * with unique combination of description and header will be greater or equal min_count_to_compile_sort_description. + * Description is compiled only if compilation attempts to compile identical description is more than min_count_to_compile_sort_description. */ void compileSortDescriptionIfNeeded(SortDescription & description, const DataTypes & sort_description_types, bool increase_compile_attemps); diff --git a/src/DataTypes/Native.h b/src/DataTypes/Native.h index 26530164f75..1950b1407df 100644 --- a/src/DataTypes/Native.h +++ b/src/DataTypes/Native.h @@ -111,7 +111,8 @@ static inline bool canBeNativeType(const IDataType & type) return canBeNativeType(*data_type_nullable.getNestedType()); } - return data_type.isNativeInt() || data_type.isNativeUInt() || data_type.isFloat() || data_type.isDate() || data_type.isDateTime() || data_type.isEnum(); + return data_type.isNativeInt() || data_type.isNativeUInt() || data_type.isFloat() || data_type.isDate() + || data_type.isDate32() || data_type.isDateTime() || data_type.isEnum(); } static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder, const DataTypePtr & type) diff --git a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp index e437304f6d2..c5a72bf66bc 100644 --- a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp @@ -38,7 +38,7 @@ MergingSortedAlgorithm::MergingSortedAlgorithm( sort_description_types.emplace_back(header.getByName(column_description.column_name).type); } - compileSortDescriptionIfNeeded(description, sort_description_types, true /*increase_unique_count*/); + compileSortDescriptionIfNeeded(description, sort_description_types, true /*increase_compile_attemps*/); } void MergingSortedAlgorithm::addInput() diff --git a/src/Processors/QueryPlan/SortingStep.cpp b/src/Processors/QueryPlan/SortingStep.cpp index 4a430853b68..2d8fcf3d6cc 100644 --- a/src/Processors/QueryPlan/SortingStep.cpp +++ b/src/Processors/QueryPlan/SortingStep.cpp @@ -197,12 +197,12 @@ void SortingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build increase_sort_description_compile_attempts = false; return std::make_shared( - header, result_description, max_block_size, limit, increase_sort_description_compile_attempts_current, - max_bytes_before_remerge / pipeline.getNumStreams(), - remerge_lowered_memory_bytes_ratio, - max_bytes_before_external_sort, - tmp_volume, - min_free_disk_space); + header, result_description, max_block_size, limit, increase_sort_description_compile_attempts_current, + max_bytes_before_remerge / pipeline.getNumStreams(), + remerge_lowered_memory_bytes_ratio, + max_bytes_before_external_sort, + tmp_volume, + min_free_disk_space); }); /// If there are several streams, then we merge them into one diff --git a/src/Processors/Transforms/SortingTransform.cpp b/src/Processors/Transforms/SortingTransform.cpp index 05a5c966c6f..37a0bf0b7d1 100644 --- a/src/Processors/Transforms/SortingTransform.cpp +++ b/src/Processors/Transforms/SortingTransform.cpp @@ -168,8 +168,7 @@ SortingTransform::SortingTransform( if (new_pos < num_columns) { - auto type = sample.safeGetByPosition(old_pos).type; - sort_description_types.emplace_back(type); + sort_description_types.emplace_back(sample.safeGetByPosition(old_pos).type); description_without_constants.push_back(column_description); } } diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index d17adb0c602..58bffaab34b 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -778,6 +778,9 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() Names sort_columns = global_ctx->metadata_snapshot->getSortingKeyColumns(); SortDescription sort_description; + sort_description.compile_sort_description = global_ctx->data->getContext()->getSettingsRef().compile_sort_description; + sort_description.min_count_to_compile_sort_description = global_ctx->data->getContext()->getSettingsRef().min_count_to_compile_sort_description; + size_t sort_columns_size = sort_columns.size(); sort_description.reserve(sort_columns_size); @@ -796,7 +799,6 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() ctx->blocks_are_granules_size = (global_ctx->chosen_merge_algorithm == MergeAlgorithm::Vertical); UInt64 merge_block_size = data_settings->merge_max_block_size; - sort_description.compile_sort_description = global_ctx->data->getContext()->getSettingsRef().compile_sort_description; switch (ctx->merging_params.mode) { diff --git a/tests/performance/jit_sort.xml b/tests/performance/jit_sort.xml index d11b4351f56..f99124a360b 100644 --- a/tests/performance/jit_sort.xml +++ b/tests/performance/jit_sort.xml @@ -119,4 +119,4 @@ DROP TABLE IF EXISTS {table} - \ No newline at end of file + From e8c024ac5cb517017e6749c8b49fc745d38eea5b Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 11 May 2022 16:21:32 +0200 Subject: [PATCH 141/171] Fixed build --- src/Core/SortDescription.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Core/SortDescription.cpp b/src/Core/SortDescription.cpp index 2edb5ea4022..e7fa6816807 100644 --- a/src/Core/SortDescription.cpp +++ b/src/Core/SortDescription.cpp @@ -142,10 +142,11 @@ void compileSortDescriptionIfNeeded(SortDescription & description, const DataTyp #else -void compileSortDescriptionIfNeeded(SortDescription & description, const DataTypes & description_types) +void compileSortDescriptionIfNeeded(SortDescription & description, const DataTypes & sort_description_types, bool increase_compile_attemps) { (void)(description); - (void)(description_types); + (void)(sort_description_types); + (void)(increase_compile_attemps); } #endif From 3d35e26e18d1f8a8faa368212ebbb14ce2e71a5e Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 11 May 2022 22:26:38 +0200 Subject: [PATCH 142/171] Update test --- tests/queries/0_stateless/02226_s3_with_cache.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02226_s3_with_cache.reference b/tests/queries/0_stateless/02226_s3_with_cache.reference index 4041f51b3f9..596e6e5345a 100644 --- a/tests/queries/0_stateless/02226_s3_with_cache.reference +++ b/tests/queries/0_stateless/02226_s3_with_cache.reference @@ -1,4 +1,4 @@ SELECT 1, * FROM test LIMIT 10 FORMAT Null; 1 0 1 SELECT 2, * FROM test LIMIT 10 FORMAT Null; 0 1 0 0 -SELECT 3, * FROM test LIMIT 10 FORMAT Null; 1 1 0 +SELECT 3, * FROM test LIMIT 10 FORMAT Null; 0 1 0 From cf96f3ce3f7ef2458e31c67188d4ff99cac64fdc Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 11 May 2022 22:52:30 +0200 Subject: [PATCH 143/171] Update Progress.cpp --- src/IO/Progress.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/IO/Progress.cpp b/src/IO/Progress.cpp index 2993319fff0..45710db5c92 100644 --- a/src/IO/Progress.cpp +++ b/src/IO/Progress.cpp @@ -159,9 +159,7 @@ void Progress::read(ReadBuffer & in, UInt64 server_revision) read_rows.store(values.read_rows, std::memory_order_relaxed); read_bytes.store(values.read_bytes, std::memory_order_relaxed); - total_rows_to_read.store(values.total_rows_to_read, std::memory_order_relaxed); - total_bytes_to_read.store(values.total_bytes_to_read, std::memory_order_relaxed); written_rows.store(values.written_rows, std::memory_order_relaxed); written_bytes.store(values.written_bytes, std::memory_order_relaxed); From 810967fae3a120105f81b00fb3c7a117e5a578c2 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 11 May 2022 23:01:25 +0200 Subject: [PATCH 144/171] Remove the check back --- src/Disks/IO/ReadBufferFromRemoteFSGather.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp index 08707965e2f..57b72d0190d 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp @@ -103,16 +103,13 @@ ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather( , common_path_prefix(common_path_prefix_) , blobs_to_read(blobs_to_read_) , settings(settings_) - , query_id(CurrentThread::getQueryId()) + , query_id(CurrentThread::isInitialized() && CurrentThread::get().getQueryContext() != nullptr ? CurrentThread::getQueryId() : "") , log(&Poco::Logger::get("ReadBufferFromRemoteFSGather")) , enable_cache_log(!query_id.empty() && settings.enable_filesystem_cache_log) { with_cache = settings.remote_fs_cache && settings.enable_filesystem_cache && (!IFileCache::isReadOnly() || settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache); - - assert(query_id.empty() - || CurrentThread::isInitialized() && CurrentThread::get().getQueryContext().get() != nullptr); } From 2d15520ddb7ab5a1045b2dd01d9d2a126ab9d2d6 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 11 May 2022 23:33:15 +0200 Subject: [PATCH 145/171] Update FileSegment.cpp --- src/Common/FileSegment.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index 386be0da75d..dc27ec4dd6e 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -348,8 +348,7 @@ FileSegment::State FileSegment::wait() if (is_detached) throw Exception( ErrorCodes::LOGICAL_ERROR, - "Cache file segment is in detached state, operation not allowed. " - "It can happen when cache was concurrently dropped with SYSTEM DROP FILESYSTEM CACHE FORCE"); + "Cache file segment is in detached state, operation not allowed"); if (downloader_id.empty()) return download_state; @@ -734,6 +733,11 @@ void FileSegment::detach( std::lock_guard & /* cache_lock */, std::lock_guard & segment_lock) { + /// Now detached status can be in 2 cases, which do not do any complex logic: + /// 1. there is only 1 remaining file segment holder + /// && it does not need this segment anymore + /// && this file segment was in cache and needs to be removed + /// 2. in read_from_cache_if_exists_otherwise_bypass_cache case if (is_detached) return; From 4483e5783a781dbd0f26a4579bca5d6746f465d5 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 11 May 2022 23:33:38 +0200 Subject: [PATCH 146/171] Update FileSegment.h --- src/Common/FileSegment.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Common/FileSegment.h b/src/Common/FileSegment.h index 38a12311bb7..7a25529ab23 100644 --- a/src/Common/FileSegment.h +++ b/src/Common/FileSegment.h @@ -231,7 +231,6 @@ private: /// "detached" file segment means that it is not owned by cache ("detached" from cache). /// In general case, all file segments are owned by cache. bool is_detached = false; - bool is_forcefully_detached = false; std::atomic is_downloaded{false}; std::atomic hits_count = 0; /// cache hits. From e9733f0cbe5c0a45a326b50526fbbdc5ff551797 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 12 May 2022 09:18:28 +0200 Subject: [PATCH 147/171] Fix fasttest ccache permissions --- tests/ci/fast_test_check.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index d659245dad9..acc07ccfb23 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -84,7 +84,6 @@ if __name__ == "__main__": stopwatch = Stopwatch() temp_path = TEMP_PATH - caches_path = CACHES_PATH if not os.path.exists(temp_path): os.makedirs(temp_path) @@ -110,7 +109,10 @@ if __name__ == "__main__": if not os.path.exists(output_path): os.makedirs(output_path) - cache_path = os.path.join(caches_path, "fasttest") + if not os.path.exists(CACHES_PATH): + os.makedirs(CACHES_PATH) + subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {CACHES_PATH}", shell=True) + cache_path = os.path.join(CACHES_PATH, "fasttest") logging.info("Will try to fetch cache for our build") ccache_for_pr = get_ccache_if_not_exists( From a15dacf80ffa9567b2b014d57d487f4e7e7e3082 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 11 May 2022 23:09:38 +0200 Subject: [PATCH 148/171] Simplify and update MacOS build instructions - instead of repeating Homebrew's installation steps (which may become outdated), just point to their homepage - Specify CC and CXX env variables instead of internal CMake variables, CMake figure out paths to ar, ranlib and objcopy automatically - use -S and -B parameters available in recent CMake versions to build directly from the source directory --- docs/en/development/build-osx.md | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/docs/en/development/build-osx.md b/docs/en/development/build-osx.md index 05ef10ad020..943e41e059b 100644 --- a/docs/en/development/build-osx.md +++ b/docs/en/development/build-osx.md @@ -10,21 +10,17 @@ description: How to build ClickHouse on Mac OS X You can install pre-built ClickHouse as described in [Quick Start](https://clickhouse.com/#quick-start). Follow **macOS (Intel)** or **macOS (Apple silicon)** installation instructions. ::: -Build should work on x86_64 (Intel) and arm64 (Apple silicon) based macOS 10.15 (Catalina) and higher with Homebrew's vanilla Clang. -It is always recommended to use vanilla `clang` compiler. +The build works on x86_64 (Intel) and arm64 (Apple Silicon) based on macOS 10.15 (Catalina) or higher with Homebrew's vanilla Clang. :::note -It is possible to use XCode's `apple-clang` or `gcc`, but it's strongly discouraged. +It is also possible to compile with Apple's XCode `apple-clang` or Homebrew's `gcc`, but it's strongly discouraged. ::: ## Install Homebrew {#install-homebrew} -``` bash -/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" -# ...and follow the printed instructions on any additional steps required to complete the installation. -``` +First install [Homebrew](https://brew.sh/) -## Install Xcode and Command Line Tools {#install-xcode-and-command-line-tools} +## For Apple's Clang (discouraged): Install Xcode and Command Line Tools {#install-xcode-and-command-line-tools} Install the latest [Xcode](https://apps.apple.com/am/app/xcode/id497799835?mt=12) from App Store. @@ -57,12 +53,12 @@ To build using Homebrew's vanilla Clang compiler (the only **recommended** way): ``` bash cd ClickHouse -rm -rf build mkdir build -cd build -cmake -DCMAKE_C_COMPILER=$(brew --prefix llvm)/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix llvm)/bin/clang++ -DCMAKE_AR=$(brew --prefix llvm)/bin/llvm-ar -DCMAKE_RANLIB=$(brew --prefix llvm)/bin/llvm-ranlib -DOBJCOPY_PATH=$(brew --prefix llvm)/bin/llvm-objcopy -DCMAKE_BUILD_TYPE=RelWithDebInfo .. -cmake --build . --config RelWithDebInfo -# The resulting binary will be created at: ./programs/clickhouse +export CC=$(brew --prefix llvm)/bin/clang +export CXX=$(brew --prefix llvm)/bin/clang++ +cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -S . -B build +cmake --build build +# The resulting binary will be created at: build/programs/clickhouse ``` To build using Xcode's native AppleClang compiler in Xcode IDE (this option is only for development builds and workflows, and is **not recommended** unless you know what you are doing): @@ -82,12 +78,12 @@ To build using Homebrew's vanilla GCC compiler (this option is only for developm ``` bash cd ClickHouse -rm -rf build mkdir build -cd build -cmake -DCMAKE_C_COMPILER=$(brew --prefix gcc)/bin/gcc-11 -DCMAKE_CXX_COMPILER=$(brew --prefix gcc)/bin/g++-11 -DCMAKE_AR=$(brew --prefix gcc)/bin/gcc-ar-11 -DCMAKE_RANLIB=$(brew --prefix gcc)/bin/gcc-ranlib-11 -DOBJCOPY_PATH=$(brew --prefix binutils)/bin/objcopy -DCMAKE_BUILD_TYPE=RelWithDebInfo .. -cmake --build . --config RelWithDebInfo -# The resulting binary will be created at: ./programs/clickhouse +export CC=$(brew --prefix gcc)/bin/gcc-11 +export CXX=$(brew --prefix gcc)/bin/g++-11 +cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -S . -B build +cmake --build build +# The resulting binary will be created at: build/programs/clickhouse ``` ## Caveats {#caveats} From d636a02e7a20a60d6e9642de8337a6786865de32 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 12 May 2022 12:16:12 +0200 Subject: [PATCH 149/171] Update FileSegment.cpp --- src/Common/FileSegment.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index dc27ec4dd6e..356ba8bf55f 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -735,7 +735,7 @@ void FileSegment::detach( { /// Now detached status can be in 2 cases, which do not do any complex logic: /// 1. there is only 1 remaining file segment holder - /// && it does not need this segment anymore + /// && it does not need this segment anymore /// && this file segment was in cache and needs to be removed /// 2. in read_from_cache_if_exists_otherwise_bypass_cache case if (is_detached) From 5ae28f6fd8cf2275e0236facd1df5200fc729834 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 12 May 2022 12:28:48 +0200 Subject: [PATCH 150/171] Update ProfileEvents.cpp --- src/Common/ProfileEvents.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 5a3982f1ec1..72ef16f2730 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -260,12 +260,12 @@ \ M(QueryMemoryLimitExceeded, "Number of times when memory limit exceeded for query.") \ \ - M(CachedReadBufferReadFromSourceMicroseconds, "") \ - M(CachedReadBufferReadFromCacheMicroseconds, "") \ - M(CachedReadBufferReadFromSourceBytes, "") \ - M(CachedReadBufferReadFromCacheBytes, "") \ - M(CachedReadBufferCacheWriteBytes, "") \ - M(CachedReadBufferCacheWriteMicroseconds, "") \ + M(CachedReadBufferReadFromSourceMicroseconds, "Time reading from source (remote fs, etc)") \ + M(CachedReadBufferReadFromCacheMicroseconds, "Time reading from cache") \ + M(CachedReadBufferReadFromSourceBytes, "Bytes read from source (remote fs, etc)") \ + M(CachedReadBufferReadFromCacheBytes, "Bytes read from cache") \ + M(CachedReadBufferCacheWriteBytes, "Bytes written from source to cache") \ + M(CachedReadBufferCacheWriteMicroseconds, "Time spent writing data into cache") \ \ M(RemoteFSSeeks, "Total number of seeks for async buffer") \ M(RemoteFSPrefetches, "Number of prefetches made with asynchronous reading from remote filesystem") \ From 355c5443a015aa17a435716f29712605745f776f Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Wed, 11 May 2022 23:20:39 +0200 Subject: [PATCH 151/171] Trying to fix sanitizer failure --- src/Functions/array/arrayDistance.cpp | 31 ++++++++++++++++----------- src/Functions/array/arrayNorm.cpp | 6 ++++-- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index ae7d89f8db4..697cf57c77d 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -5,6 +5,8 @@ #include #include #include +#include "Columns/IColumn.h" +#include "DataTypes/Serializations/ISerialization.h" namespace DB { @@ -22,7 +24,7 @@ struct LpDistance template static void compute(const Eigen::MatrixX & left, const Eigen::MatrixX & right, PaddedPODArray & array) { - auto & norms = (left - right).colwise().template lpNorm(); + auto norms = (left - right).colwise().template lpNorm(); array.reserve(norms.size()); // array.insert() failed to work with Eigen iterators for (auto n : norms) @@ -41,11 +43,11 @@ struct CosineDistance template static void compute(const Eigen::MatrixX & left, const Eigen::MatrixX & right, PaddedPODArray & array) { - auto & prod = left.cwiseProduct(right).colwise().sum(); - auto & nx = left.colwise().norm(); - auto & ny = right.colwise().norm(); - auto & nm = nx.cwiseProduct(ny).cwiseInverse(); - auto & dist = 1.0 - prod.cwiseProduct(nm).array(); + auto prod = left.cwiseProduct(right).colwise().sum(); + auto nx = left.colwise().norm(); + auto ny = right.colwise().norm(); + auto nm = nx.cwiseProduct(ny).cwiseInverse(); + auto dist = 1.0 - prod.cwiseProduct(nm).array(); array.reserve(dist.size()); for (auto d : dist) array.push_back(d); @@ -100,11 +102,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override { - const auto & type_x = typeid_cast(arguments[0].type.get())->getNestedType(); - const auto & type_y = typeid_cast(arguments[1].type.get())->getNestedType(); + DataTypePtr type_x = typeid_cast(arguments[0].type.get())->getNestedType(); + DataTypePtr type_y = typeid_cast(arguments[1].type.get())->getNestedType(); - const auto & col_x = arguments[0].column->convertToFullColumnIfConst(); - const auto & col_y = arguments[1].column->convertToFullColumnIfConst(); + ColumnPtr col_x = arguments[0].column->convertToFullColumnIfConst(); + ColumnPtr col_y = arguments[1].column->convertToFullColumnIfConst(); const auto * arr_x = assert_cast(col_x.get()); const auto * arr_y = assert_cast(col_y.get()); @@ -196,7 +198,10 @@ private: { const auto & data = typeid_cast &>(array.getData()).getData(); const auto & offsets = array.getOffsets(); - mat = Eigen::Map>(data.data(), offsets.front(), offsets.size()); + size_t cols = offsets.size(); + size_t rows = cols > 0 ? offsets.front() : 0; + + mat = Eigen::Map>(data.data(), rows, cols); } template @@ -204,7 +209,9 @@ private: { const auto & data = typeid_cast &>(array.getData()).getData(); const auto & offsets = array.getOffsets(); - size_t rows = offsets.front(), cols = offsets.size(); + size_t cols = offsets.size(); + size_t rows = cols > 0 ? offsets.front() : 0; + mat.resize(rows, cols); ColumnArray::Offset prev = 0, col = 0; diff --git a/src/Functions/array/arrayNorm.cpp b/src/Functions/array/arrayNorm.cpp index b267b1d726e..78d4c03f3ba 100644 --- a/src/Functions/array/arrayNorm.cpp +++ b/src/Functions/array/arrayNorm.cpp @@ -5,6 +5,8 @@ #include #include #include +#include "Columns/IColumn.h" +#include "DataTypes/Serializations/ISerialization.h" namespace DB { @@ -82,8 +84,8 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override { - const auto & type = typeid_cast(arguments[0].type.get())->getNestedType(); - const auto column = arguments[0].column->convertToFullColumnIfConst(); + DataTypePtr type = typeid_cast(arguments[0].type.get())->getNestedType(); + ColumnPtr column = arguments[0].column->convertToFullColumnIfConst(); const auto * arr = assert_cast(column.get()); auto result = result_type->createColumn(); From 669637108c8f096aa899688f3ddc398c51c57d9f Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Thu, 12 May 2022 14:28:29 +0200 Subject: [PATCH 152/171] Added tests for special cases with float/double --- .../02282_array_distance.reference | 12 +++++++++ .../0_stateless/02282_array_distance.sql | 12 +++++++++ .../0_stateless/02283_array_norm.reference | 16 +++++++++++ .../queries/0_stateless/02283_array_norm.sql | 27 ++++++++++++++----- 4 files changed, 61 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/02282_array_distance.reference b/tests/queries/0_stateless/02282_array_distance.reference index 75e24833e1e..158df656403 100644 --- a/tests/queries/0_stateless/02282_array_distance.reference +++ b/tests/queries/0_stateless/02282_array_distance.reference @@ -28,3 +28,15 @@ nan 2 2 1347.4008312302617 3 1 214.35251339790725 3 2 1342.8856987845243 +1 1 218.74643 +1 2 1348.2118 +2 1 219.28064 +2 2 1347.4009 +3 1 214.35251 +3 2 1342.8857 +1 1 218.74642854227358 +1 2 1348.2117786164013 +2 1 219.28064210048274 +2 2 1347.4008312302617 +3 1 214.35251339790725 +3 2 1342.8856987845243 diff --git a/tests/queries/0_stateless/02282_array_distance.sql b/tests/queries/0_stateless/02282_array_distance.sql index 0f269b11179..04de01d7d66 100644 --- a/tests/queries/0_stateless/02282_array_distance.sql +++ b/tests/queries/0_stateless/02282_array_distance.sql @@ -8,8 +8,12 @@ SELECT arrayCosineDistance([1, 2, 3], [0, 0, 0]); DROP TABLE IF EXISTS vec1; DROP TABLE IF EXISTS vec2; +DROP TABLE IF EXISTS vec2f; +DROP TABLE IF EXISTS vec2d; CREATE TABLE vec1 (id UInt64, v Array(UInt8)) ENGINE = Memory; CREATE TABLE vec2 (id UInt64, v Array(Int64)) ENGINE = Memory; +CREATE TABLE vec2f (id UInt64, v Array(Float32)) ENGINE = Memory; +CREATE TABLE vec2d (id UInt64, v Array(Float64)) ENGINE = Memory; INSERT INTO vec1 VALUES (1, [3, 4, 5]), (2, [2, 4, 8]), (3, [7, 7, 7]); SELECT arrayL1Distance(v, [0, 0, 0]) FROM vec1; @@ -22,8 +26,16 @@ SELECT arrayCosineDistance(v, materialize([1., 1., 1.])) FROM vec1; INSERT INTO vec2 VALUES (1, [100, 200, 0]), (2, [888, 777, 666]); SELECT v1.id, v2.id, arrayL2Distance(v1.v, v2.v) as dist FROM vec1 v1, vec2 v2; +INSERT INTO vec2f VALUES (1, [100, 200, 0]), (2, [888, 777, 666]); +SELECT v1.id, v2.id, arrayL2Distance(v1.v, v2.v) as dist FROM vec1 v1, vec2f v2; + +INSERT INTO vec2d VALUES (1, [100, 200, 0]), (2, [888, 777, 666]); +SELECT v1.id, v2.id, arrayL2Distance(v1.v, v2.v) as dist FROM vec1 v1, vec2d v2; + SELECT arrayL1Distance([0, 0], [1]); -- { serverError 190 } SELECT arrayL2Distance((1, 2), (3,4)); -- { serverError 43 } DROP TABLE vec1; DROP TABLE vec2; +DROP TABLE vec2f; +DROP TABLE vec2d; diff --git a/tests/queries/0_stateless/02283_array_norm.reference b/tests/queries/0_stateless/02283_array_norm.reference index 2d22f2967a2..6dd6b79e6d9 100644 --- a/tests/queries/0_stateless/02283_array_norm.reference +++ b/tests/queries/0_stateless/02283_array_norm.reference @@ -9,3 +9,19 @@ 2 11 3 11 4 11 +1 5 +2 2 +3 5.196152 +4 0 +1 11 +2 11 +3 11 +4 11 +1 5 +2 2 +3 5.196152422706632 +4 0 +1 11 +2 11 +3 11 +4 11 diff --git a/tests/queries/0_stateless/02283_array_norm.sql b/tests/queries/0_stateless/02283_array_norm.sql index e5a310bfa1e..e11caea7cc1 100644 --- a/tests/queries/0_stateless/02283_array_norm.sql +++ b/tests/queries/0_stateless/02283_array_norm.sql @@ -2,12 +2,27 @@ SELECT arrayL1Norm([1, 2, 3]); SELECT arrayL2Norm([3., 4., 5.]); SELECT arrayLinfNorm([0, 0, 2]); -DROP TABLE IF EXISTS vec; -CREATE TABLE vec (id UInt64, v Array(UInt8)) ENGINE = Memory; -INSERT INTO vec VALUES (1, [3, 4]), (2, [2]), (3, [3, 3, 3]), (4, NULL); +DROP TABLE IF EXISTS vec1; +DROP TABLE IF EXISTS vec1f; +DROP TABLE IF EXISTS vec1d; +CREATE TABLE vec1 (id UInt64, v Array(UInt8)) ENGINE = Memory; +CREATE TABLE vec1f (id UInt64, v Array(Float32)) ENGINE = Memory; +CREATE TABLE vec1d (id UInt64, v Array(Float64)) ENGINE = Memory; +INSERT INTO vec1 VALUES (1, [3, 4]), (2, [2]), (3, [3, 3, 3]), (4, NULL); +INSERT INTO vec1f VALUES (1, [3, 4]), (2, [2]), (3, [3, 3, 3]), (4, NULL); +INSERT INTO vec1d VALUES (1, [3, 4]), (2, [2]), (3, [3, 3, 3]), (4, NULL); + +SELECT id, arrayL2Norm(v) FROM vec1; +SELECT id, arrayL1Norm(materialize([5., 6.])) FROM vec1; + +SELECT id, arrayL2Norm(v) FROM vec1f; +SELECT id, arrayL1Norm(materialize([5., 6.])) FROM vec1f; + +SELECT id, arrayL2Norm(v) FROM vec1d; +SELECT id, arrayL1Norm(materialize([5., 6.])) FROM vec1d; -SELECT id, arrayL2Norm(v) FROM vec; -SELECT id, arrayL1Norm(materialize([5., 6.])) FROM vec; SELECT arrayL1Norm((1, 2,)); -- { serverError 43 } -DROP TABLE vec; +DROP TABLE vec1; +DROP TABLE vec1f; +DROP TABLE vec1d; From 7c226f60674dbf3cf9e3b005970daaa34f3454dd Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Thu, 12 May 2022 14:32:47 +0200 Subject: [PATCH 153/171] Fixed special case condition --- src/Functions/array/arrayDistance.cpp | 3 ++- src/Functions/array/arrayNorm.cpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index 697cf57c77d..94c20568668 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -193,7 +193,8 @@ private: } // optimize for float/ double - template ::value>::type> + template + requires (std::is_same_v) void fillMatrix(Eigen::MatrixX & mat, const ColumnArray & array) const { const auto & data = typeid_cast &>(array.getData()).getData(); diff --git a/src/Functions/array/arrayNorm.cpp b/src/Functions/array/arrayNorm.cpp index 78d4c03f3ba..390b11c5228 100644 --- a/src/Functions/array/arrayNorm.cpp +++ b/src/Functions/array/arrayNorm.cpp @@ -155,7 +155,8 @@ private: ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } } - template ::value>::type> + template + requires (std::is_same_v) void fillVectors(std::vector> & vec, const ColumnArray & array) const { const auto & data = typeid_cast &>(array.getData()).getData(); From 548625a003308eafd132d26e9d4762301d9fe93d Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Thu, 12 May 2022 14:33:20 +0200 Subject: [PATCH 154/171] Reserve result vectors --- src/Functions/array/arrayNorm.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Functions/array/arrayNorm.cpp b/src/Functions/array/arrayNorm.cpp index 390b11c5228..61da1c83147 100644 --- a/src/Functions/array/arrayNorm.cpp +++ b/src/Functions/array/arrayNorm.cpp @@ -161,6 +161,7 @@ private: { const auto & data = typeid_cast &>(array.getData()).getData(); const auto & offsets = array.getOffsets(); + vec.reserve(offsets.size()); ColumnArray::Offset prev = 0; for (auto off : offsets) { @@ -174,6 +175,7 @@ private: { const auto & data = typeid_cast &>(array.getData()).getData(); const auto & offsets = array.getOffsets(); + vec.reserve(offsets.size()); ColumnArray::Offset prev = 0; for (auto off : offsets) From 3c241582db425a102d53df36839467872a1e2e85 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 12 May 2022 14:54:18 +0200 Subject: [PATCH 155/171] Fix assertions --- src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp index d1e4b9fa21e..ed369b0e414 100644 --- a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp @@ -184,7 +184,7 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::nextImpl() LOG_TEST(log, "Current size: {}, offset: {}", size, offset); /// If prefetch_future is valid, size should always be greater than zero. - assert(offset < size); + assert(offset <= size); ProfileEvents::increment(ProfileEvents::AsynchronousReadWaitMicroseconds, watch.elapsedMicroseconds()); } @@ -201,7 +201,7 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::nextImpl() auto offset = result.offset; LOG_TEST(log, "Current size: {}, offset: {}", size, offset); - assert(offset < size); + assert(offset <= size); if (size) { From 7700d51ae4a232a57e132b2a6d8c0d352cbf6b77 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 12 May 2022 15:02:07 +0200 Subject: [PATCH 156/171] Update profile events descriptions --- src/Common/ProfileEvents.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 72ef16f2730..7f3b9788c1f 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -260,12 +260,12 @@ \ M(QueryMemoryLimitExceeded, "Number of times when memory limit exceeded for query.") \ \ - M(CachedReadBufferReadFromSourceMicroseconds, "Time reading from source (remote fs, etc)") \ - M(CachedReadBufferReadFromCacheMicroseconds, "Time reading from cache") \ - M(CachedReadBufferReadFromSourceBytes, "Bytes read from source (remote fs, etc)") \ - M(CachedReadBufferReadFromCacheBytes, "Bytes read from cache") \ - M(CachedReadBufferCacheWriteBytes, "Bytes written from source to cache") \ - M(CachedReadBufferCacheWriteMicroseconds, "Time spent writing data into cache") \ + M(CachedReadBufferReadFromSourceMicroseconds, "Time reading from filesystem cache source (from remote filesystem, etc)") \ + M(CachedReadBufferReadFromCacheMicroseconds, "Time reading from filesystem cache") \ + M(CachedReadBufferReadFromSourceBytes, "Bytes read from filesystem cache source (from remote fs, etc)") \ + M(CachedReadBufferReadFromCacheBytes, "Bytes read from filesystem cache") \ + M(CachedReadBufferCacheWriteBytes, "Bytes written from source (remote fs, etc) to filesystem cache") \ + M(CachedReadBufferCacheWriteMicroseconds, "Time spent writing data into filesystem cache") \ \ M(RemoteFSSeeks, "Total number of seeks for async buffer") \ M(RemoteFSPrefetches, "Number of prefetches made with asynchronous reading from remote filesystem") \ From 586eda524226509591828ec790adab052f679143 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 12 May 2022 16:12:16 +0300 Subject: [PATCH 157/171] Fix implicit cast for optimize_skip_unused_shards_rewrite_in Signed-off-by: Azat Khuzhin Fixes: #37073 --- src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp | 6 ++---- .../01756_optimize_skip_unused_shards_rewrite_in.sql | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp index 991b449196d..18ca70c7c09 100644 --- a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp +++ b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp @@ -39,10 +39,8 @@ bool shardContains( const std::string & sharding_column_name, const OptimizeShardingKeyRewriteInMatcher::Data & data) { - UInt64 field_value; - /// Convert value to numeric (if required). - if (!sharding_column_value.tryGet(field_value)) - sharding_column_value = convertFieldToType(sharding_column_value, *data.sharding_key_type); + /// Implicit conversion. + sharding_column_value = convertFieldToType(sharding_column_value, *data.sharding_key_type); /// NULL is not allowed in sharding key, /// so it should be safe to assume that shard cannot contain it. diff --git a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql index a5090551c89..a62af2d5667 100644 --- a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql +++ b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql @@ -133,6 +133,7 @@ select 'different types -- prohibited'; create table data_01756_str (key String) engine=Memory(); create table dist_01756_str as data_01756_str engine=Distributed(test_cluster_two_shards, currentDatabase(), data_01756_str, cityHash64(key)); select * from dist_01756_str where key in ('0', '2'); +select * from dist_01756_str where key in (0, 2); select * from dist_01756_str where key in ('0', Null); -- { serverError 507 } -- select * from dist_01756_str where key in (0, 2); -- { serverError 53 } -- select * from dist_01756_str where key in (0, Null); -- { serverError 53 } From e3ff630612958fe635f4ee4d58885247725acff1 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 12 May 2022 16:13:43 +0300 Subject: [PATCH 158/171] tests: add echo for 01756_optimize_skip_unused_shards_rewrite_in Signed-off-by: Azat Khuzhin --- ...ze_skip_unused_shards_rewrite_in.reference | 120 ++++++++++++++++++ ...optimize_skip_unused_shards_rewrite_in.sql | 18 ++- 2 files changed, 131 insertions(+), 7 deletions(-) diff --git a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference index 15e00db0231..836f09c20a8 100644 --- a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference +++ b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference @@ -1,31 +1,151 @@ +-- { echoOn } + +-- SELECT +-- intHash64(0) % 2, +-- intHash64(2) % 2 +-- ┌─modulo(intHash64(0), 2)─┬─modulo(intHash64(2), 2)─┐ +-- │ 0 │ 1 │ +-- └─────────────────────────┴─────────────────────────┘ +create table dist_01756 as system.one engine=Distributed(test_cluster_two_shards, system, one, intHash64(dummy)); +-- +-- w/o optimize_skip_unused_shards_rewrite_in=1 +-- +select '(0, 2)'; (0, 2) +with (select currentDatabase()) as id_no select *, ignore(id_no) from dist_01756 where dummy in (0, 2); 0 0 0 0 +system flush logs; +select query from system.query_log where + event_date >= yesterday() and + event_time > now() - interval 1 hour and + not is_initial_query and + query not like '%system%query_log%' and + query like concat('WITH%', currentDatabase(), '%AS `id_no` %') and + type = 'QueryFinish' +order by query; WITH _CAST(\'default\', \'Nullable(String)\') AS `id_no` SELECT `one`.`dummy`, ignore(`id_no`) FROM `system`.`one` WHERE `dummy` IN (0, 2) WITH _CAST(\'default\', \'Nullable(String)\') AS `id_no` SELECT `one`.`dummy`, ignore(`id_no`) FROM `system`.`one` WHERE `dummy` IN (0, 2) +-- +-- w/ optimize_skip_unused_shards_rewrite_in=1 +-- + +set optimize_skip_unused_shards_rewrite_in=1; +-- detailed coverage for realistic examples +select 'optimize_skip_unused_shards_rewrite_in(0, 2)'; optimize_skip_unused_shards_rewrite_in(0, 2) +with (select currentDatabase()) as id_02 select *, ignore(id_02) from dist_01756 where dummy in (0, 2); 0 0 +system flush logs; +select query from system.query_log where + event_date >= yesterday() and + event_time > now() - interval 1 hour and + not is_initial_query and + query not like '%system%query_log%' and + query like concat('WITH%', currentDatabase(), '%AS `id_02` %') and + type = 'QueryFinish' +order by query; WITH _CAST(\'default\', \'Nullable(String)\') AS `id_02` SELECT `one`.`dummy`, ignore(`id_02`) FROM `system`.`one` WHERE `dummy` IN tuple(0) WITH _CAST(\'default\', \'Nullable(String)\') AS `id_02` SELECT `one`.`dummy`, ignore(`id_02`) FROM `system`.`one` WHERE `dummy` IN tuple(2) +select 'optimize_skip_unused_shards_rewrite_in(2,)'; optimize_skip_unused_shards_rewrite_in(2,) +with (select currentDatabase()) as id_2 select *, ignore(id_2) from dist_01756 where dummy in (2,); +system flush logs; +select query from system.query_log where + event_date >= yesterday() and + event_time > now() - interval 1 hour and + not is_initial_query and + query not like '%system%query_log%' and + query like concat('WITH%', currentDatabase(), '%AS `id_2` %') and + type = 'QueryFinish' +order by query; WITH _CAST(\'default\', \'Nullable(String)\') AS `id_2` SELECT `one`.`dummy`, ignore(`id_2`) FROM `system`.`one` WHERE `dummy` IN tuple(2) +select 'optimize_skip_unused_shards_rewrite_in(0,)'; optimize_skip_unused_shards_rewrite_in(0,) +with (select currentDatabase()) as id_0 select *, ignore(id_0) from dist_01756 where dummy in (0,); 0 0 +system flush logs; +select query from system.query_log where + event_date >= yesterday() and + event_time > now() - interval 1 hour and + not is_initial_query and + query not like '%system%query_log%' and + query like concat('WITH%', currentDatabase(), '%AS `id_0` %') and + type = 'QueryFinish' +order by query; WITH _CAST(\'default\', \'Nullable(String)\') AS `id_0` SELECT `one`.`dummy`, ignore(`id_0`) FROM `system`.`one` WHERE `dummy` IN tuple(0) +-- signed column +select 'signed column'; signed column +create table data_01756_signed (key Int) engine=Null; +with (select currentDatabase()) as key_signed select *, ignore(key_signed) from cluster(test_cluster_two_shards, currentDatabase(), data_01756_signed, key) where key in (-1, -2); +system flush logs; +select query from system.query_log where + event_date >= yesterday() and + event_time > now() - interval 1 hour and + not is_initial_query and + query not like '%system%query_log%' and + query like concat('WITH%', currentDatabase(), '%AS `key_signed` %') and + type = 'QueryFinish' +order by query; WITH _CAST(\'default\', \'Nullable(String)\') AS `key_signed` SELECT `key`, ignore(`key_signed`) FROM `default`.`data_01756_signed` WHERE `key` IN tuple(-1) WITH _CAST(\'default\', \'Nullable(String)\') AS `key_signed` SELECT `key`, ignore(`key_signed`) FROM `default`.`data_01756_signed` WHERE `key` IN tuple(-2) +-- not tuple +select * from dist_01756 where dummy in (0); 0 +select * from dist_01756 where dummy in ('0'); 0 +-- +-- errors +-- +select 'errors'; errors +-- optimize_skip_unused_shards does not support non-constants +select * from dist_01756 where dummy in (select * from system.one); -- { serverError 507 } +select * from dist_01756 where dummy in (toUInt8(0)); -- { serverError 507 } +-- NOT IN does not supported +select * from dist_01756 where dummy not in (0, 2); -- { serverError 507 } +-- +-- others +-- +select 'others'; others +select * from dist_01756 where dummy not in (2, 3) and dummy in (0, 2); 0 +select * from dist_01756 where dummy in tuple(0, 2); 0 +select * from dist_01756 where dummy in tuple(0); 0 +select * from dist_01756 where dummy in tuple(2); +-- Identifier is NULL +select (2 IN (2,)), * from dist_01756 where dummy in (0, 2) format Null; +-- Literal is NULL +select (dummy IN (toUInt8(2),)), * from dist_01756 where dummy in (0, 2) format Null; +-- different type +select 'different types -- prohibited'; different types -- prohibited +create table data_01756_str (key String) engine=Memory(); +create table dist_01756_str as data_01756_str engine=Distributed(test_cluster_two_shards, currentDatabase(), data_01756_str, cityHash64(key)); +select * from dist_01756_str where key in ('0', '2'); +select * from dist_01756_str where key in (0, 2); +select * from dist_01756_str where key in ('0', Null); -- { serverError 507 } +-- select * from dist_01756_str where key in (0, 2); -- { serverError 53 } +-- select * from dist_01756_str where key in (0, Null); -- { serverError 53 } + +-- different type #2 +select 'different types -- conversion'; different types -- conversion +create table dist_01756_column as system.one engine=Distributed(test_cluster_two_shards, system, one, dummy); +select * from dist_01756_column where dummy in (0, '255'); 0 +select * from dist_01756_column where dummy in (0, '255foo'); -- { serverError 53 } +-- intHash64 does not accept string, but implicit conversion should be done +select * from dist_01756 where dummy in ('0', '2'); 0 +-- optimize_skip_unused_shards_limit +select 'optimize_skip_unused_shards_limit'; optimize_skip_unused_shards_limit +select * from dist_01756 where dummy in (0, 2) settings optimize_skip_unused_shards_limit=1; -- { serverError 507 } +select * from dist_01756 where dummy in (0, 2) settings optimize_skip_unused_shards_limit=1, force_optimize_skip_unused_shards=0; 0 0 diff --git a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql index a62af2d5667..ff88dd27707 100644 --- a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql +++ b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql @@ -11,6 +11,15 @@ drop table if exists dist_01756_column; drop table if exists data_01756_str; drop table if exists data_01756_signed; +-- separate log entry for localhost queries +set prefer_localhost_replica=0; +set force_optimize_skip_unused_shards=2; +set optimize_skip_unused_shards=1; +set optimize_skip_unused_shards_rewrite_in=0; +set log_queries=1; + +-- { echoOn } + -- SELECT -- intHash64(0) % 2, -- intHash64(2) % 2 @@ -19,13 +28,6 @@ drop table if exists data_01756_signed; -- └─────────────────────────┴─────────────────────────┘ create table dist_01756 as system.one engine=Distributed(test_cluster_two_shards, system, one, intHash64(dummy)); --- separate log entry for localhost queries -set prefer_localhost_replica=0; -set force_optimize_skip_unused_shards=2; -set optimize_skip_unused_shards=1; -set optimize_skip_unused_shards_rewrite_in=0; -set log_queries=1; - -- -- w/o optimize_skip_unused_shards_rewrite_in=1 -- @@ -151,6 +153,8 @@ select 'optimize_skip_unused_shards_limit'; select * from dist_01756 where dummy in (0, 2) settings optimize_skip_unused_shards_limit=1; -- { serverError 507 } select * from dist_01756 where dummy in (0, 2) settings optimize_skip_unused_shards_limit=1, force_optimize_skip_unused_shards=0; +-- { echoOff } + drop table dist_01756; drop table dist_01756_str; drop table dist_01756_column; From ef921d834d3ecbf0fceea252dbf0d058c539c626 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 12 May 2022 16:16:59 +0300 Subject: [PATCH 159/171] tests: improve 01756_optimize_skip_unused_shards_rewrite_in Signed-off-by: Azat Khuzhin --- ...56_optimize_skip_unused_shards_rewrite_in.reference | 10 ++++++++++ .../01756_optimize_skip_unused_shards_rewrite_in.sql | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference index 836f09c20a8..acf25728997 100644 --- a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference +++ b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference @@ -125,9 +125,19 @@ select (dummy IN (toUInt8(2),)), * from dist_01756 where dummy in (0, 2) format select 'different types -- prohibited'; different types -- prohibited create table data_01756_str (key String) engine=Memory(); +insert into data_01756_str values (0)(1); +-- SELECT +-- cityHash64(0) % 2, +-- cityHash64(2) % 2 +-- +-- ┌─modulo(cityHash64(0), 2)─┬─modulo(cityHash64(2), 2)─┐ +-- │ 0 │ 1 │ +-- └──────────────────────────┴──────────────────────────┘ create table dist_01756_str as data_01756_str engine=Distributed(test_cluster_two_shards, currentDatabase(), data_01756_str, cityHash64(key)); select * from dist_01756_str where key in ('0', '2'); +0 select * from dist_01756_str where key in (0, 2); +0 select * from dist_01756_str where key in ('0', Null); -- { serverError 507 } -- select * from dist_01756_str where key in (0, 2); -- { serverError 53 } -- select * from dist_01756_str where key in (0, Null); -- { serverError 53 } diff --git a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql index ff88dd27707..59709a7ee2a 100644 --- a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql +++ b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql @@ -133,6 +133,14 @@ select (dummy IN (toUInt8(2),)), * from dist_01756 where dummy in (0, 2) format -- different type select 'different types -- prohibited'; create table data_01756_str (key String) engine=Memory(); +insert into data_01756_str values (0)(1); +-- SELECT +-- cityHash64(0) % 2, +-- cityHash64(2) % 2 +-- +-- ┌─modulo(cityHash64(0), 2)─┬─modulo(cityHash64(2), 2)─┐ +-- │ 0 │ 1 │ +-- └──────────────────────────┴──────────────────────────┘ create table dist_01756_str as data_01756_str engine=Distributed(test_cluster_two_shards, currentDatabase(), data_01756_str, cityHash64(key)); select * from dist_01756_str where key in ('0', '2'); select * from dist_01756_str where key in (0, 2); From a04b6ad314a923b80b9c63a4e79c4837faa67cb5 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 3 May 2022 11:06:28 +0300 Subject: [PATCH 160/171] Mark Context::getInterserverCredentials() as const Signed-off-by: Azat Khuzhin --- src/Interpreters/Context.cpp | 2 +- src/Interpreters/Context.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 1bd3f92a5b7..803d0fa5932 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -2226,7 +2226,7 @@ bool Context::hasAuxiliaryZooKeeper(const String & name) const return getConfigRef().has("auxiliary_zookeepers." + name); } -InterserverCredentialsPtr Context::getInterserverCredentials() +InterserverCredentialsPtr Context::getInterserverCredentials() const { return shared->interserver_io_credentials.get(); } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index dbddda39aad..510f0372499 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -619,7 +619,7 @@ public: /// Credentials which server will use to communicate with others void updateInterserverCredentials(const Poco::Util::AbstractConfiguration & config); - InterserverCredentialsPtr getInterserverCredentials(); + InterserverCredentialsPtr getInterserverCredentials() const; /// Interserver requests scheme (http or https) void setInterserverScheme(const String & scheme); From a0ba113e5ec09d5e74f1683f26a847ca5fb6cc65 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 3 May 2022 11:20:50 +0300 Subject: [PATCH 161/171] Mark Context::getInterserverIOHandler() as const Signed-off-by: Azat Khuzhin --- src/Interpreters/Context.cpp | 1 + src/Interpreters/Context.h | 1 + src/Interpreters/InterserverIOHandler.h | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 803d0fa5932..34f396b978c 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -535,6 +535,7 @@ ContextMutablePtr Context::createCopy(const ContextMutablePtr & other) Context::~Context() = default; InterserverIOHandler & Context::getInterserverIOHandler() { return shared->interserver_io_handler; } +const InterserverIOHandler & Context::getInterserverIOHandler() const { return shared->interserver_io_handler; } std::unique_lock Context::getLock() const { diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 510f0372499..ddc474ca347 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -612,6 +612,7 @@ public: OutputFormatPtr getOutputFormatParallelIfPossible(const String & name, WriteBuffer & buf, const Block & sample) const; InterserverIOHandler & getInterserverIOHandler(); + const InterserverIOHandler & getInterserverIOHandler() const; /// How other servers can access this for downloading replicated data. void setInterserverIOAddress(const String & host, UInt16 port); diff --git a/src/Interpreters/InterserverIOHandler.h b/src/Interpreters/InterserverIOHandler.h index 7ad1c01cbf1..69b742db2ec 100644 --- a/src/Interpreters/InterserverIOHandler.h +++ b/src/Interpreters/InterserverIOHandler.h @@ -69,7 +69,7 @@ public: return endpoint_map.erase(name); } - InterserverIOEndpointPtr getEndpoint(const String & name) + InterserverIOEndpointPtr getEndpoint(const String & name) const try { std::lock_guard lock(mutex); @@ -84,7 +84,7 @@ private: using EndpointMap = std::map; EndpointMap endpoint_map; - std::mutex mutex; + mutable std::mutex mutex; }; } From 5086d8de29a80cb31603b949829ac2b7a211b566 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 12 May 2022 17:01:18 +0300 Subject: [PATCH 162/171] Reload listen_reuse_port/listen_backlog from config Signed-off-by: Azat Khuzhin --- programs/server/Server.cpp | 35 ++++++++++++++++++++--------------- programs/server/Server.h | 7 ++++++- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 2b1a0809143..b0931f678f7 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -334,7 +334,12 @@ Poco::Net::SocketAddress makeSocketAddress(const std::string & host, UInt16 port return socket_address; } -Poco::Net::SocketAddress Server::socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure) const +Poco::Net::SocketAddress Server::socketBindListen( + const Poco::Util::AbstractConfiguration & config, + Poco::Net::ServerSocket & socket, + const std::string & host, + UInt16 port, + [[maybe_unused]] bool secure) const { auto address = makeSocketAddress(host, port, &logger()); #if !defined(POCO_CLICKHOUSE_PATCH) || POCO_VERSION < 0x01090100 @@ -347,7 +352,7 @@ Poco::Net::SocketAddress Server::socketBindListen(Poco::Net::ServerSocket & sock #if POCO_VERSION < 0x01080000 socket.bind(address, /* reuseAddress = */ true); #else - socket.bind(address, /* reuseAddress = */ true, /* reusePort = */ config().getBool("listen_reuse_port", false)); + socket.bind(address, /* reuseAddress = */ true, /* reusePort = */ config.getBool("listen_reuse_port", false)); #endif /// If caller requests any available port from the OS, discover it after binding. @@ -357,7 +362,7 @@ Poco::Net::SocketAddress Server::socketBindListen(Poco::Net::ServerSocket & sock LOG_DEBUG(&logger(), "Requested any available port (port == 0), actual port is {:d}", address.port()); } - socket.listen(/* backlog = */ config().getUInt("listen_backlog", 4096)); + socket.listen(/* backlog = */ config.getUInt("listen_backlog", 4096)); return address; } @@ -1237,7 +1242,7 @@ int Server::main(const std::vector & /*args*/) [&](UInt16 port) -> ProtocolServerAdapter { Poco::Net::ServerSocket socket; - auto address = socketBindListen(socket, listen_host, port); + auto address = socketBindListen(config(), socket, listen_host, port); socket.setReceiveTimeout(config().getUInt64("keeper_server.socket_receive_timeout_sec", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC)); socket.setSendTimeout(config().getUInt64("keeper_server.socket_send_timeout_sec", DBMS_DEFAULT_SEND_TIMEOUT_SEC)); return ProtocolServerAdapter( @@ -1260,7 +1265,7 @@ int Server::main(const std::vector & /*args*/) { #if USE_SSL Poco::Net::SecureServerSocket socket; - auto address = socketBindListen(socket, listen_host, port, /* secure = */ true); + auto address = socketBindListen(config(), socket, listen_host, port, /* secure = */ true); socket.setReceiveTimeout(config().getUInt64("keeper_server.socket_receive_timeout_sec", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC)); socket.setSendTimeout(config().getUInt64("keeper_server.socket_send_timeout_sec", DBMS_DEFAULT_SEND_TIMEOUT_SEC)); return ProtocolServerAdapter( @@ -1797,7 +1802,7 @@ void Server::createServers( createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter { Poco::Net::ServerSocket socket; - auto address = socketBindListen(socket, listen_host, port); + auto address = socketBindListen(config, socket, listen_host, port); socket.setReceiveTimeout(settings.http_receive_timeout); socket.setSendTimeout(settings.http_send_timeout); @@ -1815,7 +1820,7 @@ void Server::createServers( { #if USE_SSL Poco::Net::SecureServerSocket socket; - auto address = socketBindListen(socket, listen_host, port, /* secure = */ true); + auto address = socketBindListen(config, socket, listen_host, port, /* secure = */ true); socket.setReceiveTimeout(settings.http_receive_timeout); socket.setSendTimeout(settings.http_send_timeout); return ProtocolServerAdapter( @@ -1836,7 +1841,7 @@ void Server::createServers( createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter { Poco::Net::ServerSocket socket; - auto address = socketBindListen(socket, listen_host, port); + auto address = socketBindListen(config, socket, listen_host, port); socket.setReceiveTimeout(settings.receive_timeout); socket.setSendTimeout(settings.send_timeout); return ProtocolServerAdapter( @@ -1855,7 +1860,7 @@ void Server::createServers( createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter { Poco::Net::ServerSocket socket; - auto address = socketBindListen(socket, listen_host, port); + auto address = socketBindListen(config, socket, listen_host, port); socket.setReceiveTimeout(settings.receive_timeout); socket.setSendTimeout(settings.send_timeout); return ProtocolServerAdapter( @@ -1875,7 +1880,7 @@ void Server::createServers( { #if USE_SSL Poco::Net::SecureServerSocket socket; - auto address = socketBindListen(socket, listen_host, port, /* secure = */ true); + auto address = socketBindListen(config, socket, listen_host, port, /* secure = */ true); socket.setReceiveTimeout(settings.receive_timeout); socket.setSendTimeout(settings.send_timeout); return ProtocolServerAdapter( @@ -1899,7 +1904,7 @@ void Server::createServers( createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter { Poco::Net::ServerSocket socket; - auto address = socketBindListen(socket, listen_host, port); + auto address = socketBindListen(config, socket, listen_host, port); socket.setReceiveTimeout(settings.http_receive_timeout); socket.setSendTimeout(settings.http_send_timeout); return ProtocolServerAdapter( @@ -1919,7 +1924,7 @@ void Server::createServers( { #if USE_SSL Poco::Net::SecureServerSocket socket; - auto address = socketBindListen(socket, listen_host, port, /* secure = */ true); + auto address = socketBindListen(config, socket, listen_host, port, /* secure = */ true); socket.setReceiveTimeout(settings.http_receive_timeout); socket.setSendTimeout(settings.http_send_timeout); return ProtocolServerAdapter( @@ -1943,7 +1948,7 @@ void Server::createServers( createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter { Poco::Net::ServerSocket socket; - auto address = socketBindListen(socket, listen_host, port, /* secure = */ true); + auto address = socketBindListen(config, socket, listen_host, port, /* secure = */ true); socket.setReceiveTimeout(Poco::Timespan()); socket.setSendTimeout(settings.send_timeout); return ProtocolServerAdapter( @@ -1957,7 +1962,7 @@ void Server::createServers( createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter { Poco::Net::ServerSocket socket; - auto address = socketBindListen(socket, listen_host, port, /* secure = */ true); + auto address = socketBindListen(config, socket, listen_host, port, /* secure = */ true); socket.setReceiveTimeout(Poco::Timespan()); socket.setSendTimeout(settings.send_timeout); return ProtocolServerAdapter( @@ -1985,7 +1990,7 @@ void Server::createServers( createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter { Poco::Net::ServerSocket socket; - auto address = socketBindListen(socket, listen_host, port); + auto address = socketBindListen(config, socket, listen_host, port); socket.setReceiveTimeout(settings.http_receive_timeout); socket.setSendTimeout(settings.http_send_timeout); return ProtocolServerAdapter( diff --git a/programs/server/Server.h b/programs/server/Server.h index 9a0fabd97c2..4235fcc2d3b 100644 --- a/programs/server/Server.h +++ b/programs/server/Server.h @@ -67,7 +67,12 @@ protected: private: ContextMutablePtr global_context; - Poco::Net::SocketAddress socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure = false) const; + Poco::Net::SocketAddress socketBindListen( + const Poco::Util::AbstractConfiguration & config, + Poco::Net::ServerSocket & socket, + const std::string & host, + UInt16 port, + [[maybe_unused]] bool secure = false) const; using CreateServerFunc = std::function; void createServer( From 1dc3d048377b1f9c8493dde0eadebcd5ea3e0b90 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Thu, 12 May 2022 16:20:19 +0200 Subject: [PATCH 163/171] Removed __SSE2__ flag for PPC64 --- cmake/cpu_features.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index 36c91d95be9..9551e05d4f8 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -61,7 +61,7 @@ else () endif () if (ARCH_PPC64LE) - set (COMPILER_FLAGS "${COMPILER_FLAGS} -maltivec -mcpu=power8 -D__SSE2__=1 -DNO_WARN_X86_INTRINSICS") + set (COMPILER_FLAGS "${COMPILER_FLAGS} -maltivec -mcpu=power8 -DNO_WARN_X86_INTRINSICS") endif () set (TEST_FLAG "-msse4.2") From 77fb1b84acb88f6b211a44c605773443758e5f18 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 12 May 2022 15:21:29 +0000 Subject: [PATCH 164/171] Enable DNS cache for HTTPSClientSession. --- contrib/poco | 2 +- src/IO/HTTPCommon.cpp | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/contrib/poco b/contrib/poco index 6c1a233744d..5d11f0aa6fd 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit 6c1a233744d13414e8e8db396c75177b857b2c22 +Subproject commit 5d11f0aa6fd2faad0a7b34aa355cd50c4deb27e6 diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index 9f53572fcde..b319389925a 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -75,7 +75,13 @@ namespace { #if USE_SSL /// Cannot resolve host in advance, otherwise SNI won't work in Poco. - session = std::make_shared(host, port); + + String resolved_host = resolve_host ? DNSResolver::instance().resolveHost(host).toString() : host; + auto https_session = std::make_shared(host, port); + if (resolve_host) + https_session->setResolvedHost(DNSResolver::instance().resolveHost(host).toString()); + + session = std::move(https_session); #else throw Exception("ClickHouse was built without HTTPS support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); #endif From f8bd4fa0f06ac6a65c919d9a26497cf6df176c51 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 12 May 2022 15:24:36 +0000 Subject: [PATCH 165/171] Remove old comment. --- src/IO/HTTPCommon.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index b319389925a..7ed2f343209 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -74,8 +74,6 @@ namespace if (https) { #if USE_SSL - /// Cannot resolve host in advance, otherwise SNI won't work in Poco. - String resolved_host = resolve_host ? DNSResolver::instance().resolveHost(host).toString() : host; auto https_session = std::make_shared(host, port); if (resolve_host) From f35c893fbeea031832d8c55eeaf16ac537d44e6b Mon Sep 17 00:00:00 2001 From: Dan Roscigno Date: Thu, 12 May 2022 14:32:13 -0400 Subject: [PATCH 166/171] move UK and Taxi datasets up --- docs/en/getting-started/example-datasets/nyc-taxi.md | 1 + docs/en/getting-started/example-datasets/uk-price-paid.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/en/getting-started/example-datasets/nyc-taxi.md b/docs/en/getting-started/example-datasets/nyc-taxi.md index 270aeb4929c..360f9eed1c8 100644 --- a/docs/en/getting-started/example-datasets/nyc-taxi.md +++ b/docs/en/getting-started/example-datasets/nyc-taxi.md @@ -1,5 +1,6 @@ --- sidebar_label: New York Taxi Data +sidebar_position: 2 description: Data for billions of taxi and for-hire vehicle (Uber, Lyft, etc.) trips originating in New York City since 2009 --- diff --git a/docs/en/getting-started/example-datasets/uk-price-paid.md b/docs/en/getting-started/example-datasets/uk-price-paid.md index e0f20639aea..eaec6e53ed4 100644 --- a/docs/en/getting-started/example-datasets/uk-price-paid.md +++ b/docs/en/getting-started/example-datasets/uk-price-paid.md @@ -1,5 +1,6 @@ --- sidebar_label: UK Property Price Paid +sidebar_position: 1 --- # UK Property Price Paid From 8e5a726c875671ae774bcd7050c3785fe6bdc73c Mon Sep 17 00:00:00 2001 From: Dan Roscigno Date: Thu, 12 May 2022 17:39:29 -0400 Subject: [PATCH 167/171] Empty-Commit From 8b3fb22c6d4c12ffa4a488cbd35c8fcf9a5f11b9 Mon Sep 17 00:00:00 2001 From: qieqieplus Date: Fri, 13 May 2022 15:06:36 +0800 Subject: [PATCH 168/171] check array sizes for short cut --- src/Functions/array/arrayDistance.cpp | 83 ++++++++++++++------------- src/Functions/array/arrayNorm.cpp | 34 ++++++----- 2 files changed, 62 insertions(+), 55 deletions(-) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index 94c20568668..a533cb2c0cc 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -1,12 +1,13 @@ #include +#include #include #include +#include #include -#include #include #include -#include "Columns/IColumn.h" -#include "DataTypes/Serializations/ISerialization.h" + +#include namespace DB { @@ -92,7 +93,8 @@ public: case TypeIndex::Float64: return std::make_shared(); default: - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Arguments of function {} has nested type {}. " "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", getName(), common_type->getName()); @@ -121,7 +123,7 @@ public: executeWithType(*arr_x, *arr_y, type_x, type_y, result); break; default: - throw Exception("Unexpected result type.", ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected result type."); } return result; } @@ -141,7 +143,8 @@ private: if (mx.rows() && my.rows() && mx.rows() != my.rows()) { - throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, + throw Exception( + ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, "Arguments of function {} have different array sizes: {} and {}", getName(), mx.rows(), my.rows()); } @@ -152,40 +155,56 @@ private: template void columnToMatrix(const ColumnArray & array, const DataTypePtr & nested_type, Eigen::MatrixX & mat) const { + const auto & offsets = array.getOffsets(); + size_t cols = offsets.size(); + size_t rows = cols > 0 ? offsets.front() : 0; + + ColumnArray::Offset prev = 0; + for (ColumnArray::Offset off : offsets) + { + if (off - prev != rows) + throw Exception( + ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, + "Arrays in a column passed to function {} have different sizes: {} and {}", + getName(), rows, off - prev); + prev = off; + } + switch (nested_type->getTypeId()) { case TypeIndex::UInt8: - fillMatrix(mat, array); + fillMatrix(mat, array, rows, cols); break; case TypeIndex::UInt16: - fillMatrix(mat, array); + fillMatrix(mat, array, rows, cols); break; case TypeIndex::UInt32: - fillMatrix(mat, array); + fillMatrix(mat, array, rows, cols); break; case TypeIndex::UInt64: - fillMatrix(mat, array); + fillMatrix(mat, array, rows, cols); break; case TypeIndex::Int8: - fillMatrix(mat, array); + fillMatrix(mat, array, rows, cols); break; case TypeIndex::Int16: - fillMatrix(mat, array); + fillMatrix(mat, array, rows, cols); break; case TypeIndex::Int32: - fillMatrix(mat, array); + fillMatrix(mat, array, rows, cols); break; case TypeIndex::Int64: - fillMatrix(mat, array); + fillMatrix(mat, array, rows, cols); break; case TypeIndex::Float32: - fillMatrix(mat, array); + fillMatrix(mat, array, rows, cols); break; case TypeIndex::Float64: - fillMatrix(mat, array); + fillMatrix(mat, array, rows, cols); break; default: - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Arguments of function {} has nested type {}. " "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", getName(), nested_type->getName()); @@ -194,41 +213,25 @@ private: // optimize for float/ double template - requires (std::is_same_v) - void fillMatrix(Eigen::MatrixX & mat, const ColumnArray & array) const + requires std::is_same_v + void fillMatrix(Eigen::MatrixX & mat, const ColumnArray & array, size_t rows, size_t cols) const { const auto & data = typeid_cast &>(array.getData()).getData(); - const auto & offsets = array.getOffsets(); - size_t cols = offsets.size(); - size_t rows = cols > 0 ? offsets.front() : 0; - mat = Eigen::Map>(data.data(), rows, cols); } template - void fillMatrix(Eigen::MatrixX & mat, const ColumnArray & array) const + void fillMatrix(Eigen::MatrixX & mat, const ColumnArray & array, size_t rows, size_t cols) const { const auto & data = typeid_cast &>(array.getData()).getData(); - const auto & offsets = array.getOffsets(); - size_t cols = offsets.size(); - size_t rows = cols > 0 ? offsets.front() : 0; - mat.resize(rows, cols); - - ColumnArray::Offset prev = 0, col = 0; - for (ColumnArray::Offset off : offsets) + for (size_t col = 0; col < cols; ++col) { - if (off - prev != rows) - throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, - "Arrays in a column passed to function {} have different sizes: {} and {}", - getName(), rows, off - prev); - - for (ColumnArray::Offset row = 0; row < off - prev; ++row) + for (size_t row = 0; row < rows; ++row) { - mat(row, col) = static_cast(data[prev + row]); + size_t off = col * rows; + mat(row, col) = static_cast(data[off + row]); } - ++col; - prev = off; } } }; diff --git a/src/Functions/array/arrayNorm.cpp b/src/Functions/array/arrayNorm.cpp index 61da1c83147..20fe85d7491 100644 --- a/src/Functions/array/arrayNorm.cpp +++ b/src/Functions/array/arrayNorm.cpp @@ -1,12 +1,13 @@ #include +#include #include #include +#include #include -#include #include #include -#include "Columns/IColumn.h" -#include "DataTypes/Serializations/ISerialization.h" + +#include namespace DB { @@ -54,7 +55,7 @@ public: { const auto * array_type = checkAndGetDataType(argument.type.get()); if (!array_type) - throw Exception("Argument of function " + getName() + " must be array. ", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Argument of function {} must be array.", getName()); types.push_back(array_type->getNestedType()); } @@ -75,9 +76,10 @@ public: return std::make_shared(); default: throw Exception( - "Arguments of function " + getName() + " has nested type " + common_type->getName() - + ". Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Arguments of function {} has nested type {}. " + "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", + getName(), common_type->getName()); } } @@ -98,7 +100,7 @@ public: executeWithType(*arr, type, result); break; default: - throw Exception("Unexpected result type.", ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected result type."); } return result; } @@ -107,10 +109,10 @@ private: template void executeWithType(const ColumnArray & array, const DataTypePtr & type, MutableColumnPtr & column) const { - std::vector> v; - columnToVectors(array, type, v); + std::vector> vec; + columnToVectors(array, type, vec); auto & data = assert_cast &>(*column).getData(); - Kernel::compute(v, data); + Kernel::compute(vec, data); } template @@ -150,13 +152,15 @@ private: break; default: throw Exception( - "Arguments of function " + getName() + " has nested type " + nested_type->getName() - + ". Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Arguments of function {} has nested type {}. " + "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", + getName(), nested_type->getName()); } } + template - requires (std::is_same_v) + requires std::is_same_v void fillVectors(std::vector> & vec, const ColumnArray & array) const { const auto & data = typeid_cast &>(array.getData()).getData(); From 07aaa922996131d84fc5eca01a77cba246c0a588 Mon Sep 17 00:00:00 2001 From: Vxider Date: Fri, 13 May 2022 09:25:50 +0000 Subject: [PATCH 169/171] fix windowview watch --- src/Storages/WindowView/WindowViewSource.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Storages/WindowView/WindowViewSource.h b/src/Storages/WindowView/WindowViewSource.h index a726cdc8712..7b914933035 100644 --- a/src/Storages/WindowView/WindowViewSource.h +++ b/src/Storages/WindowView/WindowViewSource.h @@ -51,6 +51,8 @@ protected: Block block; UInt32 watermark; std::tie(block, watermark) = generateImpl(); + if (!block) + return Chunk(); if (is_events) { return Chunk( From ef7e21ea46bd4bdfa6d18b4a9f90ed7f86465371 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 13 May 2022 15:20:04 +0200 Subject: [PATCH 170/171] Function h3kRing added cast --- src/Functions/h3kRing.cpp | 8 +++++--- tests/queries/0_stateless/01042_h3_k_ring.reference | 2 ++ tests/queries/0_stateless/01042_h3_k_ring.sql | 3 +++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Functions/h3kRing.cpp b/src/Functions/h3kRing.cpp index a801fd299d6..a68f2a5e23d 100644 --- a/src/Functions/h3kRing.cpp +++ b/src/Functions/h3kRing.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -51,10 +52,10 @@ public: arg->getName(), 1, getName()); arg = arguments[1].get(); - if (!WhichDataType(arg).isUInt16()) + if (!WhichDataType(arg).isNativeUInt()) throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type {} of argument {} of function {}. Must be UInt16", + "Illegal type {} of argument {} of function {}. Must be unsigned native integer.", arg->getName(), 2, getName()); @@ -80,7 +81,8 @@ public: const auto & data_hindex = col_hindex->getData(); /// ColumnUInt16 is sufficient as the max value of 2nd arg is checked (arg > 0 < 10000) in implementation below - const auto * col_k = checkAndGetColumn(non_const_arguments[1].column.get()); + auto cast_result = castColumnAccurate(non_const_arguments[1], std::make_shared()); + const auto * col_k = checkAndGetColumn(cast_result.get()); if (!col_k) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/tests/queries/0_stateless/01042_h3_k_ring.reference b/tests/queries/0_stateless/01042_h3_k_ring.reference index 24f4b9885ce..770468da350 100644 --- a/tests/queries/0_stateless/01042_h3_k_ring.reference +++ b/tests/queries/0_stateless/01042_h3_k_ring.reference @@ -1,5 +1,7 @@ [581250224954015743,581259021047037951,581267817140060159,581276613233082367,581500913605148671,581518505791193087,581764796395814911] [581276613233082367] +[581250224954015743,581259021047037951,581267817140060159,581276613233082367,581500913605148671,581518505791193087,581764796395814911] +[581276613233082367] [578466261512486911,578712552117108735,578888473977552895,579205133326352383,579275502070530047,579662530163507199,579768083279773695] [580995138256371711,581144671837749247,581162264023793663,581166662070304767,581171060116815871,581250224954015743,581254623000526847,581259021047037951,581263419093549055,581267817140060159,581272215186571263,581276613233082367,581531699930726399,581536097977237503,581549292116770815,581553690163281919,581558088209793023,581747204209770495,581764796395814911] [589624655266971647,589625205022785535,589626854290227199,589627404046041087,589642797208829951,589644996232085503,589708218150682623,589708767906496511,589709317662310399,589709867418124287,589710417173938175,589710966929752063,589711516685565951,589714815220449279,589715914732077055,589725810336727039,589726909848354815,589727459604168703,589728009359982591,589729108871610367,589734606429749247,589735156185563135,589735705941377023,589736255697190911,589736805453004799,589737355208818687,589737904964632575,589742303011143679,589744502034399231,589745051790213119,589752198615793663,589752748371607551,589753298127421439,589753847883235327,589754397639049215,589754947394863103,589755497150676991] diff --git a/tests/queries/0_stateless/01042_h3_k_ring.sql b/tests/queries/0_stateless/01042_h3_k_ring.sql index d450954ab7a..8931efc44c2 100644 --- a/tests/queries/0_stateless/01042_h3_k_ring.sql +++ b/tests/queries/0_stateless/01042_h3_k_ring.sql @@ -5,6 +5,9 @@ SELECT h3kRing(581276613233082367, toUInt16(0)); SELECT h3kRing(581276613233082367, -1); -- { serverError 43 } SELECT h3kRing(581276613233082367, toUInt16(-1)); -- { serverError 12 } +SELECT arraySort(h3kRing(581276613233082367, 1)); +SELECT h3kRing(581276613233082367, 0); +SELECT h3kRing(581276613233082367, -1); -- { serverError 43 } DROP TABLE IF EXISTS h3_indexes; From 16e4b0db2029181f16b503a569b366cdb3e46a60 Mon Sep 17 00:00:00 2001 From: bkuschel Date: Fri, 13 May 2022 10:22:23 -0700 Subject: [PATCH 171/171] Fix coverity build --- tests/ci/build_check.py | 3 +++ tests/ci/ci_config.py | 1 + 2 files changed, 4 insertions(+) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 6a3615fc5db..b73bf057393 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -254,6 +254,9 @@ def main(): logging.info("Got version from repo %s", version.string) official_flag = pr_info.number == 0 + if "official" in build_config: + official_flag = build_config["official"] + version_type = "testing" if "release" in pr_info.labels or "release-lts" in pr_info.labels: version_type = "stable" diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index c9915c1c7f4..49ee3563722 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -37,6 +37,7 @@ CI_CONFIG = { "splitted": "unsplitted", "tidy": "disable", "with_coverage": False, + "official": False, }, # FIXME update to gcc-12 and turn on # "binary_gcc": {