mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-25 17:12:03 +00:00
Merge
This commit is contained in:
Evgeniy Gatov
commit
be0071c072
@ -1,366 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
test_table="hits_100m"
|
||||
|
||||
start_date="'2013-07-01'"
|
||||
early_stop_date="'2013-07-02'"
|
||||
stop_date="'2013-07-31'"
|
||||
counter_id=34
|
||||
|
||||
function run_ck_server
|
||||
{
|
||||
sudo sh -c " ulimit -v 54000000; /etc/init.d/clickhouse-server restart"
|
||||
}
|
||||
|
||||
# execute queries
|
||||
function execute()
|
||||
{
|
||||
queries=("${@}")
|
||||
queries_count=${#queries[@]}
|
||||
|
||||
if [ -z $TIMES ]; then
|
||||
TIMES=1
|
||||
fi
|
||||
|
||||
index=0
|
||||
comment_re='\#.*'
|
||||
while [ "$index" -lt "$queries_count" ]; do
|
||||
query=${queries[$index]}
|
||||
|
||||
if [[ $query =~ $comment_re ]]; then
|
||||
echo "$query"
|
||||
echo
|
||||
else
|
||||
sync
|
||||
sudo sh -c "echo 3 > /proc/sys/vm/drop_caches"
|
||||
|
||||
for i in $(seq $TIMES)
|
||||
do
|
||||
|
||||
expect -f ./expect.tcl "$query"
|
||||
if [ "$?" != "0" ]; then
|
||||
echo "Error: $?"
|
||||
#break
|
||||
fi
|
||||
|
||||
# restart clickhouse if failed
|
||||
ps aux | grep -P '\d+ clickhouse-server'
|
||||
if [ "$?" != "0" ]; then
|
||||
run_ck_server
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
let "index = $index + 1"
|
||||
echo "Ran $index queries." >&2
|
||||
done
|
||||
}
|
||||
|
||||
init_queries=(
|
||||
# DB structure with array arguments
|
||||
#"CREATE TABLE $test_table ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, Refresh UInt8, RefererCategoryID UInt16, RefererRegionID UInt32, URLCategoryID UInt16, URLRegionID UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, OriginalURL String, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), LocalEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming UInt32, DNSTiming UInt32, ConnectTiming UInt32, ResponseStartTiming UInt32, ResponseEndTiming UInt32, FetchTiming UInt32, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32 ) ENGINE = MergeTree(EventDate, intHash32(UserID), tuple(CounterID, EventDate, intHash32(UserID), EventTime), 8192);"
|
||||
|
||||
#DB structure without array arguments
|
||||
#"CREATE TABLE $test_table ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, Refresh UInt8, RefererCategoryID UInt16, RefererRegionID UInt32, URLCategoryID UInt16, URLRegionID UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, OriginalURL String, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), LocalEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, RemoteIP UInt32, WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming UInt32, DNSTiming UInt32, ConnectTiming UInt32, ResponseStartTiming UInt32, ResponseEndTiming UInt32, FetchTiming UInt32, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32 ) ENGINE = MergeTree(EventDate, intHash32(UserID), tuple(CounterID, EventDate, intHash32(UserID), EventTime), 8192);"
|
||||
|
||||
#modified table without uint
|
||||
"CREATE TABLE $test_table ( WatchID Int64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, RegionID UInt32, UserID Int64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, Refresh UInt8, RefererCategoryID UInt16, RefererRegionID UInt32, URLCategoryID UInt16, URLRegionID UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID Int64, OriginalURL String, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), LocalEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, RemoteIP UInt32, WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming UInt32, DNSTiming UInt32, ConnectTiming UInt32, ResponseStartTiming UInt32, ResponseEndTiming UInt32, FetchTiming UInt32, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash Int64, URLHash Int64, CLID UInt32, UserIDHash UInt64 ) ENGINE = MergeTree(EventDate, intHash32(UserID), tuple(CounterID, EventDate, intHash32(UserID), EventTime), 8192);"
|
||||
|
||||
)
|
||||
|
||||
test_queries=(
|
||||
"SELECT count() FROM $test_table;"
|
||||
"SELECT count() FROM $test_table WHERE AdvEngineID != 0;"
|
||||
"SELECT sum(AdvEngineID), count(), avg(ResolutionWidth) FROM $test_table;"
|
||||
"SELECT sum(UserID) FROM $test_table;"
|
||||
"SELECT uniq(UserID) FROM $test_table;"
|
||||
"SELECT uniq(SearchPhrase) FROM $test_table;"
|
||||
"SELECT min(EventDate), max(EventDate) FROM $test_table;"
|
||||
|
||||
"SELECT AdvEngineID, count() FROM $test_table WHERE AdvEngineID != 0 GROUP BY AdvEngineID ORDER BY count() DESC;"
|
||||
"#- мощная фильтрация. После фильтрации почти ничего не остаётся, но делаем ещё агрегацию.;"
|
||||
|
||||
"SELECT RegionID, uniq(UserID) AS u FROM $test_table GROUP BY RegionID ORDER BY u DESC LIMIT 10;"
|
||||
"#- агрегация, среднее количество ключей.;"
|
||||
|
||||
"SELECT RegionID, sum(AdvEngineID), count() AS c, avg(ResolutionWidth), uniq(UserID) FROM $test_table GROUP BY RegionID ORDER BY c DESC LIMIT 10;"
|
||||
"#- агрегация, среднее количество ключей, несколько агрегатных функций.;"
|
||||
|
||||
"SELECT MobilePhoneModel, uniq(UserID) AS u FROM $test_table WHERE MobilePhoneModel != '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;"
|
||||
"#- мощная фильтрация по строкам, затем агрегация по строкам.;"
|
||||
|
||||
"SELECT MobilePhone, MobilePhoneModel, uniq(UserID) AS u FROM $test_table WHERE MobilePhoneModel != '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;"
|
||||
"#- мощная фильтрация по строкам, затем агрегация по паре из числа и строки.;"
|
||||
|
||||
"SELECT SearchPhrase, count() AS c FROM $test_table WHERE SearchPhrase != '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;"
|
||||
"#- средняя фильтрация по строкам, затем агрегация по строкам, большое количество ключей.;"
|
||||
|
||||
"SELECT SearchPhrase, uniq(UserID) AS u FROM $test_table WHERE SearchPhrase != '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;"
|
||||
"#- агрегация чуть сложнее.;"
|
||||
|
||||
"SELECT SearchEngineID, SearchPhrase, count() AS c FROM $test_table WHERE SearchPhrase != '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;"
|
||||
"#- агрегация по числу и строке, большое количество ключей.;"
|
||||
|
||||
"SELECT UserID, count() FROM $test_table GROUP BY UserID ORDER BY count() DESC LIMIT 10;"
|
||||
"#- агрегация по очень большому количеству ключей, может не хватить оперативки.;"
|
||||
|
||||
"SELECT UserID, SearchPhrase, count() FROM $test_table GROUP BY UserID, SearchPhrase ORDER BY count() DESC LIMIT 10;"
|
||||
"#- ещё более сложная агрегация.;"
|
||||
|
||||
"SELECT UserID, SearchPhrase, count() FROM $test_table GROUP BY UserID, SearchPhrase LIMIT 10;"
|
||||
"#- то же самое, но без сортировки.;"
|
||||
|
||||
"SELECT UserID, toMinute(EventTime) AS m, SearchPhrase, count() FROM $test_table GROUP BY UserID, m, SearchPhrase ORDER BY count() DESC LIMIT 10;"
|
||||
"#- ещё более сложная агрегация, не стоит выполнять на больших таблицах.;"
|
||||
|
||||
"SELECT UserID FROM $test_table WHERE UserID = 12345678901234567890;"
|
||||
"#- мощная фильтрация по столбцу типа UInt64.;"
|
||||
|
||||
"SELECT count() FROM $test_table WHERE URL LIKE '%metrika%';"
|
||||
"#- фильтрация по поиску подстроки в строке.;"
|
||||
|
||||
"SELECT SearchPhrase, any(URL), count() AS c FROM $test_table WHERE URL LIKE '%metrika%' AND SearchPhrase != '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;"
|
||||
"#- вынимаем большие столбцы, фильтрация по строке.;"
|
||||
|
||||
"SELECT SearchPhrase, any(URL), any(Title), count() AS c, uniq(UserID) FROM $test_table WHERE Title LIKE '%Яндекс%' AND URL NOT LIKE '%.yandex.%' AND SearchPhrase != '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;"
|
||||
"#- чуть больше столбцы.;"
|
||||
|
||||
"SELECT * FROM $test_table WHERE URL LIKE '%metrika%' ORDER BY EventTime LIMIT 10;"
|
||||
"#- плохой запрос - вынимаем все столбцы.;"
|
||||
|
||||
"SELECT SearchPhrase FROM $test_table WHERE SearchPhrase != '' ORDER BY EventTime LIMIT 10;"
|
||||
"#- большая сортировка.;"
|
||||
|
||||
"SELECT SearchPhrase FROM $test_table WHERE SearchPhrase != '' ORDER BY SearchPhrase LIMIT 10;"
|
||||
"#- большая сортировка по строкам.;"
|
||||
|
||||
"SELECT SearchPhrase FROM $test_table WHERE SearchPhrase != '' ORDER BY EventTime, SearchPhrase LIMIT 10;"
|
||||
"#- большая сортировка по кортежу.;"
|
||||
|
||||
"SELECT CounterID, avg(length(URL)) AS l, count() AS c FROM $test_table WHERE URL != '' GROUP BY CounterID HAVING c > 100000 ORDER BY l DESC LIMIT 25;"
|
||||
"#- считаем средние длины URL для крупных счётчиков.;"
|
||||
|
||||
"SELECT domainWithoutWWW(Referer) AS key, avg(length(Referer)) AS l, count() AS c, any(Referer) FROM $test_table WHERE Referer != '' GROUP BY key HAVING c > 100000 ORDER BY l DESC LIMIT 25;"
|
||||
"#- то же самое, но с разбивкой по доменам.;"
|
||||
|
||||
"SELECT sum(ResolutionWidth), sum(ResolutionWidth + 1), sum(ResolutionWidth + 2), sum(ResolutionWidth + 3), sum(ResolutionWidth + 4), sum(ResolutionWidth + 5), sum(ResolutionWidth + 6), sum(ResolutionWidth + 7), sum(ResolutionWidth + 8), sum(ResolutionWidth + 9), sum(ResolutionWidth + 10), sum(ResolutionWidth + 11), sum(ResolutionWidth + 12), sum(ResolutionWidth + 13), sum(ResolutionWidth + 14), sum(ResolutionWidth + 15), sum(ResolutionWidth + 16), sum(ResolutionWidth + 17), sum(ResolutionWidth + 18), sum(ResolutionWidth + 19), sum(ResolutionWidth + 20), sum(ResolutionWidth + 21), sum(ResolutionWidth + 22), sum(ResolutionWidth + 23), sum(ResolutionWidth + 24), sum(ResolutionWidth + 25), sum(ResolutionWidth + 26), sum(ResolutionWidth + 27), sum(ResolutionWidth + 28), sum(ResolutionWidth + 29), sum(ResolutionWidth + 30), sum(ResolutionWidth + 31), sum(ResolutionWidth + 32), sum(ResolutionWidth + 33), sum(ResolutionWidth + 34), sum(ResolutionWidth + 35), sum(ResolutionWidth + 36), sum(ResolutionWidth + 37), sum(ResolutionWidth + 38), sum(ResolutionWidth + 39), sum(ResolutionWidth + 40), sum(ResolutionWidth + 41), sum(ResolutionWidth + 42), sum(ResolutionWidth + 43), sum(ResolutionWidth + 44), sum(ResolutionWidth + 45), sum(ResolutionWidth + 46), sum(ResolutionWidth + 47), sum(ResolutionWidth + 48), sum(ResolutionWidth + 49), sum(ResolutionWidth + 50), sum(ResolutionWidth + 51), sum(ResolutionWidth + 52), sum(ResolutionWidth + 53), sum(ResolutionWidth + 54), sum(ResolutionWidth + 55), sum(ResolutionWidth + 56), sum(ResolutionWidth + 57), sum(ResolutionWidth + 58), sum(ResolutionWidth + 59), sum(ResolutionWidth + 60), sum(ResolutionWidth + 61), sum(ResolutionWidth + 62), sum(ResolutionWidth + 63), sum(ResolutionWidth + 64), sum(ResolutionWidth + 65), sum(ResolutionWidth + 66), sum(ResolutionWidth + 67), sum(ResolutionWidth + 68), sum(ResolutionWidth + 69), sum(ResolutionWidth + 70), sum(ResolutionWidth + 71), sum(ResolutionWidth + 72), sum(ResolutionWidth + 73), sum(ResolutionWidth + 74), sum(ResolutionWidth + 75), sum(ResolutionWidth + 76), sum(ResolutionWidth + 77), sum(ResolutionWidth + 78), sum(ResolutionWidth + 79), sum(ResolutionWidth + 80), sum(ResolutionWidth + 81), sum(ResolutionWidth + 82), sum(ResolutionWidth + 83), sum(ResolutionWidth + 84), sum(ResolutionWidth + 85), sum(ResolutionWidth + 86), sum(ResolutionWidth + 87), sum(ResolutionWidth + 88), sum(ResolutionWidth + 89) FROM $test_table;"
|
||||
"#- много тупых агрегатных функций.;"
|
||||
|
||||
"SELECT SearchEngineID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM $test_table WHERE SearchPhrase != '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;"
|
||||
"#- сложная агрегация, для больших таблиц может не хватить оперативки.;"
|
||||
|
||||
"SELECT WatchID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM $test_table WHERE SearchPhrase != '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;"
|
||||
"#- агрегация по двум полям, которая ничего не агрегирует. Для больших таблиц выполнить не получится.;"
|
||||
|
||||
"SELECT WatchID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM $test_table GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;"
|
||||
"#- то же самое, но ещё и без фильтрации.;"
|
||||
|
||||
"SELECT URL, count() AS c FROM $test_table GROUP BY URL ORDER BY c DESC LIMIT 10;"
|
||||
"#- агрегация по URL.;"
|
||||
|
||||
"SELECT 1, URL, count() AS c FROM $test_table GROUP BY 1, URL ORDER BY c DESC LIMIT 10;"
|
||||
"#- агрегация по URL и числу.;"
|
||||
|
||||
"SELECT ClientIP AS x, x - 1, x - 2, x - 3, count() AS c FROM $test_table GROUP BY x, x - 1, x - 2, x - 3 ORDER BY c DESC LIMIT 10;"
|
||||
|
||||
"SELECT
|
||||
URL,
|
||||
count() AS PageViews
|
||||
FROM $test_table
|
||||
WHERE
|
||||
CounterID = $counter_id
|
||||
AND EventDate >= toDate($start_date)
|
||||
AND EventDate <= toDate($stop_date)
|
||||
AND NOT DontCountHits
|
||||
AND NOT Refresh
|
||||
AND notEmpty(URL)
|
||||
GROUP BY URL
|
||||
ORDER BY PageViews DESC
|
||||
LIMIT 10;"
|
||||
|
||||
|
||||
"SELECT
|
||||
Title,
|
||||
count() AS PageViews
|
||||
FROM $test_table
|
||||
WHERE
|
||||
CounterID = $counter_id
|
||||
AND EventDate >= toDate($start_date)
|
||||
AND EventDate <= toDate($stop_date)
|
||||
AND NOT DontCountHits
|
||||
AND NOT Refresh
|
||||
AND notEmpty(Title)
|
||||
GROUP BY Title
|
||||
ORDER BY PageViews DESC
|
||||
LIMIT 10;"
|
||||
|
||||
"SELECT
|
||||
URL,
|
||||
count() AS PageViews
|
||||
FROM $test_table
|
||||
WHERE
|
||||
CounterID = $counter_id
|
||||
AND EventDate >= toDate($start_date)
|
||||
AND EventDate <= toDate($stop_date)
|
||||
AND NOT Refresh
|
||||
AND IsLink
|
||||
AND NOT IsDownload
|
||||
GROUP BY URL
|
||||
ORDER BY PageViews DESC
|
||||
LIMIT 1000;"
|
||||
|
||||
"SELECT
|
||||
TraficSourceID,
|
||||
SearchEngineID,
|
||||
AdvEngineID,
|
||||
((SearchEngineID = 0 AND AdvEngineID = 0) ? Referer : '') AS Src,
|
||||
URL AS Dst,
|
||||
count() AS PageViews
|
||||
FROM $test_table
|
||||
WHERE
|
||||
CounterID = $counter_id
|
||||
AND EventDate >= toDate($start_date)
|
||||
AND EventDate <= toDate($stop_date)
|
||||
AND NOT Refresh
|
||||
GROUP BY
|
||||
TraficSourceID,
|
||||
SearchEngineID,
|
||||
AdvEngineID,
|
||||
Src,
|
||||
Dst
|
||||
ORDER BY PageViews DESC
|
||||
LIMIT 1000;"
|
||||
|
||||
"SELECT
|
||||
URLHash,
|
||||
EventDate,
|
||||
count() AS PageViews
|
||||
FROM $test_table
|
||||
WHERE
|
||||
CounterID = $counter_id
|
||||
AND EventDate >= toDate($start_date)
|
||||
AND EventDate <= toDate($stop_date)
|
||||
AND NOT Refresh
|
||||
AND TraficSourceID IN (-1, 6)
|
||||
AND RefererHash = halfMD5('http://example.ru/')
|
||||
GROUP BY
|
||||
URLHash,
|
||||
EventDate
|
||||
ORDER BY PageViews DESC
|
||||
LIMIT 100000;"
|
||||
|
||||
|
||||
"SELECT
|
||||
WindowClientWidth,
|
||||
WindowClientHeight,
|
||||
count() AS PageViews
|
||||
FROM $test_table
|
||||
WHERE
|
||||
CounterID = $counter_id
|
||||
AND EventDate >= toDate($start_date)
|
||||
AND EventDate <= toDate($stop_date)
|
||||
AND NOT Refresh
|
||||
AND NOT DontCountHits
|
||||
AND URLHash = halfMD5('http://example.ru/')
|
||||
GROUP BY
|
||||
WindowClientWidth,
|
||||
WindowClientHeight
|
||||
ORDER BY PageViews DESC
|
||||
LIMIT 10000;"
|
||||
|
||||
"SELECT
|
||||
toStartOfMinute(EventTime) AS Minute,
|
||||
count() AS PageViews
|
||||
FROM $test_table
|
||||
WHERE
|
||||
CounterID = $counter_id
|
||||
AND EventDate >= toDate($start_date)
|
||||
AND EventDate <= toDate($early_stop_date)
|
||||
AND NOT Refresh
|
||||
AND NOT DontCountHits
|
||||
GROUP BY
|
||||
Minute
|
||||
ORDER BY Minute;"
|
||||
|
||||
)
|
||||
|
||||
function test {
|
||||
TIMES=3
|
||||
execute "${test_queries[@]}"
|
||||
}
|
||||
|
||||
function init {
|
||||
execute "${init_queries[@]}"
|
||||
}
|
||||
|
||||
function debug {
|
||||
TIMES=3
|
||||
debug_queries=(
|
||||
)
|
||||
execute "${debug_queries[@]}"
|
||||
}
|
||||
|
||||
function usage {
|
||||
cat <<EOF
|
||||
usage: $0 options
|
||||
|
||||
This script run benhmark for clickhouse
|
||||
|
||||
OPTIONS:
|
||||
-h Show this message
|
||||
-d Run debug queries
|
||||
-i Init database
|
||||
-p log_file Parse log file to columns with result
|
||||
-t Run tests
|
||||
EOF
|
||||
}
|
||||
|
||||
function parse_log {
|
||||
results=$(cat $1 | grep -P 'Elapsed: \d+.\d+ ' | awk '{print $6}')
|
||||
|
||||
index=1
|
||||
for res in $results
|
||||
do
|
||||
echo -n "$res "
|
||||
let "index=$index % 3"
|
||||
if [ "$index" == "0" ]; then
|
||||
echo
|
||||
fi
|
||||
let "index=$index + 1"
|
||||
done
|
||||
}
|
||||
|
||||
if [ "$#" == "0" ]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Start date" $(date)
|
||||
|
||||
while getopts “hitdp:” OPTION
|
||||
do
|
||||
case $OPTION in
|
||||
h)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
i)
|
||||
init
|
||||
;;
|
||||
t)
|
||||
test
|
||||
;;
|
||||
d)
|
||||
debug
|
||||
;;
|
||||
p)
|
||||
parse_log $OPTARG
|
||||
;;
|
||||
?)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "Stop date" $(date)
|
||||
@ -1,4 +0,0 @@
|
||||
CONF_DIR=/home/kartavyy/benchmark/clickhouse
|
||||
expect_file=$CONF_DIR/expect.tcl
|
||||
test_file=$CONF_DIR/queries.sql
|
||||
etc_init_d_service=/etc/init.d/clickhouse-server-metrika-yandex
|
||||
@ -1,13 +0,0 @@
|
||||
#!/bin/expect
|
||||
|
||||
# Set timeout
|
||||
set timeout 600
|
||||
|
||||
# Get arguments
|
||||
set query [lindex $argv 0]
|
||||
|
||||
spawn clickhouse-client --multiline;
|
||||
expect ":) "
|
||||
send "$query;\r";
|
||||
expect ":) "
|
||||
send "quit";
|
||||
@ -1,26 +0,0 @@
|
||||
#!/bin/sh
|
||||
if [[ $# -ne 0 ]]; then
|
||||
echo "usage: if memory limit is exceeded kill process with biggest memory consumption"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
while [ 1=1 ];
|
||||
do
|
||||
FREE_MEMORY_MB=$(free -m | sed -n '3,3p' | awk '{print $4}')
|
||||
|
||||
PID="$(ps -eF --sort -rss | sed -n '2,2p' | awk '{print $2}')"
|
||||
NAME="$(ps -eF --sort -rss | sed -n '2,2p' | awk '{print $11}')"
|
||||
SIZEGB="$(ps -eF --sort -rss | sed -n '2,2p' | awk '{print $6}')"
|
||||
SIZEGB=$(($SIZEGB/1024/1024))
|
||||
|
||||
echo "Process id ="$PID" Size = "$SIZEGB" GB" "Free Memory = " $FREE_MEMORY_MB" MB"
|
||||
if (( $FREE_MEMORY_MB < 512 ));
|
||||
then echo "Killing the process with biggest memory consumption......"
|
||||
sudo kill -9 $PID
|
||||
echo "$(date) Killed the process with PID: $PID NAME: $NAME"
|
||||
else
|
||||
echo "SIZE has not yet exceeding"
|
||||
fi
|
||||
|
||||
sleep 10
|
||||
done
|
||||
@ -1,114 +0,0 @@
|
||||
from optparse import OptionParser
|
||||
import argparse
|
||||
|
||||
import re
|
||||
import sys
|
||||
|
||||
def log_to_rows(filename, pattern_select, time_pattern, pattern_ignore):
|
||||
time_matcher = re.compile(time_pattern)
|
||||
select_matcher = re.compile(pattern_select, re.IGNORECASE);
|
||||
ignore_matcher = re.compile(pattern_ignore)
|
||||
|
||||
f = open(filename, 'r');
|
||||
|
||||
query = ''
|
||||
raw_time = ''
|
||||
for line in f:
|
||||
if ignore_matcher.match(line):
|
||||
continue
|
||||
|
||||
m = select_matcher.search(line)
|
||||
if m :
|
||||
if line != query:
|
||||
query = line
|
||||
sys.stdout.write("\n")
|
||||
raw_time = raw_time + "\n"
|
||||
|
||||
m = time_matcher.search(line)
|
||||
if m:
|
||||
sec = 0
|
||||
minute = 0
|
||||
ms = 0
|
||||
if 'min' in m.groupdict() and m.group('min'):
|
||||
minute = float(m.group('min').replace(',','.'))
|
||||
if 'sec' in m.groupdict() and m.group('sec'):
|
||||
sec = float(m.group('sec').replace(',','.'))
|
||||
if 'ms' in m.groupdict() and m.group('ms'):
|
||||
ms = float(m.group('ms').replace(',', '.'))
|
||||
|
||||
sys.stdout.write( str(minute*60 + sec + ms/1000.) + " " )
|
||||
raw_time = raw_time + " | " + m.group('time')
|
||||
|
||||
print
|
||||
print " =======raw time====== \n" + raw_time
|
||||
|
||||
|
||||
def process_log(filename, pattern_select, time_pattern, pattern_ignore, error_pattern):
|
||||
time_matcher = re.compile(time_pattern)
|
||||
select_matcher = re.compile(pattern_select, re.IGNORECASE);
|
||||
ignore_matcher = re.compile(pattern_ignore)
|
||||
error_matcher = re.compile(error_pattern, re.IGNORECASE)
|
||||
|
||||
f = open(filename, 'r');
|
||||
|
||||
query = ''
|
||||
for line in f:
|
||||
if error_matcher.match(line):
|
||||
print line
|
||||
continue
|
||||
|
||||
if ignore_matcher.match(line):
|
||||
continue
|
||||
|
||||
m = select_matcher.search(line)
|
||||
if m :
|
||||
if line != query:
|
||||
sys.stdout.flush()
|
||||
query = line
|
||||
print "\n\n"
|
||||
print query
|
||||
|
||||
m = time_matcher.search(line)
|
||||
if m:
|
||||
sys.stdout.write(m.group('time') + " " )
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Process log files form different databases")
|
||||
parser.add_argument('log_file', metavar = 'log_file', help = 'database log file')
|
||||
parser.add_argument('db_name', metavar = 'db_name', help = ' database name one of clickhouse, vertica, infinidb, monetdb, infobright, hive (... more later)')
|
||||
args = parser.parse_args()
|
||||
|
||||
log_file = args.log_file
|
||||
db_name = args.db_name
|
||||
|
||||
time_pattern = ''
|
||||
select_pattern = r'query: select '
|
||||
ignore_pattern = r'#'
|
||||
error_pattern = r'error .*'
|
||||
if db_name == 'clickhouse':
|
||||
time_pattern = r'(?P<time>(?P<sec>\d+.\d{3}) sec\.)'
|
||||
select_pattern = r'query\: select '
|
||||
ignore_pattern = r':\).*'
|
||||
elif db_name == 'vertica' :
|
||||
time_pattern = r'(?P<time>(?P<ms>\d+.\d+) ms\.)'
|
||||
select_pattern = r'select '
|
||||
ignore_pattern = r'(.*dbadmin=>|query:|.*Timing is on\.).*'
|
||||
elif db_name == 'infinidb' :
|
||||
time_pattern = r'(?P<time>(?:(?P<min>\d+) min )?(?P<sec>\d+.\d+) sec)'
|
||||
ignore_pattern = r'Query OK, 0 rows affected \(0\.00 sec\)'
|
||||
elif db_name == 'monetdb' :
|
||||
time_pattern = r'tuples? \((?P<time>(?:(?P<min>\d+)m )?(?:(?P<sec>\d+.?\d+)s)?(?:(?P<ms>\d+.\d+)ms)?)\)'
|
||||
elif db_name == 'infobright' :
|
||||
time_pattern = r'(?P<time>(?:(?P<min>\d+) min ){0,1}(?P<sec>\d+.\d+) sec)'
|
||||
elif db_name == 'hive':
|
||||
time_pattern = r'Time taken\: (?P<time>(?:(?P<sec>\d+.?\d+) seconds))'
|
||||
error_pattern = r'failed\: .*'
|
||||
elif db_name == 'mysql':
|
||||
time_pattern = r'(?P<time>(?:(?P<min>\d+) min )?(?P<sec>\d+.\d+) sec)'
|
||||
else:
|
||||
sys.exit("unknown db_name")
|
||||
|
||||
process_log(log_file, select_pattern, time_pattern, ignore_pattern, error_pattern )
|
||||
log_to_rows(log_file, select_pattern, time_pattern, ignore_pattern )
|
||||
|
||||
main()
|
||||
@ -28,7 +28,7 @@ PS. Note that Vertica doesn't support IPv6.
|
||||
How to prepare data
|
||||
-------------------
|
||||
|
||||
Prepare dumps with script create_dump.sh for tables hits_10m, hits_100m, hits_1000m. It takes about 5 hours (1m41.882s, 25m11.103s, ).
|
||||
Prepare dumps with script create_dump.sh for tables hits_10m, hits_100m, hits_1000m. It takes about 5 hours (1m41.882s, 25m11.103s, 276m36.388s).
|
||||
Start vsql command line client.
|
||||
Create tables with queries from hits_define_schema.sql.
|
||||
|
||||
|
||||
24
dbms/benchmark/vertica/benchmark.sh
Normal file
24
dbms/benchmark/vertica/benchmark.sh
Normal file
@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
|
||||
QUERIES_FILE="queries.sql"
|
||||
TABLE=$1
|
||||
TRIES=3
|
||||
|
||||
cat "$QUERIES_FILE" | sed "s/{table}/${TABLE}/g" | while read query; do
|
||||
sync
|
||||
echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null
|
||||
|
||||
echo -n "["
|
||||
for i in $(seq 1 $TRIES); do
|
||||
|
||||
RES=$((echo '\timing'; echo "$query") |
|
||||
/opt/vertica/bin/vsql -U dbadmin |
|
||||
grep -oP 'All rows formatted: [^ ]+ ms' |
|
||||
ssed -R -e 's/^All rows formatted: ([\d,]+) ms$/\1/' |
|
||||
tr ',' '.')
|
||||
|
||||
[[ "$?" == "0" ]] && echo -n "$(perl -e "print ${RES} / 1000")" || echo -n "null"
|
||||
[[ "$i" != $TRIES ]] && echo -n ", "
|
||||
done
|
||||
echo "],"
|
||||
done
|
||||
@ -1,43 +1,43 @@
|
||||
SELECT count(*) FROM hits_100m_meshed;
|
||||
SELECT count(*) FROM hits_100m_meshed WHERE AdvEngineID != 0;
|
||||
SELECT sum(AdvEngineID), count(*), avg(ResolutionWidth) FROM hits_100m_meshed;
|
||||
SELECT sum_float(UserID) FROM hits_100m_meshed;
|
||||
SELECT count(DISTINCT UserID) FROM hits_100m_meshed;
|
||||
SELECT count(DISTINCT SearchPhrase) FROM hits_100m_meshed;
|
||||
SELECT min(EventDate), max(EventDate) FROM hits_100m_meshed;
|
||||
SELECT AdvEngineID, count(*) FROM hits_100m_meshed WHERE AdvEngineID != 0 GROUP BY AdvEngineID ORDER BY count(*) DESC;
|
||||
SELECT RegionID, count(DISTINCT UserID) AS u FROM hits_100m_meshed GROUP BY RegionID ORDER BY u DESC LIMIT 10;
|
||||
SELECT RegionID, sum(AdvEngineID), count(*) AS c, avg(ResolutionWidth), count(DISTINCT UserID) FROM hits_100m_meshed GROUP BY RegionID ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT MobilePhoneModel, count(DISTINCT UserID) AS u FROM hits_100m_meshed WHERE MobilePhoneModel != '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;
|
||||
SELECT MobilePhone, MobilePhoneModel, count(DISTINCT UserID) AS u FROM hits_100m_meshed WHERE MobilePhoneModel != '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;
|
||||
SELECT SearchPhrase, count(*) FROM hits_100m_meshed WHERE SearchPhrase != '' GROUP BY SearchPhrase ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT SearchPhrase, count(DISTINCT UserID) AS u FROM hits_100m_meshed WHERE SearchPhrase != '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;
|
||||
SELECT SearchEngineID, SearchPhrase, count(*) FROM hits_100m_meshed WHERE SearchPhrase != '' GROUP BY SearchEngineID, SearchPhrase ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT UserID, count(*) FROM hits_100m_meshed GROUP BY UserID ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT UserID, SearchPhrase, count(*) FROM hits_100m_meshed GROUP BY UserID, SearchPhrase ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT UserID, SearchPhrase, count(*) FROM hits_100m_meshed GROUP BY UserID, SearchPhrase LIMIT 10;
|
||||
SELECT UserID, Minute(EventTime) AS m, SearchPhrase, count(*) FROM hits_100m_meshed GROUP BY UserID, m, SearchPhrase ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT UserID FROM hits_100m_meshed WHERE UserID = 12345678901234567890;
|
||||
SELECT count(*) FROM hits_100m_meshed WHERE URL LIKE '%metrika%';
|
||||
SELECT SearchPhrase, MAX(URL), count(*) FROM hits_100m_meshed WHERE URL LIKE '%metrika%' AND SearchPhrase != '' GROUP BY SearchPhrase ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT SearchPhrase, MAX(URL), MAX(Title), count(*) AS c, count(DISTINCT UserID) FROM hits_100m_meshed WHERE Title LIKE '%Яндекс%' AND URL NOT LIKE '%.yandex.%' AND SearchPhrase != '' GROUP BY SearchPhrase ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT * FROM hits_100m_meshed WHERE URL LIKE '%metrika%' ORDER BY EventTime LIMIT 10;
|
||||
SELECT SearchPhrase FROM hits_100m_meshed WHERE SearchPhrase != '' ORDER BY EventTime LIMIT 10;
|
||||
SELECT SearchPhrase FROM hits_100m_meshed WHERE SearchPhrase != '' ORDER BY SearchPhrase LIMIT 10;
|
||||
SELECT SearchPhrase FROM hits_100m_meshed WHERE SearchPhrase != '' ORDER BY EventTime, SearchPhrase LIMIT 10;
|
||||
SELECT CounterID, avg(length(URL)) AS l, count(*) FROM hits_100m_meshed WHERE URL != '' GROUP BY CounterID HAVING count(*) > 100000 ORDER BY l DESC LIMIT 25;
|
||||
SELECT SUBSTRING(SUBSTRING(Referer, POSITION('//' IN Referer) + 2), 1, GREATEST(0, POSITION('/' IN SUBSTRING(Referer, POSITION('//' IN Referer) + 2)) - 1)) AS key, avg(length(Referer)) AS l, count(*) AS c, MAX(Referer) FROM hits_10m_meshed WHERE Referer != '' GROUP BY key HAVING count(*) > 100000 ORDER BY l DESC LIMIT 25;
|
||||
SELECT sum(ResolutionWidth), sum(ResolutionWidth + 1), sum(ResolutionWidth + 2), sum(ResolutionWidth + 3), sum(ResolutionWidth + 4), sum(ResolutionWidth + 5), sum(ResolutionWidth + 6), sum(ResolutionWidth + 7), sum(ResolutionWidth + 8), sum(ResolutionWidth + 9), sum(ResolutionWidth + 10), sum(ResolutionWidth + 11), sum(ResolutionWidth + 12), sum(ResolutionWidth + 13), sum(ResolutionWidth + 14), sum(ResolutionWidth + 15), sum(ResolutionWidth + 16), sum(ResolutionWidth + 17), sum(ResolutionWidth + 18), sum(ResolutionWidth + 19), sum(ResolutionWidth + 20), sum(ResolutionWidth + 21), sum(ResolutionWidth + 22), sum(ResolutionWidth + 23), sum(ResolutionWidth + 24), sum(ResolutionWidth + 25), sum(ResolutionWidth + 26), sum(ResolutionWidth + 27), sum(ResolutionWidth + 28), sum(ResolutionWidth + 29), sum(ResolutionWidth + 30), sum(ResolutionWidth + 31), sum(ResolutionWidth + 32), sum(ResolutionWidth + 33), sum(ResolutionWidth + 34), sum(ResolutionWidth + 35), sum(ResolutionWidth + 36), sum(ResolutionWidth + 37), sum(ResolutionWidth + 38), sum(ResolutionWidth + 39), sum(ResolutionWidth + 40), sum(ResolutionWidth + 41), sum(ResolutionWidth + 42), sum(ResolutionWidth + 43), sum(ResolutionWidth + 44), sum(ResolutionWidth + 45), sum(ResolutionWidth + 46), sum(ResolutionWidth + 47), sum(ResolutionWidth + 48), sum(ResolutionWidth + 49), sum(ResolutionWidth + 50), sum(ResolutionWidth + 51), sum(ResolutionWidth + 52), sum(ResolutionWidth + 53), sum(ResolutionWidth + 54), sum(ResolutionWidth + 55), sum(ResolutionWidth + 56), sum(ResolutionWidth + 57), sum(ResolutionWidth + 58), sum(ResolutionWidth + 59), sum(ResolutionWidth + 60), sum(ResolutionWidth + 61), sum(ResolutionWidth + 62), sum(ResolutionWidth + 63), sum(ResolutionWidth + 64), sum(ResolutionWidth + 65), sum(ResolutionWidth + 66), sum(ResolutionWidth + 67), sum(ResolutionWidth + 68), sum(ResolutionWidth + 69), sum(ResolutionWidth + 70), sum(ResolutionWidth + 71), sum(ResolutionWidth + 72), sum(ResolutionWidth + 73), sum(ResolutionWidth + 74), sum(ResolutionWidth + 75), sum(ResolutionWidth + 76), sum(ResolutionWidth + 77), sum(ResolutionWidth + 78), sum(ResolutionWidth + 79), sum(ResolutionWidth + 80), sum(ResolutionWidth + 81), sum(ResolutionWidth + 82), sum(ResolutionWidth + 83), sum(ResolutionWidth + 84), sum(ResolutionWidth + 85), sum(ResolutionWidth + 86), sum(ResolutionWidth + 87), sum(ResolutionWidth + 88), sum(ResolutionWidth + 89) FROM hits_100m_meshed;
|
||||
SELECT SearchEngineID, ClientIP, count(*) AS c, sum(Refresh), avg(ResolutionWidth) FROM hits_100m_meshed WHERE SearchPhrase != '' GROUP BY SearchEngineID, ClientIP ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT WatchID, ClientIP, count(*) AS c, sum(Refresh), avg(ResolutionWidth) FROM hits_100m_meshed WHERE SearchPhrase != '' GROUP BY WatchID, ClientIP ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT WatchID, ClientIP, count(*) AS c, sum(Refresh), avg(ResolutionWidth) FROM hits_100m_meshed GROUP BY WatchID, ClientIP ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT URL, count(*) FROM hits_100m_meshed GROUP BY URL ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT 1, URL, count(*) FROM hits_100m_meshed GROUP BY 1, URL ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, count(*) FROM hits_100m_meshed GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT URL, count(*) AS PageViews FROM hits_100m_meshed WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND URL != '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;
|
||||
SELECT Title, count(*) AS PageViews FROM hits_100m_meshed WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND Title != '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;
|
||||
SELECT URL, count(*) AS PageViews FROM hits_100m_meshed WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000;
|
||||
SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN SearchEngineID = 0 AND AdvEngineID = 0 THEN Referer ELSE '' END AS Src, URL AS Dst, count(*) AS PageViews FROM hits_100m_meshed WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000;
|
||||
SELECT URLHash, EventDate, count(*) AS PageViews FROM hits_100m_meshed WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = 6202628419148573758 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100000;
|
||||
SELECT WindowClientWidth, WindowClientHeight, count(*) AS PageViews FROM hits_100m_meshed WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND NOT DontCountHits AND URLHash = 6202628419148573758 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000;
|
||||
SELECT TIME_SLICE(EventTime, 1, 'MINUTE') AS Minute, count(*) AS PageViews FROM hits_100m_meshed WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-02') AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute;
|
||||
SELECT count(*) FROM {table};
|
||||
SELECT count(*) FROM {table} WHERE AdvEngineID != 0;
|
||||
SELECT sum(AdvEngineID), count(*), avg(ResolutionWidth) FROM {table};
|
||||
SELECT sum_float(UserID) FROM {table};
|
||||
SELECT count(DISTINCT UserID) FROM {table};
|
||||
SELECT count(DISTINCT SearchPhrase) FROM {table};
|
||||
SELECT min(EventDate), max(EventDate) FROM {table};
|
||||
SELECT AdvEngineID, count(*) FROM {table} WHERE AdvEngineID != 0 GROUP BY AdvEngineID ORDER BY count(*) DESC;
|
||||
SELECT RegionID, count(DISTINCT UserID) AS u FROM {table} GROUP BY RegionID ORDER BY u DESC LIMIT 10;
|
||||
SELECT RegionID, sum(AdvEngineID), count(*) AS c, avg(ResolutionWidth), count(DISTINCT UserID) FROM {table} GROUP BY RegionID ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT MobilePhoneModel, count(DISTINCT UserID) AS u FROM {table} WHERE MobilePhoneModel != '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;
|
||||
SELECT MobilePhone, MobilePhoneModel, count(DISTINCT UserID) AS u FROM {table} WHERE MobilePhoneModel != '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;
|
||||
SELECT SearchPhrase, count(*) FROM {table} WHERE SearchPhrase != '' GROUP BY SearchPhrase ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT SearchPhrase, count(DISTINCT UserID) AS u FROM {table} WHERE SearchPhrase != '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;
|
||||
SELECT SearchEngineID, SearchPhrase, count(*) FROM {table} WHERE SearchPhrase != '' GROUP BY SearchEngineID, SearchPhrase ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT UserID, count(*) FROM {table} GROUP BY UserID ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT UserID, SearchPhrase, count(*) FROM {table} GROUP BY UserID, SearchPhrase ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT UserID, SearchPhrase, count(*) FROM {table} GROUP BY UserID, SearchPhrase LIMIT 10;
|
||||
SELECT UserID, Minute(EventTime) AS m, SearchPhrase, count(*) FROM {table} GROUP BY UserID, m, SearchPhrase ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT UserID FROM {table} WHERE UserID = 12345678901234567890;
|
||||
SELECT count(*) FROM {table} WHERE URL LIKE '%metrika%';
|
||||
SELECT SearchPhrase, MAX(URL), count(*) FROM {table} WHERE URL LIKE '%metrika%' AND SearchPhrase != '' GROUP BY SearchPhrase ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT SearchPhrase, MAX(URL), MAX(Title), count(*) AS c, count(DISTINCT UserID) FROM {table} WHERE Title LIKE '%Яндекс%' AND URL NOT LIKE '%.yandex.%' AND SearchPhrase != '' GROUP BY SearchPhrase ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT * FROM {table} WHERE URL LIKE '%metrika%' ORDER BY EventTime LIMIT 10;
|
||||
SELECT SearchPhrase FROM {table} WHERE SearchPhrase != '' ORDER BY EventTime LIMIT 10;
|
||||
SELECT SearchPhrase FROM {table} WHERE SearchPhrase != '' ORDER BY SearchPhrase LIMIT 10;
|
||||
SELECT SearchPhrase FROM {table} WHERE SearchPhrase != '' ORDER BY EventTime, SearchPhrase LIMIT 10;
|
||||
SELECT CounterID, avg(length(URL)) AS l, count(*) FROM {table} WHERE URL != '' GROUP BY CounterID HAVING count(*) > 100000 ORDER BY l DESC LIMIT 25;
|
||||
SELECT SUBSTRING(SUBSTRING(Referer, POSITION('//' IN Referer) + 2), 1, GREATEST(0, POSITION('/' IN SUBSTRING(Referer, POSITION('//' IN Referer) + 2)) - 1)) AS key, avg(length(Referer)) AS l, count(*) AS c, MAX(Referer) FROM {table} WHERE Referer != '' GROUP BY key HAVING count(*) > 100000 ORDER BY l DESC LIMIT 25;
|
||||
SELECT sum(ResolutionWidth), sum(ResolutionWidth + 1), sum(ResolutionWidth + 2), sum(ResolutionWidth + 3), sum(ResolutionWidth + 4), sum(ResolutionWidth + 5), sum(ResolutionWidth + 6), sum(ResolutionWidth + 7), sum(ResolutionWidth + 8), sum(ResolutionWidth + 9), sum(ResolutionWidth + 10), sum(ResolutionWidth + 11), sum(ResolutionWidth + 12), sum(ResolutionWidth + 13), sum(ResolutionWidth + 14), sum(ResolutionWidth + 15), sum(ResolutionWidth + 16), sum(ResolutionWidth + 17), sum(ResolutionWidth + 18), sum(ResolutionWidth + 19), sum(ResolutionWidth + 20), sum(ResolutionWidth + 21), sum(ResolutionWidth + 22), sum(ResolutionWidth + 23), sum(ResolutionWidth + 24), sum(ResolutionWidth + 25), sum(ResolutionWidth + 26), sum(ResolutionWidth + 27), sum(ResolutionWidth + 28), sum(ResolutionWidth + 29), sum(ResolutionWidth + 30), sum(ResolutionWidth + 31), sum(ResolutionWidth + 32), sum(ResolutionWidth + 33), sum(ResolutionWidth + 34), sum(ResolutionWidth + 35), sum(ResolutionWidth + 36), sum(ResolutionWidth + 37), sum(ResolutionWidth + 38), sum(ResolutionWidth + 39), sum(ResolutionWidth + 40), sum(ResolutionWidth + 41), sum(ResolutionWidth + 42), sum(ResolutionWidth + 43), sum(ResolutionWidth + 44), sum(ResolutionWidth + 45), sum(ResolutionWidth + 46), sum(ResolutionWidth + 47), sum(ResolutionWidth + 48), sum(ResolutionWidth + 49), sum(ResolutionWidth + 50), sum(ResolutionWidth + 51), sum(ResolutionWidth + 52), sum(ResolutionWidth + 53), sum(ResolutionWidth + 54), sum(ResolutionWidth + 55), sum(ResolutionWidth + 56), sum(ResolutionWidth + 57), sum(ResolutionWidth + 58), sum(ResolutionWidth + 59), sum(ResolutionWidth + 60), sum(ResolutionWidth + 61), sum(ResolutionWidth + 62), sum(ResolutionWidth + 63), sum(ResolutionWidth + 64), sum(ResolutionWidth + 65), sum(ResolutionWidth + 66), sum(ResolutionWidth + 67), sum(ResolutionWidth + 68), sum(ResolutionWidth + 69), sum(ResolutionWidth + 70), sum(ResolutionWidth + 71), sum(ResolutionWidth + 72), sum(ResolutionWidth + 73), sum(ResolutionWidth + 74), sum(ResolutionWidth + 75), sum(ResolutionWidth + 76), sum(ResolutionWidth + 77), sum(ResolutionWidth + 78), sum(ResolutionWidth + 79), sum(ResolutionWidth + 80), sum(ResolutionWidth + 81), sum(ResolutionWidth + 82), sum(ResolutionWidth + 83), sum(ResolutionWidth + 84), sum(ResolutionWidth + 85), sum(ResolutionWidth + 86), sum(ResolutionWidth + 87), sum(ResolutionWidth + 88), sum(ResolutionWidth + 89) FROM {table};
|
||||
SELECT SearchEngineID, ClientIP, count(*) AS c, sum(Refresh), avg(ResolutionWidth) FROM {table} WHERE SearchPhrase != '' GROUP BY SearchEngineID, ClientIP ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT WatchID, ClientIP, count(*) AS c, sum(Refresh), avg(ResolutionWidth) FROM {table} WHERE SearchPhrase != '' GROUP BY WatchID, ClientIP ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT WatchID, ClientIP, count(*) AS c, sum(Refresh), avg(ResolutionWidth) FROM {table} GROUP BY WatchID, ClientIP ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT URL, count(*) FROM {table} GROUP BY URL ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT 1, URL, count(*) FROM {table} GROUP BY 1, URL ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, count(*) FROM {table} GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY count(*) DESC LIMIT 10;
|
||||
SELECT URL, count(*) AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND URL != '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;
|
||||
SELECT Title, count(*) AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND Title != '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;
|
||||
SELECT URL, count(*) AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000;
|
||||
SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN SearchEngineID = 0 AND AdvEngineID = 0 THEN Referer ELSE '' END AS Src, URL AS Dst, count(*) AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000;
|
||||
SELECT URLHash, EventDate, count(*) AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = 6202628419148573758 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100000;
|
||||
SELECT WindowClientWidth, WindowClientHeight, count(*) AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND NOT DontCountHits AND URLHash = 6202628419148573758 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000;
|
||||
SELECT TIME_SLICE(EventTime, 1, 'MINUTE') AS Minute, count(*) AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-02') AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user