mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 08:40:50 +00:00
Merge branch 'master' into add-support-for-settings-alias
This commit is contained in:
Antonio Andelic
commit
f613bfd8d2
@ -1,6 +1,10 @@
|
||||
set (SOURCE_DIR "${CMAKE_SOURCE_DIR}/contrib/snappy")
|
||||
|
||||
set (SNAPPY_IS_BIG_ENDIAN 0)
|
||||
if (ARCH_S390X)
|
||||
set (SNAPPY_IS_BIG_ENDIAN 1)
|
||||
else ()
|
||||
set (SNAPPY_IS_BIG_ENDIAN 0)
|
||||
endif()
|
||||
|
||||
set (HAVE_BYTESWAP_H 1)
|
||||
set (HAVE_SYS_MMAN_H 1)
|
||||
|
||||
@ -11,6 +11,18 @@ set -x
|
||||
# core.COMM.PID-TID
|
||||
sysctl kernel.core_pattern='core.%e.%p-%P'
|
||||
|
||||
OK="\tOK\t\\N\t"
|
||||
FAIL="\tFAIL\t\\N\t"
|
||||
function escaped()
|
||||
{
|
||||
# That's the simplest way I found to escape a string in bash. Yep, bash is the most convenient programming language.
|
||||
clickhouse local -S 's String' --input-format=LineAsString -q "select * from table format CustomSeparated settings format_custom_row_after_delimiter='\\\\\\\\n'"
|
||||
}
|
||||
|
||||
function head_escaped()
|
||||
{
|
||||
head -50 $1 | escaped
|
||||
}
|
||||
|
||||
function install_packages()
|
||||
{
|
||||
@ -33,7 +45,9 @@ function configure()
|
||||
ln -s /usr/share/clickhouse-test/ci/get_previous_release_tag.py /usr/bin/get_previous_release_tag
|
||||
|
||||
# avoid too slow startup
|
||||
sudo cat /etc/clickhouse-server/config.d/keeper_port.xml | sed "s|<snapshot_distance>100000</snapshot_distance>|<snapshot_distance>10000</snapshot_distance>|" > /etc/clickhouse-server/config.d/keeper_port.xml.tmp
|
||||
sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \
|
||||
| sed "s|<snapshot_distance>100000</snapshot_distance>|<snapshot_distance>10000</snapshot_distance>|" \
|
||||
> /etc/clickhouse-server/config.d/keeper_port.xml.tmp
|
||||
sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml
|
||||
sudo chown clickhouse /etc/clickhouse-server/config.d/keeper_port.xml
|
||||
sudo chgrp clickhouse /etc/clickhouse-server/config.d/keeper_port.xml
|
||||
@ -136,6 +150,7 @@ function stop()
|
||||
clickhouse stop --max-tries "$max_tries" --do-not-kill && return
|
||||
|
||||
# We failed to stop the server with SIGTERM. Maybe it hang, let's collect stacktraces.
|
||||
echo -e "Possible deadlock on shutdown (see gdb.log)$FAIL" >> /test_output/test_results.tsv
|
||||
kill -TERM "$(pidof gdb)" ||:
|
||||
sleep 5
|
||||
echo "thread apply all backtrace (on stop)" >> /test_output/gdb.log
|
||||
@ -151,10 +166,11 @@ function start()
|
||||
if [ "$counter" -gt ${1:-120} ]
|
||||
then
|
||||
echo "Cannot start clickhouse-server"
|
||||
echo -e "Cannot start clickhouse-server\tFAIL" >> /test_output/test_results.tsv
|
||||
rg --text "<Error>.*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt ||:
|
||||
echo -e "Cannot start clickhouse-server$FAIL$(head_escaped /test_output/application_errors.txt)" >> /test_output/test_results.tsv
|
||||
cat /var/log/clickhouse-server/stdout.log
|
||||
tail -n1000 /var/log/clickhouse-server/stderr.log
|
||||
tail -n100000 /var/log/clickhouse-server/clickhouse-server.log | rg -F -v -e '<Warning> RaftInstance:' -e '<Information> RaftInstance' | tail -n1000
|
||||
tail -n100 /var/log/clickhouse-server/stderr.log
|
||||
tail -n100000 /var/log/clickhouse-server/clickhouse-server.log | rg -F -v -e '<Warning> RaftInstance:' -e '<Information> RaftInstance' | tail -n100
|
||||
break
|
||||
fi
|
||||
# use root to match with current uid
|
||||
@ -252,9 +268,92 @@ start
|
||||
clickhouse-client --query "SHOW TABLES FROM datasets"
|
||||
clickhouse-client --query "SHOW TABLES FROM test"
|
||||
|
||||
clickhouse-client --query "CREATE TABLE test.hits_s3 (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
|
||||
clickhouse-client --query "CREATE TABLE test.hits (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
|
||||
clickhouse-client --query "CREATE TABLE test.visits (CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8, VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32, Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String, EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32, SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32, SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16, UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16, FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8, Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), Params Array(String), Goals Nested(ID UInt32, Serial UInt32, EventTime DateTime, Price Int64, OrderID String, CurrencyID UInt32), WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64, ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32, ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32, ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32, ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16, ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime, PredLastVisit Date, LastVisit Date, TotalVisits UInt32, TraficSource Nested(ID Int8, SearchEngineID UInt16, AdvEngineID UInt8, PlaceID UInt16, SocialSourceNetworkID UInt8, Domain String, SearchPhrase String, SocialSourcePage String), Attendance FixedString(16), CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64, StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64, OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64, UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), Market Nested(Type UInt8, GoalID UInt32, OrderID String, OrderPrice Int64, PP UInt32, DirectPlaceID UInt32, DirectOrderID UInt32, DirectBannerID UInt32, GoodID String, GoodName String, GoodQuantity Int32, GoodPrice Int64), IslandID FixedString(16)) ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
|
||||
clickhouse-client --query "CREATE TABLE test.hits_s3 (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16,
|
||||
EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32,
|
||||
UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String,
|
||||
Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32),
|
||||
RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8,
|
||||
FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2),
|
||||
CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String,
|
||||
IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8,
|
||||
WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8,
|
||||
SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32,
|
||||
IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8,
|
||||
IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8,
|
||||
Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32,
|
||||
RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2),
|
||||
BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32,
|
||||
DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32,
|
||||
RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32,
|
||||
LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32,
|
||||
RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String,
|
||||
ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String,
|
||||
OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String,
|
||||
UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64,
|
||||
URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String,
|
||||
ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64),
|
||||
IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate)
|
||||
ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
|
||||
clickhouse-client --query "CREATE TABLE test.hits (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16,
|
||||
EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32,
|
||||
UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String,
|
||||
RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16),
|
||||
URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8,
|
||||
FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16,
|
||||
UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8,
|
||||
MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16,
|
||||
SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16,
|
||||
ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32,
|
||||
SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8,
|
||||
FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8,
|
||||
HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8,
|
||||
GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32,
|
||||
HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String,
|
||||
HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32,
|
||||
FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32,
|
||||
LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32,
|
||||
RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String,
|
||||
ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String,
|
||||
OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String,
|
||||
UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64,
|
||||
URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String,
|
||||
ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64),
|
||||
IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate)
|
||||
ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
|
||||
clickhouse-client --query "CREATE TABLE test.visits (CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8,
|
||||
VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32,
|
||||
Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String,
|
||||
EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String,
|
||||
AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32),
|
||||
RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32,
|
||||
SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32,
|
||||
ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32,
|
||||
SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16,
|
||||
UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16,
|
||||
FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8,
|
||||
FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8,
|
||||
Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8,
|
||||
BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16),
|
||||
Params Array(String), Goals Nested(ID UInt32, Serial UInt32, EventTime DateTime, Price Int64, OrderID String, CurrencyID UInt32),
|
||||
WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64,
|
||||
ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32,
|
||||
ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32,
|
||||
ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32,
|
||||
ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16,
|
||||
ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32,
|
||||
OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String,
|
||||
UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime,
|
||||
PredLastVisit Date, LastVisit Date, TotalVisits UInt32, TraficSource Nested(ID Int8, SearchEngineID UInt16, AdvEngineID UInt8,
|
||||
PlaceID UInt16, SocialSourceNetworkID UInt8, Domain String, SearchPhrase String, SocialSourcePage String), Attendance FixedString(16),
|
||||
CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64,
|
||||
StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64,
|
||||
OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64,
|
||||
UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32,
|
||||
ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64),
|
||||
Market Nested(Type UInt8, GoalID UInt32, OrderID String, OrderPrice Int64, PP UInt32, DirectPlaceID UInt32, DirectOrderID UInt32,
|
||||
DirectBannerID UInt32, GoodID String, GoodName String, GoodQuantity Int32, GoodPrice Int64), IslandID FixedString(16))
|
||||
ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
|
||||
SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
|
||||
|
||||
clickhouse-client --query "INSERT INTO test.hits_s3 SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0"
|
||||
clickhouse-client --query "INSERT INTO test.hits SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0"
|
||||
@ -275,7 +374,9 @@ export ZOOKEEPER_FAULT_INJECTION=1
|
||||
configure
|
||||
|
||||
# But we still need default disk because some tables loaded only into it
|
||||
sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml | sed "s|<main><disk>s3</disk></main>|<main><disk>s3</disk></main><default><disk>default</disk></default>|" > /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp
|
||||
sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml \
|
||||
| sed "s|<main><disk>s3</disk></main>|<main><disk>s3</disk></main><default><disk>default</disk></default>|" \
|
||||
> /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp
|
||||
mv /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
|
||||
sudo chown clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
|
||||
sudo chgrp clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
|
||||
@ -283,8 +384,12 @@ sudo chgrp clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_defau
|
||||
start
|
||||
|
||||
./stress --hung-check --drop-databases --output-folder test_output --skip-func-tests "$SKIP_TESTS_OPTION" --global-time-limit 1200 \
|
||||
&& echo -e 'Test script exit code\tOK' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Test script failed\tFAIL' >> /test_output/test_results.tsv
|
||||
&& echo -e "Test script exit code$OK" >> /test_output/test_results.tsv \
|
||||
|| echo -e "Test script failed$FAIL script exit code: $?" >> /test_output/test_results.tsv
|
||||
|
||||
# NOTE Hung check is implemented in docker/tests/stress/stress
|
||||
rg -Fa "No queries hung" /test_output/test_results.tsv | grep -Fa "OK" \
|
||||
|| echo -e "Hung check failed, possible deadlock found (see hung_check.log)$FAIL$(head_escaped /test_output/hung_check.log)"
|
||||
|
||||
stop
|
||||
mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.stress.log
|
||||
@ -295,9 +400,10 @@ unset "${!THREAD_@}"
|
||||
|
||||
start
|
||||
|
||||
clickhouse-client --query "SELECT 'Server successfully started', 'OK'" >> /test_output/test_results.tsv \
|
||||
|| (echo -e 'Server failed to start (see application_errors.txt and clickhouse-server.clean.log)\tFAIL' >> /test_output/test_results.tsv \
|
||||
&& rg --text "<Error>.*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt)
|
||||
clickhouse-client --query "SELECT 'Server successfully started', 'OK', NULL, ''" >> /test_output/test_results.tsv \
|
||||
|| (rg --text "<Error>.*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt \
|
||||
&& echo -e "Server failed to start (see application_errors.txt and clickhouse-server.clean.log)$FAIL$(head_escaped /test_output/application_errors.txt)" \
|
||||
>> /test_output/test_results.tsv)
|
||||
|
||||
stop
|
||||
|
||||
@ -310,49 +416,49 @@ stop
|
||||
rg -Fa "==================" /var/log/clickhouse-server/stderr.log | rg -v "in query:" >> /test_output/tmp
|
||||
rg -Fa "WARNING" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
|
||||
rg -Fav -e "ASan doesn't fully support makecontext/swapcontext functions" -e "DB::Exception" /test_output/tmp > /dev/null \
|
||||
&& echo -e 'Sanitizer assert (in stderr.log)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'No sanitizer asserts\tOK' >> /test_output/test_results.tsv
|
||||
&& echo -e "Sanitizer assert (in stderr.log)$FAIL$(head_escaped /test_output/tmp)" >> /test_output/test_results.tsv \
|
||||
|| echo -e "No sanitizer asserts$OK" >> /test_output/test_results.tsv
|
||||
rm -f /test_output/tmp
|
||||
|
||||
# OOM
|
||||
rg -Fa " <Fatal> Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server*.log > /dev/null \
|
||||
&& echo -e 'OOM killer (or signal 9) in clickhouse-server.log\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'No OOM messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
|
||||
&& echo -e "Signal 9 in clickhouse-server.log$FAIL" >> /test_output/test_results.tsv \
|
||||
|| echo -e "No OOM messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv
|
||||
|
||||
# Logical errors
|
||||
rg -Fa "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server*.log > /test_output/logical_errors.txt \
|
||||
&& echo -e 'Logical error thrown (see clickhouse-server.log or logical_errors.txt)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'No logical errors\tOK' >> /test_output/test_results.tsv
|
||||
rg -Fa "Code: 49. DB::Exception: " /var/log/clickhouse-server/clickhouse-server*.log > /test_output/logical_errors.txt \
|
||||
&& echo -e "Logical error thrown (see clickhouse-server.log or logical_errors.txt)$FAIL$(head_escaped /test_output/logical_errors.txt)" >> /test_output/test_results.tsv \
|
||||
|| echo -e "No logical errors$OK" >> /test_output/test_results.tsv
|
||||
|
||||
# Remove file logical_errors.txt if it's empty
|
||||
[ -s /test_output/logical_errors.txt ] || rm /test_output/logical_errors.txt
|
||||
|
||||
# No such key errors
|
||||
rg --text "Code: 499.*The specified key does not exist" /var/log/clickhouse-server/clickhouse-server*.log > /test_output/no_such_key_errors.txt \
|
||||
&& echo -e 'S3_ERROR No such key thrown (see clickhouse-server.log or no_such_key_errors.txt)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'No lost s3 keys\tOK' >> /test_output/test_results.tsv
|
||||
&& echo -e "S3_ERROR No such key thrown (see clickhouse-server.log or no_such_key_errors.txt)$FAIL$(head_escaped /test_output/no_such_key_errors.txt)" >> /test_output/test_results.tsv \
|
||||
|| echo -e "No lost s3 keys$OK" >> /test_output/test_results.tsv
|
||||
|
||||
# Remove file no_such_key_errors.txt if it's empty
|
||||
[ -s /test_output/no_such_key_errors.txt ] || rm /test_output/no_such_key_errors.txt
|
||||
|
||||
# Crash
|
||||
rg -Fa "########################################" /var/log/clickhouse-server/clickhouse-server*.log > /dev/null \
|
||||
&& echo -e 'Killed by signal (in clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Not crashed\tOK' >> /test_output/test_results.tsv
|
||||
&& echo -e "Killed by signal (in clickhouse-server.log)$FAIL" >> /test_output/test_results.tsv \
|
||||
|| echo -e "Not crashed$OK" >> /test_output/test_results.tsv
|
||||
|
||||
# It also checks for crash without stacktrace (printed by watchdog)
|
||||
rg -Fa " <Fatal> " /var/log/clickhouse-server/clickhouse-server*.log > /test_output/fatal_messages.txt \
|
||||
&& echo -e 'Fatal message in clickhouse-server.log (see fatal_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'No fatal messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
|
||||
&& echo -e "Fatal message in clickhouse-server.log (see fatal_messages.txt)$FAIL$(head_escaped /test_output/fatal_messages.txt)" >> /test_output/test_results.tsv \
|
||||
|| echo -e "No fatal messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv
|
||||
|
||||
# Remove file fatal_messages.txt if it's empty
|
||||
[ -s /test_output/fatal_messages.txt ] || rm /test_output/fatal_messages.txt
|
||||
|
||||
rg -Fa "########################################" /test_output/* > /dev/null \
|
||||
&& echo -e 'Killed by signal (output files)\tFAIL' >> /test_output/test_results.tsv
|
||||
&& echo -e "Killed by signal (output files)$FAIL" >> /test_output/test_results.tsv
|
||||
|
||||
rg -Fa " received signal " /test_output/gdb.log > /dev/null \
|
||||
&& echo -e 'Found signal in gdb.log\tFAIL' >> /test_output/test_results.tsv
|
||||
&& echo -e "Found signal in gdb.log$FAIL$(rg -A50 -Fa " received signal " /test_output/gdb.log | escaped)" >> /test_output/test_results.tsv
|
||||
|
||||
if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
|
||||
echo -e "Backward compatibility check\n"
|
||||
@ -367,8 +473,8 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
|
||||
echo "Download clickhouse-server from the previous release"
|
||||
mkdir previous_release_package_folder
|
||||
|
||||
echo $previous_release_tag | download_release_packages && echo -e 'Download script exit code\tOK' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Download script failed\tFAIL' >> /test_output/test_results.tsv
|
||||
echo $previous_release_tag | download_release_packages && echo -e "Download script exit code$OK" >> /test_output/test_results.tsv \
|
||||
|| echo -e "Download script failed$FAIL" >> /test_output/test_results.tsv
|
||||
|
||||
mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.clean.log
|
||||
for table in query_log trace_log
|
||||
@ -381,13 +487,13 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
|
||||
# Check if we cloned previous release repository successfully
|
||||
if ! [ "$(ls -A previous_release_repository/tests/queries)" ]
|
||||
then
|
||||
echo -e "Backward compatibility check: Failed to clone previous release tests\tFAIL" >> /test_output/test_results.tsv
|
||||
echo -e "Backward compatibility check: Failed to clone previous release tests$FAIL" >> /test_output/test_results.tsv
|
||||
elif ! [ "$(ls -A previous_release_package_folder/clickhouse-common-static_*.deb && ls -A previous_release_package_folder/clickhouse-server_*.deb)" ]
|
||||
then
|
||||
echo -e "Backward compatibility check: Failed to download previous release packages\tFAIL" >> /test_output/test_results.tsv
|
||||
echo -e "Backward compatibility check: Failed to download previous release packages$FAIL" >> /test_output/test_results.tsv
|
||||
else
|
||||
echo -e "Successfully cloned previous release tests\tOK" >> /test_output/test_results.tsv
|
||||
echo -e "Successfully downloaded previous release packages\tOK" >> /test_output/test_results.tsv
|
||||
echo -e "Successfully cloned previous release tests$OK" >> /test_output/test_results.tsv
|
||||
echo -e "Successfully downloaded previous release packages$OK" >> /test_output/test_results.tsv
|
||||
|
||||
# Uninstall current packages
|
||||
dpkg --remove clickhouse-client
|
||||
@ -446,9 +552,10 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
|
||||
|
||||
mkdir tmp_stress_output
|
||||
|
||||
./stress --test-cmd="/usr/bin/clickhouse-test --queries=\"previous_release_repository/tests/queries\"" --backward-compatibility-check --output-folder tmp_stress_output --global-time-limit=1200 \
|
||||
&& echo -e 'Backward compatibility check: Test script exit code\tOK' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: Test script failed\tFAIL' >> /test_output/test_results.tsv
|
||||
./stress --test-cmd="/usr/bin/clickhouse-test --queries=\"previous_release_repository/tests/queries\"" \
|
||||
--backward-compatibility-check --output-folder tmp_stress_output --global-time-limit=1200 \
|
||||
&& echo -e "Backward compatibility check: Test script exit code$OK" >> /test_output/test_results.tsv \
|
||||
|| echo -e "Backward compatibility check: Test script failed$FAIL" >> /test_output/test_results.tsv
|
||||
rm -rf tmp_stress_output
|
||||
|
||||
# We experienced deadlocks in this command in very rare cases. Let's debug it:
|
||||
@ -470,9 +577,9 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
|
||||
export ZOOKEEPER_FAULT_INJECTION=0
|
||||
configure
|
||||
start 500
|
||||
clickhouse-client --query "SELECT 'Backward compatibility check: Server successfully started', 'OK'" >> /test_output/test_results.tsv \
|
||||
|| (echo -e 'Backward compatibility check: Server failed to start\tFAIL' >> /test_output/test_results.tsv \
|
||||
&& rg --text "<Error>.*Application" /var/log/clickhouse-server/clickhouse-server.log >> /test_output/bc_check_application_errors.txt)
|
||||
clickhouse-client --query "SELECT 'Backward compatibility check: Server successfully started', 'OK', NULL, ''" >> /test_output/test_results.tsv \
|
||||
|| (rg --text "<Error>.*Application" /var/log/clickhouse-server/clickhouse-server.log >> /test_output/bc_check_application_errors.txt \
|
||||
&& echo -e "Backward compatibility check: Server failed to start$FAIL$(head_escaped /test_output/bc_check_application_errors.txt)" >> /test_output/test_results.tsv)
|
||||
|
||||
clickhouse-client --query="SELECT 'Server version: ', version()"
|
||||
|
||||
@ -488,8 +595,6 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
|
||||
# FIXME Not sure if it's expected, but some tests from BC check may not be finished yet when we restarting server.
|
||||
# Let's just ignore all errors from queries ("} <Error> TCPHandler: Code:", "} <Error> executeQuery: Code:")
|
||||
# FIXME https://github.com/ClickHouse/ClickHouse/issues/39197 ("Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'")
|
||||
# NOTE Incompatibility was introduced in https://github.com/ClickHouse/ClickHouse/pull/39263, it's expected
|
||||
# ("This engine is deprecated and is not supported in transactions", "[Queue = DB::MergeMutateRuntimeQueue]: Code: 235. DB::Exception: Part")
|
||||
# FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 - bad mutation does not indicate backward incompatibility
|
||||
echo "Check for Error messages in server log:"
|
||||
rg -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \
|
||||
@ -519,7 +624,6 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
|
||||
-e "} <Error> TCPHandler: Code:" \
|
||||
-e "} <Error> executeQuery: Code:" \
|
||||
-e "Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'" \
|
||||
-e "This engine is deprecated and is not supported in transactions" \
|
||||
-e "[Queue = DB::MergeMutateRuntimeQueue]: Code: 235. DB::Exception: Part" \
|
||||
-e "The set of parts restored in place of" \
|
||||
-e "(ReplicatedMergeTreeAttachThread): Initialization failed. Error" \
|
||||
@ -530,8 +634,9 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
|
||||
-e "Session expired" \
|
||||
-e "TOO_MANY_PARTS" \
|
||||
/var/log/clickhouse-server/clickhouse-server.backward.dirty.log | rg -Fa "<Error>" > /test_output/bc_check_error_messages.txt \
|
||||
&& echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
|
||||
&& echo -e "Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)$FAIL$(head_escaped /test_output/bc_check_error_messages.txt)" \
|
||||
>> /test_output/test_results.tsv \
|
||||
|| echo -e "Backward compatibility check: No Error messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv
|
||||
|
||||
# Remove file bc_check_error_messages.txt if it's empty
|
||||
[ -s /test_output/bc_check_error_messages.txt ] || rm /test_output/bc_check_error_messages.txt
|
||||
@ -540,34 +645,36 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
|
||||
rg -Fa "==================" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
|
||||
rg -Fa "WARNING" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
|
||||
rg -Fav -e "ASan doesn't fully support makecontext/swapcontext functions" -e "DB::Exception" /test_output/tmp > /dev/null \
|
||||
&& echo -e 'Backward compatibility check: Sanitizer assert (in stderr.log)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: No sanitizer asserts\tOK' >> /test_output/test_results.tsv
|
||||
&& echo -e "Backward compatibility check: Sanitizer assert (in stderr.log)$FAIL$(head_escaped /test_output/tmp)" >> /test_output/test_results.tsv \
|
||||
|| echo -e "Backward compatibility check: No sanitizer asserts$OK" >> /test_output/test_results.tsv
|
||||
rm -f /test_output/tmp
|
||||
|
||||
# OOM
|
||||
rg -Fa " <Fatal> Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /dev/null \
|
||||
&& echo -e 'Backward compatibility check: OOM killer (or signal 9) in clickhouse-server.log\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: No OOM messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
|
||||
&& echo -e "Backward compatibility check: Signal 9 in clickhouse-server.log$FAIL" >> /test_output/test_results.tsv \
|
||||
|| echo -e "Backward compatibility check: No OOM messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv
|
||||
|
||||
# Logical errors
|
||||
echo "Check for Logical errors in server log:"
|
||||
rg -Fa -A20 "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_logical_errors.txt \
|
||||
&& echo -e 'Backward compatibility check: Logical error thrown (see clickhouse-server.log or bc_check_logical_errors.txt)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: No logical errors\tOK' >> /test_output/test_results.tsv
|
||||
rg -Fa -A20 "Code: 49. DB::Exception:" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_logical_errors.txt \
|
||||
&& echo -e "Backward compatibility check: Logical error thrown (see clickhouse-server.log or bc_check_logical_errors.txt)$FAIL$(head_escaped /test_output/bc_check_logical_errors.txt)" \
|
||||
>> /test_output/test_results.tsv \
|
||||
|| echo -e "Backward compatibility check: No logical errors$OK" >> /test_output/test_results.tsv
|
||||
|
||||
# Remove file bc_check_logical_errors.txt if it's empty
|
||||
[ -s /test_output/bc_check_logical_errors.txt ] || rm /test_output/bc_check_logical_errors.txt
|
||||
|
||||
# Crash
|
||||
rg -Fa "########################################" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /dev/null \
|
||||
&& echo -e 'Backward compatibility check: Killed by signal (in clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: Not crashed\tOK' >> /test_output/test_results.tsv
|
||||
&& echo -e "Backward compatibility check: Killed by signal (in clickhouse-server.log)$FAIL" >> /test_output/test_results.tsv \
|
||||
|| echo -e "Backward compatibility check: Not crashed$OK" >> /test_output/test_results.tsv
|
||||
|
||||
# It also checks for crash without stacktrace (printed by watchdog)
|
||||
echo "Check for Fatal message in server log:"
|
||||
rg -Fa " <Fatal> " /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_fatal_messages.txt \
|
||||
&& echo -e 'Backward compatibility check: Fatal message in clickhouse-server.log (see bc_check_fatal_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: No fatal messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
|
||||
&& echo -e "Backward compatibility check: Fatal message in clickhouse-server.log (see bc_check_fatal_messages.txt)$FAIL$(head_escaped /test_output/bc_check_fatal_messages.txt)" \
|
||||
>> /test_output/test_results.tsv \
|
||||
|| echo -e "Backward compatibility check: No fatal messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv
|
||||
|
||||
# Remove file bc_check_fatal_messages.txt if it's empty
|
||||
[ -s /test_output/bc_check_fatal_messages.txt ] || rm /test_output/bc_check_fatal_messages.txt
|
||||
@ -575,7 +682,8 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
|
||||
tar -chf /test_output/coordination.backward.tar /var/lib/clickhouse/coordination ||:
|
||||
for table in query_log trace_log
|
||||
do
|
||||
clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.backward.tsv.zst ||:
|
||||
clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" \
|
||||
| zstd --threads=0 > /test_output/$table.backward.tsv.zst ||:
|
||||
done
|
||||
fi
|
||||
fi
|
||||
@ -584,13 +692,28 @@ dmesg -T > /test_output/dmesg.log
|
||||
|
||||
# OOM in dmesg -- those are real
|
||||
grep -q -F -e 'Out of memory: Killed process' -e 'oom_reaper: reaped process' -e 'oom-kill:constraint=CONSTRAINT_NONE' /test_output/dmesg.log \
|
||||
&& echo -e 'OOM in dmesg\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'No OOM in dmesg\tOK' >> /test_output/test_results.tsv
|
||||
&& echo -e "OOM in dmesg$FAIL$(head_escaped /test_output/dmesg.log)" >> /test_output/test_results.tsv \
|
||||
|| echo -e "No OOM in dmesg$OK" >> /test_output/test_results.tsv
|
||||
|
||||
mv /var/log/clickhouse-server/stderr.log /test_output/
|
||||
|
||||
# Write check result into check_status.tsv
|
||||
clickhouse-local --structure "test String, res String" -q "SELECT 'failure', test FROM table WHERE res != 'OK' order by (lower(test) like '%hung%'), rowNumberInAllBlocks() LIMIT 1" < /test_output/test_results.tsv > /test_output/check_status.tsv
|
||||
# Try to choose most specific error for the whole check status
|
||||
clickhouse-local --structure "test String, res String" -q "SELECT 'failure', test FROM table WHERE res != 'OK' order by
|
||||
(test like 'Backward compatibility check%'), -- BC check goes last
|
||||
(test like '%Sanitizer%') DESC,
|
||||
(test like '%Killed by signal%') DESC,
|
||||
(test like '%gdb.log%') DESC,
|
||||
(test ilike '%possible deadlock%') DESC,
|
||||
(test like '%start%') DESC,
|
||||
(test like '%dmesg%') DESC,
|
||||
(test like '%OOM%') DESC,
|
||||
(test like '%Signal 9%') DESC,
|
||||
(test like '%Fatal message%') DESC,
|
||||
(test like '%Error message%') DESC,
|
||||
(test like '%previous release%') DESC,
|
||||
rowNumberInAllBlocks()
|
||||
LIMIT 1" < /test_output/test_results.tsv > /test_output/check_status.tsv
|
||||
[ -s /test_output/check_status.tsv ] || echo -e "success\tNo errors found" > /test_output/check_status.tsv
|
||||
|
||||
# Core dumps
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
from multiprocessing import cpu_count
|
||||
from subprocess import Popen, call, check_output, STDOUT
|
||||
from subprocess import Popen, call, check_output, STDOUT, PIPE
|
||||
import os
|
||||
import argparse
|
||||
import logging
|
||||
@ -299,14 +299,19 @@ if __name__ == "__main__":
|
||||
"00001_select_1",
|
||||
]
|
||||
)
|
||||
res = call(cmd, shell=True, stderr=STDOUT)
|
||||
hung_check_status = "No queries hung\tOK\n"
|
||||
hung_check_log = os.path.join(args.output_folder, "hung_check.log")
|
||||
tee = Popen(['/usr/bin/tee', hung_check_log], stdin=PIPE)
|
||||
res = call(cmd, shell=True, stdout=tee.stdin, stderr=STDOUT)
|
||||
tee.stdin.close()
|
||||
if res != 0 and have_long_running_queries:
|
||||
logging.info("Hung check failed with exit code {}".format(res))
|
||||
hung_check_status = "Hung check failed\tFAIL\n"
|
||||
with open(
|
||||
os.path.join(args.output_folder, "test_results.tsv"), "w+"
|
||||
) as results:
|
||||
results.write(hung_check_status)
|
||||
else:
|
||||
hung_check_status = "No queries hung\tOK\t\\N\t\n"
|
||||
with open(
|
||||
os.path.join(args.output_folder, "test_results.tsv"), "w+"
|
||||
) as results:
|
||||
results.write(hung_check_status)
|
||||
os.remove(hung_check_log)
|
||||
|
||||
|
||||
logging.info("Stress test finished")
|
||||
|
||||
112
docs/en/operations/query-cache.md
Normal file
112
docs/en/operations/query-cache.md
Normal file
@ -0,0 +1,112 @@
|
||||
---
|
||||
slug: /en/operations/query-cache
|
||||
sidebar_position: 65
|
||||
sidebar_label: Query Cache [experimental]
|
||||
---
|
||||
|
||||
# Query Cache [experimental]
|
||||
|
||||
The query cache allows to compute `SELECT` queries just once and to serve further executions of the same query directly from the cache.
|
||||
Depending on the type of the queries, this can dramatically reduce latency and resource consumption of the ClickHouse server.
|
||||
|
||||
## Background, Design and Limitations
|
||||
|
||||
Query caches can generally be viewed as transactionally consistent or inconsistent.
|
||||
|
||||
- In transactionally consistent caches, the database invalidates (discards) cached query results if the result of the `SELECT` query changes
|
||||
or potentially changes. In ClickHouse, operations which change the data include inserts/updates/deletes in/of/from tables or collapsing
|
||||
merges. Transactionally consistent caching is especially suitable for OLTP databases, for example
|
||||
[MySQL](https://dev.mysql.com/doc/refman/5.6/en/query-cache.html) (which removed query cache after v8.0) and
|
||||
[Oracle](https://docs.oracle.com/database/121/TGDBA/tune_result_cache.htm).
|
||||
- In transactionally inconsistent caches, slight inaccuracies in query results are accepted under the assumption that all cache entries are
|
||||
assigned a validity period after which they expire (e.g. 1 minute) and that the underlying data changes only little during this period.
|
||||
This approach is overall more suitable for OLAP databases. As an example where transactionally inconsistent caching is sufficient,
|
||||
consider an hourly sales report in a reporting tool which is simultaneously accessed by multiple users. Sales data changes typically
|
||||
slowly enough that the database only needs to compute the report once (represented by the first `SELECT` query). Further queries can be
|
||||
served directly from the query cache. In this example, a reasonable validity period could be 30 min.
|
||||
|
||||
Transactionally inconsistent caching is traditionally provided by client tools or proxy packages interacting with the database. As a result,
|
||||
the same caching logic and configuration is often duplicated. With ClickHouse's query cache, the caching logic moves to the server side.
|
||||
This reduces maintenance effort and avoids redundancy.
|
||||
|
||||
:::warning
|
||||
The query cache is an experimental feature that should not be used in production. There are known cases (e.g. in distributed query
|
||||
processing) where wrong results are returned.
|
||||
:::
|
||||
|
||||
## Configuration Settings and Usage
|
||||
|
||||
As long as the result cache is experimental it must be activated using the following configuration setting:
|
||||
|
||||
```sql
|
||||
SET allow_experimental_query_cache = true;
|
||||
```
|
||||
|
||||
Afterwards, setting [use_query_cache](settings/settings.md#use-query-cache) can be used to control whether a specific query or all queries
|
||||
of the current session should utilize the query cache. For example, the first execution of query
|
||||
|
||||
```sql
|
||||
SELECT some_expensive_calculation(column_1, column_2)
|
||||
FROM table
|
||||
SETTINGS use_query_cache = true;
|
||||
```
|
||||
|
||||
will store the query result in the query cache. Subsequent executions of the same query (also with parameter `use_query_cache = true`) will
|
||||
read the computed result from the cache and return it immediately.
|
||||
|
||||
The way the cache is utilized can be configured in more detail using settings [enable_writes_to_query_cache](settings/settings.md#enable-writes-to-query-cache)
|
||||
and [enable_reads_from_query_cache](settings/settings.md#enable-reads-from-query-cache) (both `true` by default). The former setting
|
||||
controls whether query results are stored in the cache, whereas the latter setting determines if the database should try to retrieve query
|
||||
results from the cache. For example, the following query will use the cache only passively, i.e. attempt to read from it but not store its
|
||||
result in it:
|
||||
|
||||
```sql
|
||||
SELECT some_expensive_calculation(column_1, column_2)
|
||||
FROM table
|
||||
SETTINGS use_query_cache = true, enable_writes_to_query_cache = false;
|
||||
```
|
||||
|
||||
For maximum control, it is generally recommended to provide settings "use_query_cache", "enable_writes_to_query_cache" and
|
||||
"enable_reads_from_query_cache" only with specific queries. It is also possible to enable caching at user or profile level (e.g. via `SET
|
||||
use_query_cache = true`) but one should keep in mind that all `SELECT` queries including monitoring or debugging queries to system tables
|
||||
may return cached results then.
|
||||
|
||||
The query cache can be cleared using statement `SYSTEM DROP QUERY CACHE`. The content of the query cache is displayed in system table
|
||||
`system.query_cache`. The number of query cache hits and misses are shown as events "QueryCacheHits" and "QueryCacheMisses" in system table
|
||||
`system.events`. Both counters are only updated for `SELECT` queries which run with setting "use_query_cache = true". Other queries do not
|
||||
affect the cache miss counter.
|
||||
|
||||
The query cache exists once per ClickHouse server process. However, cache results are by default not shared between users. This can be
|
||||
changed (see below) but doing so is not recommended for security reasons.
|
||||
|
||||
Query results are referenced in the query cache by the [Abstract Syntax Tree (AST)](https://en.wikipedia.org/wiki/Abstract_syntax_tree) of
|
||||
their query. This means that caching is agnostic to upper/lowercase, for example `SELECT 1` and `select 1` are treated as the same query. To
|
||||
make the matching more natural, all query-level settings related to the query cache are removed from the AST.
|
||||
|
||||
If the query was aborted due to an exception or user cancellation, no entry is written into the query cache.
|
||||
|
||||
The size of the query cache, the maximum number of cache entries and the maximum size of cache entries (in bytes and in records) can
|
||||
be configured using different [server configuration options](server-configuration-parameters/settings.md#server_configuration_parameters_query-cache).
|
||||
|
||||
To define how long a query must run at least such that its result can be cached, you can use setting
|
||||
[query_cache_min_query_duration](settings/settings.md#query-cache-min-query-duration). For example, the result of query
|
||||
|
||||
``` sql
|
||||
SELECT some_expensive_calculation(column_1, column_2)
|
||||
FROM table
|
||||
SETTINGS use_query_cache = true, query_cache_min_query_duration = 5000;
|
||||
```
|
||||
|
||||
is only cached if the query runs longer than 5 seconds. It is also possible to specify how often a query needs to run until its result is
|
||||
cached - for that use setting [query_cache_min_query_runs](settings/settings.md#query-cache-min-query-runs).
|
||||
|
||||
Entries in the query cache become stale after a certain time period (time-to-live). By default, this period is 60 seconds but a different
|
||||
value can be specified at session, profile or query level using setting [query_cache_ttl](settings/settings.md#query-cache-ttl).
|
||||
|
||||
Also, results of queries with non-deterministic functions such as `rand()` and `now()` are not cached. This can be overruled using
|
||||
setting [query_cache_store_results_of_queries_with_nondeterministic_functions](settings/settings.md#query-cache-store-results-of-queries-with-nondeterministic-functions).
|
||||
|
||||
Finally, entries in the query cache are not shared between users due to security reasons. For example, user A must not be able to bypass a
|
||||
row policy on a table by running the same query as another user B for whom no such policy exists. However, if necessary, cache entries can
|
||||
be marked accessible by other users (i.e. shared) by supplying setting
|
||||
[query_cache_share_between_users](settings/settings.md#query-cache-share-between-users).
|
||||
@ -72,8 +72,8 @@ For maximum control, it is generally recommended to provide settings "use_query_
|
||||
system tables may return cached results then.
|
||||
|
||||
The query result cache can be cleared using statement `SYSTEM DROP QUERY RESULT CACHE`. The content of the query result cache is displayed
|
||||
in system table `SYSTEM.QUERY_RESULT_CACHE`. The number of query result cache hits and misses are shown as events "QueryResultCacheHits" and
|
||||
"QueryResultCacheMisses" in system table `SYSTEM.EVENTS`. Both counters are only updated for `SELECT` queries which run with setting
|
||||
in system table `SYSTEM.QUERY_RESULT_CACHE`. The number of query result cache hits and misses are shown as events "QueryCacheHits" and
|
||||
"QueryCacheMisses" in system table `SYSTEM.EVENTS`. Both counters are only updated for `SELECT` queries which run with setting
|
||||
"use_query_result_cache = true". Other queries do not affect the cache miss counter.
|
||||
|
||||
The query result cache exists once per ClickHouse server process. However, cache results are by default not shared between users. This can
|
||||
|
||||
@ -1270,30 +1270,30 @@ If the table does not exist, ClickHouse will create it. If the structure of the
|
||||
</query_log>
|
||||
```
|
||||
|
||||
## query_result_cache {#server_configuration_parameters_query-result-cache}
|
||||
## query_cache {#server_configuration_parameters_query-cache}
|
||||
|
||||
[Query result cache](../query-result-cache.md) configuration.
|
||||
[Query cache](../query-cache.md) configuration.
|
||||
|
||||
The following settings are available:
|
||||
|
||||
- `size`: The maximum cache size in bytes. 0 means the query result cache is disabled. Default value: `1073741824` (1 GiB).
|
||||
- `max_entries`: The maximum number of SELECT query results stored in the cache. Default value: `1024`.
|
||||
- `max_entry_size`: The maximum size in bytes SELECT query results may have to be saved in the cache. Default value: `1048576` (1 MiB).
|
||||
- `max_entry_records`: The maximum number of records SELECT query results may have to be saved in the cache. Default value: `30000000` (30 mil).
|
||||
- `size`: The maximum cache size in bytes. 0 means the query cache is disabled. Default value: `1073741824` (1 GiB).
|
||||
- `max_entries`: The maximum number of `SELECT` query results stored in the cache. Default value: `1024`.
|
||||
- `max_entry_size`: The maximum size in bytes `SELECT` query results may have to be saved in the cache. Default value: `1048576` (1 MiB).
|
||||
- `max_entry_records`: The maximum number of records `SELECT` query results may have to be saved in the cache. Default value: `30000000` (30 mil).
|
||||
|
||||
:::warning
|
||||
Data for the query result cache is allocated in DRAM. If memory is scarce, make sure to set a small value for `size` or disable the query result cache altogether.
|
||||
Data for the query cache is allocated in DRAM. If memory is scarce, make sure to set a small value for `size` or disable the query cache altogether.
|
||||
:::
|
||||
|
||||
**Example**
|
||||
|
||||
```xml
|
||||
<query_result_cache>
|
||||
<query_cache>
|
||||
<size>1073741824</size>
|
||||
<max_entries>1024</max_entries>
|
||||
<max_entry_size>1048576</max_entry_size>
|
||||
<max_entry_records>30000000</max_entry_records>
|
||||
</query_result_cache>
|
||||
</query_cache>
|
||||
```
|
||||
|
||||
## query_thread_log {#server_configuration_parameters-query_thread_log}
|
||||
|
||||
@ -233,7 +233,7 @@ Possible values:
|
||||
|
||||
Default value: 100.
|
||||
|
||||
Normally, the `use_async_block_ids_cache` updates as soon as there are updates in the watching keeper path. However, the cache updates might be too frequent and become a heavy burden. This minimum interval prevents the cache from updating too fast. Note that if we set this value too long, the block with duplicated inserts will have a longer retry time.
|
||||
Normally, the `use_async_block_ids_cache` updates as soon as there are updates in the watching keeper path. However, the cache updates might be too frequent and become a heavy burden. This minimum interval prevents the cache from updating too fast. Note that if we set this value too long, the block with duplicated inserts will have a longer retry time.
|
||||
|
||||
## max_replicated_logs_to_keep
|
||||
|
||||
|
||||
@ -1301,10 +1301,10 @@ Possible values:
|
||||
|
||||
Default value: `3`.
|
||||
|
||||
## use_query_result_cache {#use-query-result-cache}
|
||||
## use_query_cache {#use-query-cache}
|
||||
|
||||
If turned on, SELECT queries may utilize the [query result cache](../query-result-cache.md). Parameters [enable_reads_from_query_result_cache](#enable-reads-from-query-result-cache)
|
||||
and [enable_writes_to_query_result_cache](#enable-writes-to-query-result-cache) control in more detail how the cache is used.
|
||||
If turned on, `SELECT` queries may utilize the [query cache](../query-cache.md). Parameters [enable_reads_from_query_cache](#enable-readsfrom-query-cache)
|
||||
and [enable_writes_to_query_cache](#enable-writes-to-query-cache) control in more detail how the cache is used.
|
||||
|
||||
Possible values:
|
||||
|
||||
@ -1313,9 +1313,9 @@ Possible values:
|
||||
|
||||
Default value: `0`.
|
||||
|
||||
## enable_reads_from_query_result_cache {#enable-reads-from-query-result-cache}
|
||||
## enable_reads_from_query_cache {#enable-reads-from-query-cache}
|
||||
|
||||
If turned on, results of SELECT queries are retrieved from the [query result cache](../query-result-cache.md).
|
||||
If turned on, results of `SELECT` queries are retrieved from the [query cache](../query-cache.md).
|
||||
|
||||
Possible values:
|
||||
|
||||
@ -1324,9 +1324,9 @@ Possible values:
|
||||
|
||||
Default value: `1`.
|
||||
|
||||
## enable_writes_to_query_result_cache {#enable-writes-to-query-result-cache}
|
||||
## enable_writes_to_query_cache {#enable-writes-to-query-cache}
|
||||
|
||||
If turned on, results of SELECT queries are stored in the [query result cache](../query-result-cache.md).
|
||||
If turned on, results of `SELECT` queries are stored in the [query cache](../query-cache.md).
|
||||
|
||||
Possible values:
|
||||
|
||||
@ -1335,9 +1335,9 @@ Possible values:
|
||||
|
||||
Default value: `1`.
|
||||
|
||||
## query_result_cache_store_results_of_queries_with_nondeterministic_functions {#query-result-cache-store-results-of-queries-with-nondeterministic-functions}
|
||||
## query_cache_store_results_of_queries_with_nondeterministic_functions {#query--store-results-of-queries-with-nondeterministic-functions}
|
||||
|
||||
If turned on, then results of SELECT queries with non-deterministic functions (e.g. `rand()`, `now()`) can be cached in the [query result cache](../query-result-cache.md).
|
||||
If turned on, then results of `SELECT` queries with non-deterministic functions (e.g. `rand()`, `now()`) can be cached in the [query cache](../query-cache.md).
|
||||
|
||||
Possible values:
|
||||
|
||||
@ -1346,9 +1346,9 @@ Possible values:
|
||||
|
||||
Default value: `0`.
|
||||
|
||||
## query_result_cache_min_query_runs {#query-result-cache-min-query-runs}
|
||||
## query_cache_min_query_runs {#query-cache-min-query-runs}
|
||||
|
||||
Minimum number of times a SELECT query must run before its result is stored in the [query result cache](../query-result-cache.md).
|
||||
Minimum number of times a `SELECT` query must run before its result is stored in the [query cache](../query-cache.md).
|
||||
|
||||
Possible values:
|
||||
|
||||
@ -1356,9 +1356,9 @@ Possible values:
|
||||
|
||||
Default value: `0`
|
||||
|
||||
## query_result_cache_min_query_duration {#query-result-cache-min-query-duration}
|
||||
## query_cache_min_query_duration {#query-cache-min-query-duration}
|
||||
|
||||
Minimum duration in milliseconds a query needs to run for its result to be stored in the [query result cache](../query-result-cache.md).
|
||||
Minimum duration in milliseconds a query needs to run for its result to be stored in the [query cache](../query-cache.md).
|
||||
|
||||
Possible values:
|
||||
|
||||
@ -1366,9 +1366,9 @@ Possible values:
|
||||
|
||||
Default value: `0`
|
||||
|
||||
## query_result_cache_ttl {#query-result-cache-ttl}
|
||||
## query_cache_ttl {#query-cache-ttl}
|
||||
|
||||
After this time in seconds entries in the [query result cache](../query-result-cache.md) become stale.
|
||||
After this time in seconds entries in the [query cache](../query-cache.md) become stale.
|
||||
|
||||
Possible values:
|
||||
|
||||
@ -1376,9 +1376,9 @@ Possible values:
|
||||
|
||||
Default value: `60`
|
||||
|
||||
## query_result_cache_share_between_users {#query-result-cache-share-between-users}
|
||||
## query_cache_share_between_users {#query-cache-share-between-users}
|
||||
|
||||
If turned on, the result of SELECT queries cached in the [query result cache](../query-result-cache.md) can be read by other users.
|
||||
If turned on, the result of `SELECT` queries cached in the [query cache](../query-cache.md) can be read by other users.
|
||||
It is not recommended to enable this setting due to security reasons.
|
||||
|
||||
Possible values:
|
||||
@ -3689,6 +3689,30 @@ Default value: `0`.
|
||||
|
||||
- [optimize_move_to_prewhere](#optimize_move_to_prewhere) setting
|
||||
|
||||
## optimize_using_constraints
|
||||
|
||||
Use [constraints](../../sql-reference/statements/create/table#constraints) for query optimization. The default is `false`.
|
||||
|
||||
Possible values:
|
||||
|
||||
- true, false
|
||||
|
||||
## optimize_append_index
|
||||
|
||||
Use [constraints](../../sql-reference/statements/create/table#constraints) in order to append index condition. The default is `false`.
|
||||
|
||||
Possible values:
|
||||
|
||||
- true, false
|
||||
|
||||
## optimize_substitute_columns
|
||||
|
||||
Use [constraints](../../sql-reference/statements/create/table#constraints) for column substitution. The default is `false`.
|
||||
|
||||
Possible values:
|
||||
|
||||
- true, false
|
||||
|
||||
## describe_include_subcolumns {#describe_include_subcolumns}
|
||||
|
||||
Enables describing subcolumns for a [DESCRIBE](../../sql-reference/statements/describe-table.md) query. For example, members of a [Tuple](../../sql-reference/data-types/tuple.md) or subcolumns of a [Map](../../sql-reference/data-types/map.md/#map-subcolumns), [Nullable](../../sql-reference/data-types/nullable.md/#finding-null) or an [Array](../../sql-reference/data-types/array.md/#array-size) data type.
|
||||
|
||||
@ -5,7 +5,7 @@ sidebar_label: Storing Dictionaries in Memory
|
||||
---
|
||||
import CloudDetails from '@site/docs/en/sql-reference/dictionaries/external-dictionaries/_snippet_dictionary_in_cloud.md';
|
||||
|
||||
# Storing Dictionaries in Memory
|
||||
# Storing Dictionaries in Memory
|
||||
|
||||
There are a variety of ways to store dictionaries in memory.
|
||||
|
||||
@ -25,7 +25,7 @@ ClickHouse generates an exception for errors with dictionaries. Examples of erro
|
||||
|
||||
You can view the list of dictionaries and their statuses in the [system.dictionaries](../../../operations/system-tables/dictionaries.md) table.
|
||||
|
||||
<CloudDetails />
|
||||
<CloudDetails />
|
||||
|
||||
The configuration looks like this:
|
||||
|
||||
@ -299,11 +299,11 @@ Example: The table contains discounts for each advertiser in the format:
|
||||
|
||||
To use a sample for date ranges, define the `range_min` and `range_max` elements in the [structure](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md). These elements must contain elements `name` and `type` (if `type` is not specified, the default type will be used - Date). `type` can be any numeric type (Date / DateTime / UInt64 / Int32 / others).
|
||||
|
||||
:::warning
|
||||
:::warning
|
||||
Values of `range_min` and `range_max` should fit in `Int64` type.
|
||||
:::
|
||||
|
||||
Example:
|
||||
Example:
|
||||
|
||||
``` xml
|
||||
<layout>
|
||||
@ -459,7 +459,7 @@ select dictGet('discounts_dict', 'amount', 1, toDate('2015-01-14')) res;
|
||||
│ 0.1 │ -- the only one range is matching: 2015-01-01 - Null
|
||||
└─────┘
|
||||
|
||||
select dictGet('discounts_dict', 'amount', 1, toDate('2015-01-16')) res;
|
||||
select dictGet('discounts_dict', 'amount', 1, toDate('2015-01-16')) res;
|
||||
┌─res─┐
|
||||
│ 0.2 │ -- two ranges are matching, range_min 2015-01-15 (0.2) is bigger than 2015-01-01 (0.1)
|
||||
└─────┘
|
||||
@ -496,7 +496,7 @@ select dictGet('discounts_dict', 'amount', 1, toDate('2015-01-14')) res;
|
||||
│ 0.1 │ -- the only one range is matching: 2015-01-01 - Null
|
||||
└─────┘
|
||||
|
||||
select dictGet('discounts_dict', 'amount', 1, toDate('2015-01-16')) res;
|
||||
select dictGet('discounts_dict', 'amount', 1, toDate('2015-01-16')) res;
|
||||
┌─res─┐
|
||||
│ 0.1 │ -- two ranges are matching, range_min 2015-01-01 (0.1) is less than 2015-01-15 (0.2)
|
||||
└─────┘
|
||||
@ -588,7 +588,7 @@ Set a large enough cache size. You need to experiment to select the number of ce
|
||||
3. Assess memory consumption using the `system.dictionaries` table.
|
||||
4. Increase or decrease the number of cells until the required memory consumption is reached.
|
||||
|
||||
:::warning
|
||||
:::warning
|
||||
Do not use ClickHouse as a source, because it is slow to process queries with random reads.
|
||||
:::
|
||||
|
||||
@ -660,25 +660,30 @@ This type of storage is for use with composite [keys](../../../sql-reference/dic
|
||||
|
||||
This type of storage is for mapping network prefixes (IP addresses) to metadata such as ASN.
|
||||
|
||||
Example: The table contains network prefixes and their corresponding AS number and country code:
|
||||
**Example**
|
||||
|
||||
``` text
|
||||
+-----------|-----|------+
|
||||
| prefix | asn | cca2 |
|
||||
+=================+=======+========+
|
||||
| 202.79.32.0/20 | 17501 | NP |
|
||||
+-----------|-----|------+
|
||||
| 2620:0:870::/48 | 3856 | US |
|
||||
+-----------|-----|------+
|
||||
| 2a02:6b8:1::/48 | 13238 | RU |
|
||||
+-----------|-----|------+
|
||||
| 2001:db8::/32 | 65536 | ZZ |
|
||||
+-----------|-----|------+
|
||||
Suppose we have a table in ClickHouse that contains our IP prefixes and mappings:
|
||||
|
||||
```sql
|
||||
CREATE TABLE my_ip_addresses (
|
||||
prefix String,
|
||||
asn UInt32,
|
||||
cca2 String
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PRIMARY KEY prefix;
|
||||
```
|
||||
|
||||
When using this type of layout, the structure must have a composite key.
|
||||
```sql
|
||||
INSERT INTO my_ip_addresses VALUES
|
||||
('202.79.32.0/20', 17501, 'NP'),
|
||||
('2620:0:870::/48', 3856, 'US'),
|
||||
('2a02:6b8:1::/48', 13238, 'RU'),
|
||||
('2001:db8::/32', 65536, 'ZZ')
|
||||
;
|
||||
```
|
||||
|
||||
Example:
|
||||
Let's define an `ip_trie` dictionary for this table. The `ip_trie` layout requires a composite key:
|
||||
|
||||
``` xml
|
||||
<structure>
|
||||
@ -712,26 +717,29 @@ Example:
|
||||
or
|
||||
|
||||
``` sql
|
||||
CREATE DICTIONARY somedict (
|
||||
CREATE DICTIONARY my_ip_trie_dictionary (
|
||||
prefix String,
|
||||
asn UInt32,
|
||||
cca2 String DEFAULT '??'
|
||||
)
|
||||
PRIMARY KEY prefix
|
||||
SOURCE(CLICKHOUSE(TABLE 'my_ip_addresses'))
|
||||
LAYOUT(IP_TRIE)
|
||||
LIFETIME(3600);
|
||||
```
|
||||
|
||||
The key must have only one String type attribute that contains an allowed IP prefix. Other types are not supported yet.
|
||||
The key must have only one `String` type attribute that contains an allowed IP prefix. Other types are not supported yet.
|
||||
|
||||
For queries, you must use the same functions (`dictGetT` with a tuple) as for dictionaries with composite keys:
|
||||
For queries, you must use the same functions (`dictGetT` with a tuple) as for dictionaries with composite keys. The syntax is:
|
||||
|
||||
``` sql
|
||||
dictGetT('dict_name', 'attr_name', tuple(ip))
|
||||
```
|
||||
|
||||
The function takes either `UInt32` for IPv4, or `FixedString(16)` for IPv6:
|
||||
The function takes either `UInt32` for IPv4, or `FixedString(16)` for IPv6. For example:
|
||||
|
||||
``` sql
|
||||
dictGetString('prefix', 'asn', tuple(IPv6StringToNum('2001:db8::1')))
|
||||
select dictGet('my_ip_trie_dictionary', 'asn', tuple(IPv6StringToNum('2001:db8::1')))
|
||||
```
|
||||
|
||||
Other types are not supported yet. The function returns the attribute for the prefix that corresponds to this IP address. If there are overlapping prefixes, the most specific one is returned.
|
||||
|
||||
@ -304,7 +304,7 @@ Result:
|
||||
└──────────────┘
|
||||
```
|
||||
|
||||
## s2RectUinion
|
||||
## s2RectUnion
|
||||
|
||||
Returns the smallest rectangle containing the union of this rectangle and the given rectangle. In the S2 system, a rectangle is represented by a type of S2Region called a `S2LatLngRect` that represents a rectangle in latitude-longitude space.
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@ sidebar_label: DELETE
|
||||
# DELETE Statement
|
||||
|
||||
``` sql
|
||||
DELETE FROM [db.]table [WHERE expr]
|
||||
DELETE FROM [db.]table [ON CLUSTER cluster] [WHERE expr]
|
||||
```
|
||||
|
||||
`DELETE FROM` removes rows from table `[db.]table` that match expression `expr`. The deleted rows are marked as deleted immediately and will be automatically filtered out of all subsequent queries. Cleanup of data happens asynchronously in background. This feature is only available for MergeTree table engine family.
|
||||
|
||||
@ -103,9 +103,9 @@ Its size can be configured using the server-level setting [uncompressed_cache_si
|
||||
Reset the compiled expression cache.
|
||||
The compiled expression cache is enabled/disabled with the query/user/profile-level setting [compile_expressions](../../operations/settings/settings.md#compile-expressions).
|
||||
|
||||
## DROP QUERY RESULT CACHE
|
||||
## DROP QUERY CACHE
|
||||
|
||||
Resets the [query result cache](../../operations/query-result-cache.md).
|
||||
Resets the [query cache](../../operations/query-cache.md).
|
||||
|
||||
## FLUSH LOGS
|
||||
|
||||
|
||||
@ -6,7 +6,6 @@
|
||||
#include <Common/ZooKeeper/ZooKeeper.h>
|
||||
#include <Common/ZooKeeper/KeeperException.h>
|
||||
#include <Common/setThreadName.h>
|
||||
#include <IO/ConnectionTimeoutsContext.h>
|
||||
#include <Interpreters/InterpreterInsertQuery.h>
|
||||
#include <Interpreters/InterpreterSelectWithUnionQuery.h>
|
||||
#include <Parsers/ASTFunction.h>
|
||||
|
||||
@ -1517,13 +1517,13 @@ try
|
||||
global_context->setMMappedFileCache(mmap_cache_size);
|
||||
|
||||
/// A cache for query results.
|
||||
size_t query_result_cache_size = config().getUInt64("query_result_cache.size", 1_GiB);
|
||||
if (query_result_cache_size)
|
||||
global_context->setQueryResultCache(
|
||||
query_result_cache_size,
|
||||
config().getUInt64("query_result_cache.max_entries", 1024),
|
||||
config().getUInt64("query_result_cache.max_entry_size", 1_MiB),
|
||||
config().getUInt64("query_result_cache.max_entry_records", 30'000'000));
|
||||
size_t query_cache_size = config().getUInt64("query_cache.size", 1_GiB);
|
||||
if (query_cache_size)
|
||||
global_context->setQueryCache(
|
||||
query_cache_size,
|
||||
config().getUInt64("query_cache.max_entries", 1024),
|
||||
config().getUInt64("query_cache.max_entry_size", 1_MiB),
|
||||
config().getUInt64("query_cache.max_entry_records", 30'000'000));
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
/// 128 MB
|
||||
|
||||
@ -1466,13 +1466,13 @@
|
||||
</rocksdb>
|
||||
-->
|
||||
|
||||
<!-- Configuration for the query result cache -->
|
||||
<!-- <query_result_cache> -->
|
||||
<!-- Configuration for the query cache -->
|
||||
<!-- <query_cache> -->
|
||||
<!-- <size>1073741824</size> -->
|
||||
<!-- <max_entries>1024</max_entries> -->
|
||||
<!-- <max_entry_size>1048576</max_entry_size> -->
|
||||
<!-- <max_entry_records>30000000</max_entry_records> -->
|
||||
<!-- </query_result_cache> -->
|
||||
<!-- </query_cache> -->
|
||||
|
||||
<!-- Uncomment if enable merge tree metadata cache -->
|
||||
<!--merge_tree_metadata_cache>
|
||||
|
||||
@ -142,7 +142,7 @@ enum class AccessType
|
||||
M(SYSTEM_DROP_MARK_CACHE, "SYSTEM DROP MARK, DROP MARK CACHE, DROP MARKS", GLOBAL, SYSTEM_DROP_CACHE) \
|
||||
M(SYSTEM_DROP_UNCOMPRESSED_CACHE, "SYSTEM DROP UNCOMPRESSED, DROP UNCOMPRESSED CACHE, DROP UNCOMPRESSED", GLOBAL, SYSTEM_DROP_CACHE) \
|
||||
M(SYSTEM_DROP_MMAP_CACHE, "SYSTEM DROP MMAP, DROP MMAP CACHE, DROP MMAP", GLOBAL, SYSTEM_DROP_CACHE) \
|
||||
M(SYSTEM_DROP_QUERY_RESULT_CACHE, "SYSTEM DROP QUERY RESULT, DROP QUERY RESULT CACHE, DROP QUERY RESULT", GLOBAL, SYSTEM_DROP_CACHE) \
|
||||
M(SYSTEM_DROP_QUERY_CACHE, "SYSTEM DROP QUERY, DROP QUERY CACHE, DROP QUERY", GLOBAL, SYSTEM_DROP_CACHE) \
|
||||
M(SYSTEM_DROP_COMPILED_EXPRESSION_CACHE, "SYSTEM DROP COMPILED EXPRESSION, DROP COMPILED EXPRESSION CACHE, DROP COMPILED EXPRESSIONS", GLOBAL, SYSTEM_DROP_CACHE) \
|
||||
M(SYSTEM_DROP_FILESYSTEM_CACHE, "SYSTEM DROP FILESYSTEM CACHE, DROP FILESYSTEM CACHE", GLOBAL, SYSTEM_DROP_CACHE) \
|
||||
M(SYSTEM_DROP_SCHEMA_CACHE, "SYSTEM DROP SCHEMA CACHE, DROP SCHEMA CACHE", GLOBAL, SYSTEM_DROP_CACHE) \
|
||||
|
||||
@ -1,14 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
#include <string_view>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <AggregateFunctions/IAggregateFunction.h>
|
||||
#include <base/range.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Common/PODArray.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
#include <Common/HashTable/HashMap.h>
|
||||
#include <Columns/IColumn.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -105,24 +110,12 @@ private:
|
||||
bool specified_min_max_x;
|
||||
|
||||
template <class T>
|
||||
String getBar(const T value) const
|
||||
size_t updateFrame(ColumnString::Chars & frame, const T value) const
|
||||
{
|
||||
if (isNaN(value) || value > 8 || value < 1)
|
||||
return " ";
|
||||
|
||||
// ▁▂▃▄▅▆▇█
|
||||
switch (static_cast<UInt8>(value))
|
||||
{
|
||||
case 1: return "▁";
|
||||
case 2: return "▂";
|
||||
case 3: return "▃";
|
||||
case 4: return "▄";
|
||||
case 5: return "▅";
|
||||
case 6: return "▆";
|
||||
case 7: return "▇";
|
||||
case 8: return "█";
|
||||
}
|
||||
return " ";
|
||||
static constexpr std::array<std::string_view, 9> bars{" ", "▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"};
|
||||
const auto & bar = (isNaN(value) || value > 8 || value < 1) ? bars[0] : bars[static_cast<UInt8>(value)];
|
||||
frame.insert(bar.begin(), bar.end());
|
||||
return bar.size();
|
||||
}
|
||||
|
||||
/**
|
||||
@ -136,11 +129,19 @@ private:
|
||||
* the actual y value of the first position + the actual second position y*0.1, and the remaining y*0.9 is reserved for the next bucket.
|
||||
* The next bucket will use the last y*0.9 + the actual third position y*0.2, and the remaining y*0.8 will be reserved for the next bucket. And so on.
|
||||
*/
|
||||
String render(const AggregateFunctionSparkbarData<X, Y> & data) const
|
||||
void render(ColumnString & to_column, const AggregateFunctionSparkbarData<X, Y> & data) const
|
||||
{
|
||||
String value;
|
||||
size_t sz = 0;
|
||||
auto & values = to_column.getChars();
|
||||
auto & offsets = to_column.getOffsets();
|
||||
auto update_column = [&] ()
|
||||
{
|
||||
values.push_back('\0');
|
||||
offsets.push_back(offsets.empty() ? sz + 1 : offsets.back() + sz + 1);
|
||||
};
|
||||
|
||||
if (data.points.empty() || !width)
|
||||
return value;
|
||||
return update_column();
|
||||
|
||||
size_t diff_x;
|
||||
X min_x_local;
|
||||
@ -167,13 +168,13 @@ private:
|
||||
{
|
||||
auto it = data.points.find(static_cast<X>(min_x_local + i));
|
||||
bool found = it != data.points.end();
|
||||
value += getBar(found ? std::round(((it->getMapped() - min_y) / diff_y) * 7) + 1 : 0.0);
|
||||
sz += updateFrame(values, found ? std::round(((it->getMapped() - min_y) / diff_y) * 7) + 1 : 0.0);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for (size_t i = 0; i <= diff_x; ++i)
|
||||
value += getBar(data.points.has(min_x_local + static_cast<X>(i)) ? 1 : 0);
|
||||
sz += updateFrame(values, data.points.has(min_x_local + static_cast<X>(i)) ? 1 : 0);
|
||||
}
|
||||
}
|
||||
else
|
||||
@ -236,25 +237,25 @@ private:
|
||||
}
|
||||
|
||||
if (!min_y || !max_y) // No value is set
|
||||
return {};
|
||||
return update_column();
|
||||
|
||||
Float64 diff_y = max_y.value() - min_y.value();
|
||||
|
||||
auto get_bars = [&] (const std::optional<Float64> & point_y)
|
||||
auto update_frame = [&] (const std::optional<Float64> & point_y)
|
||||
{
|
||||
value += getBar(point_y ? std::round(((point_y.value() - min_y.value()) / diff_y) * 7) + 1 : 0);
|
||||
sz += updateFrame(values, point_y ? std::round(((point_y.value() - min_y.value()) / diff_y) * 7) + 1 : 0);
|
||||
};
|
||||
auto get_bars_for_constant = [&] (const std::optional<Float64> & point_y)
|
||||
auto update_frame_for_constant = [&] (const std::optional<Float64> & point_y)
|
||||
{
|
||||
value += getBar(point_y ? 1 : 0);
|
||||
sz += updateFrame(values, point_y ? 1 : 0);
|
||||
};
|
||||
|
||||
if (diff_y != 0.0)
|
||||
std::for_each(new_points.begin(), new_points.end(), get_bars);
|
||||
std::for_each(new_points.begin(), new_points.end(), update_frame);
|
||||
else
|
||||
std::for_each(new_points.begin(), new_points.end(), get_bars_for_constant);
|
||||
std::for_each(new_points.begin(), new_points.end(), update_frame_for_constant);
|
||||
}
|
||||
return value;
|
||||
update_column();
|
||||
}
|
||||
|
||||
|
||||
@ -314,8 +315,7 @@ public:
|
||||
{
|
||||
auto & to_column = assert_cast<ColumnString &>(to);
|
||||
const auto & data = this->data(place);
|
||||
const String & value = render(data);
|
||||
to_column.insertData(value.data(), value.size());
|
||||
render(to_column, data);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -313,7 +313,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_q
|
||||
if (select_limit_by_limit)
|
||||
current_query_tree->getLimitByLimit() = buildExpression(select_limit_by_limit, current_context);
|
||||
|
||||
auto select_limit_by_offset = select_query_typed.limitOffset();
|
||||
auto select_limit_by_offset = select_query_typed.limitByOffset();
|
||||
if (select_limit_by_offset)
|
||||
current_query_tree->getLimitByOffset() = buildExpression(select_limit_by_offset, current_context);
|
||||
|
||||
|
||||
@ -6,7 +6,6 @@
|
||||
#include <Poco/Net/HTTPRequest.h>
|
||||
#include <Common/ShellCommand.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <IO/ConnectionTimeoutsContext.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
#include "LibraryBridgeHelper.h"
|
||||
|
||||
#include <IO/ConnectionTimeoutsContext.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
|
||||
@ -53,8 +53,8 @@
|
||||
M(TableFunctionExecute, "Number of table function calls.") \
|
||||
M(MarkCacheHits, "Number of times an entry has been found in the mark cache, so we didn't have to load a mark file.") \
|
||||
M(MarkCacheMisses, "Number of times an entry has not been found in the mark cache, so we had to load a mark file in memory, which is a costly operation, adding to query latency.") \
|
||||
M(QueryResultCacheHits, "Number of times a query result has been found in the query result cache (and query computation was avoided).") \
|
||||
M(QueryResultCacheMisses, "Number of times a query result has not been found in the query result cache (and required query computation).") \
|
||||
M(QueryCacheHits, "Number of times a query result has been found in the query cache (and query computation was avoided).") \
|
||||
M(QueryCacheMisses, "Number of times a query result has not been found in the query cache (and required query computation).") \
|
||||
M(CreatedReadBufferOrdinary, "Number of times ordinary read buffer was created for reading data (while choosing among other read methods).") \
|
||||
M(CreatedReadBufferDirectIO, "Number of times a read buffer with O_DIRECT was created for reading data (while choosing among other read methods).") \
|
||||
M(CreatedReadBufferDirectIOFailed, "Number of times a read buffer with O_DIRECT was attempted to be created for reading data (while choosing among other read methods), but the OS did not allow it (due to lack of filesystem support or other reasons) and we fallen back to the ordinary reading method.") \
|
||||
|
||||
@ -194,6 +194,9 @@ inline bool parseIPv6(T * &src, EOFfunction eof, unsigned char * dst, int32_t fi
|
||||
if (groups <= 1 && zptr == nullptr) /// IPv4 block can't be the first
|
||||
return clear_dst();
|
||||
|
||||
if (group_start) /// first octet of IPv4 should be already parsed as an IPv6 group
|
||||
return clear_dst();
|
||||
|
||||
++src;
|
||||
if (eof())
|
||||
return clear_dst();
|
||||
|
||||
@ -157,21 +157,24 @@ public:
|
||||
assert(file_buffer && current_file_description);
|
||||
|
||||
assert(record.header.index - getStartIndex() <= current_file_description->expectedEntriesCountInLog());
|
||||
const bool log_is_complete = record.header.index - getStartIndex() == current_file_description->expectedEntriesCountInLog();
|
||||
|
||||
if (log_is_complete)
|
||||
rotate(record.header.index);
|
||||
|
||||
// writing at least 1 log is requirement - we don't want empty log files
|
||||
// we use count() that can be unreliable for more complex WriteBuffers, so we should be careful if we change the type of it in the future
|
||||
const bool log_too_big = record.header.index != getStartIndex() && log_file_settings.max_size != 0
|
||||
&& initial_file_size + file_buffer->count() > log_file_settings.max_size;
|
||||
|
||||
if (log_too_big)
|
||||
// check if log file reached the limit for amount of records it can contain
|
||||
if (record.header.index - getStartIndex() == current_file_description->expectedEntriesCountInLog())
|
||||
{
|
||||
LOG_TRACE(log, "Log file reached maximum allowed size ({} bytes), creating new log file", log_file_settings.max_size);
|
||||
rotate(record.header.index);
|
||||
}
|
||||
else
|
||||
{
|
||||
// writing at least 1 log is requirement - we don't want empty log files
|
||||
// we use count() that can be unreliable for more complex WriteBuffers, so we should be careful if we change the type of it in the future
|
||||
const bool log_too_big = record.header.index != getStartIndex() && log_file_settings.max_size != 0
|
||||
&& initial_file_size + file_buffer->count() > log_file_settings.max_size;
|
||||
|
||||
if (log_too_big)
|
||||
{
|
||||
LOG_TRACE(log, "Log file reached maximum allowed size ({} bytes), creating new log file", log_file_settings.max_size);
|
||||
rotate(record.header.index);
|
||||
}
|
||||
}
|
||||
|
||||
if (!prealloc_done) [[unlikely]]
|
||||
{
|
||||
|
||||
@ -540,14 +540,14 @@ class IColumn;
|
||||
M(Bool, describe_extend_object_types, false, "Deduce concrete type of columns of type Object in DESCRIBE query", 0) \
|
||||
M(Bool, describe_include_subcolumns, false, "If true, subcolumns of all table columns will be included into result of DESCRIBE query", 0) \
|
||||
\
|
||||
M(Bool, use_query_result_cache, false, "Enable the query result cache", 0) \
|
||||
M(Bool, enable_writes_to_query_result_cache, true, "Enable storing results of SELECT queries in the query result cache", 0) \
|
||||
M(Bool, enable_reads_from_query_result_cache, true, "Enable reading results of SELECT queries from the query result cache", 0) \
|
||||
M(Bool, query_result_cache_store_results_of_queries_with_nondeterministic_functions, false, "Store results of queries with non-deterministic functions (e.g. rand(), now()) in the query result cache", 0) \
|
||||
M(UInt64, query_result_cache_min_query_runs, 0, "Minimum number a SELECT query must run before its result is stored in the query result cache", 0) \
|
||||
M(Milliseconds, query_result_cache_min_query_duration, 0, "Minimum time in milliseconds for a query to run for its result to be stored in the query result cache.", 0) \
|
||||
M(Seconds, query_result_cache_ttl, 60, "After this time in seconds entries in the query result cache become stale", 0) \
|
||||
M(Bool, query_result_cache_share_between_users, false, "Allow other users to read entry in the query result cache", 0) \
|
||||
M(Bool, use_query_cache, false, "Enable the query cache", 0) \
|
||||
M(Bool, enable_writes_to_query_cache, true, "Enable storing results of SELECT queries in the query cache", 0) \
|
||||
M(Bool, enable_reads_from_query_cache, true, "Enable reading results of SELECT queries from the query cache", 0) \
|
||||
M(Bool, query_cache_store_results_of_queries_with_nondeterministic_functions, false, "Store results of queries with non-deterministic functions (e.g. rand(), now()) in the query cache", 0) \
|
||||
M(UInt64, query_cache_min_query_runs, 0, "Minimum number a SELECT query must run before its result is stored in the query cache", 0) \
|
||||
M(Milliseconds, query_cache_min_query_duration, 0, "Minimum time in milliseconds for a query to run for its result to be stored in the query cache.", 0) \
|
||||
M(Seconds, query_cache_ttl, 60, "After this time in seconds entries in the query cache become stale", 0) \
|
||||
M(Bool, query_cache_share_between_users, false, "Allow other users to read entry in the query cache", 0) \
|
||||
\
|
||||
M(Bool, optimize_rewrite_sum_if_to_count_if, false, "Rewrite sumIf() and sum(if()) function countIf() function when logically equivalent", 0) \
|
||||
M(UInt64, insert_shard_id, 0, "If non zero, when insert into a distributed table, the data will be inserted into the shard `insert_shard_id` synchronously. Possible values range from 1 to `shards_number` of corresponding distributed table", 0) \
|
||||
@ -669,7 +669,7 @@ class IColumn;
|
||||
M(Bool, allow_experimental_nlp_functions, false, "Enable experimental functions for natural language processing.", 0) \
|
||||
M(Bool, allow_experimental_hash_functions, false, "Enable experimental hash functions (hashid, etc)", 0) \
|
||||
M(Bool, allow_experimental_object_type, false, "Allow Object and JSON data types", 0) \
|
||||
M(Bool, allow_experimental_query_result_cache, false, "Enable experimental query result cache", 0) \
|
||||
M(Bool, allow_experimental_query_cache, false, "Enable experimental query cache", 0) \
|
||||
M(String, insert_deduplication_token, "", "If not empty, used for duplicate detection instead of data digest", 0) \
|
||||
M(String, ann_index_select_query_params, "", "Parameters passed to ANN indexes in SELECT queries, the format is 'param1=x, param2=y, ...'", 0) \
|
||||
M(UInt64, max_limit_for_ann_queries, 1000000, "Maximum limit value for using ANN indexes is used to prevent memory overflow in search queries for indexes", 0) \
|
||||
|
||||
@ -4,7 +4,6 @@
|
||||
#include <base/sleep.h>
|
||||
#include <Core/Types.h>
|
||||
#include <IO/ReadWriteBufferFromHTTP.h>
|
||||
#include <IO/ConnectionTimeoutsContext.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
#include <IO/Operators.h>
|
||||
#include <thread>
|
||||
|
||||
@ -3,7 +3,6 @@
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Common/escapeForFileName.h>
|
||||
|
||||
#include <IO/ConnectionTimeoutsContext.h>
|
||||
#include <IO/ReadWriteBufferFromHTTP.h>
|
||||
#include <IO/SeekAvoidingReadBuffer.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
|
||||
@ -444,7 +444,19 @@ namespace
|
||||
|
||||
bool tryInferDate(std::string_view field)
|
||||
{
|
||||
if (field.empty())
|
||||
return false;
|
||||
|
||||
ReadBufferFromString buf(field);
|
||||
Float64 tmp_float;
|
||||
/// Check if it's just a number, and if so, don't try to infer Date from it,
|
||||
/// because we can interpret this number as a Date (for example 20000101 will be 2000-01-01)
|
||||
/// and it will lead to inferring Date instead of simple Int64/UInt64 in some cases.
|
||||
if (tryReadFloatText(tmp_float, buf) && buf.eof())
|
||||
return false;
|
||||
|
||||
buf.seek(0, SEEK_SET); /// Return position to the beginning
|
||||
|
||||
DayNum tmp;
|
||||
return tryReadDateText(tmp, buf) && buf.eof();
|
||||
}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
#include "Interpreters/Cache/QueryResultCache.h"
|
||||
#include "Interpreters/Cache/QueryCache.h"
|
||||
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Interpreters/Context.h>
|
||||
@ -16,8 +16,8 @@
|
||||
|
||||
namespace ProfileEvents
|
||||
{
|
||||
extern const Event QueryResultCacheHits;
|
||||
extern const Event QueryResultCacheMisses;
|
||||
extern const Event QueryCacheHits;
|
||||
extern const Event QueryCacheMisses;
|
||||
};
|
||||
|
||||
namespace DB
|
||||
@ -64,7 +64,7 @@ bool astContainsNonDeterministicFunctions(ASTPtr ast, ContextPtr context)
|
||||
namespace
|
||||
{
|
||||
|
||||
class RemoveQueryResultCacheSettingsMatcher
|
||||
class RemoveQueryCacheSettingsMatcher
|
||||
{
|
||||
public:
|
||||
struct Data {};
|
||||
@ -77,65 +77,65 @@ public:
|
||||
{
|
||||
chassert(!set_clause->is_standalone);
|
||||
|
||||
auto is_query_result_cache_related_setting = [](const auto & change)
|
||||
auto is_query_cache_related_setting = [](const auto & change)
|
||||
{
|
||||
return change.name == "allow_experimental_query_result_cache"
|
||||
|| change.name.starts_with("query_result_cache")
|
||||
|| change.name.ends_with("query_result_cache");
|
||||
return change.name == "allow_experimental_query_cache"
|
||||
|| change.name.starts_with("query_cache")
|
||||
|| change.name.ends_with("query_cache");
|
||||
};
|
||||
|
||||
std::erase_if(set_clause->changes, is_query_result_cache_related_setting);
|
||||
std::erase_if(set_clause->changes, is_query_cache_related_setting);
|
||||
}
|
||||
}
|
||||
|
||||
/// TODO further improve AST cleanup, e.g. remove SETTINGS clause completely if it is empty
|
||||
/// E.g. SELECT 1 SETTINGS use_query_result_cache = true
|
||||
/// E.g. SELECT 1 SETTINGS use_query_cache = true
|
||||
/// and SELECT 1;
|
||||
/// currently don't match.
|
||||
};
|
||||
|
||||
using RemoveQueryResultCacheSettingsVisitor = InDepthNodeVisitor<RemoveQueryResultCacheSettingsMatcher, true>;
|
||||
using RemoveQueryCacheSettingsVisitor = InDepthNodeVisitor<RemoveQueryCacheSettingsMatcher, true>;
|
||||
|
||||
/// Consider
|
||||
/// (1) SET use_query_result_cache = true;
|
||||
/// SELECT expensiveComputation(...) SETTINGS max_threads = 64, query_result_cache_ttl = 300;
|
||||
/// SET use_query_result_cache = false;
|
||||
/// (1) SET use_query_cache = true;
|
||||
/// SELECT expensiveComputation(...) SETTINGS max_threads = 64, query_cache_ttl = 300;
|
||||
/// SET use_query_cache = false;
|
||||
/// and
|
||||
/// (2) SELECT expensiveComputation(...) SETTINGS max_threads = 64, use_query_result_cache = true;
|
||||
/// (2) SELECT expensiveComputation(...) SETTINGS max_threads = 64, use_query_cache = true;
|
||||
///
|
||||
/// The SELECT queries in (1) and (2) are basically the same and the user expects that the second invocation is served from the query result
|
||||
/// The SELECT queries in (1) and (2) are basically the same and the user expects that the second invocation is served from the query
|
||||
/// cache. However, query results are indexed by their query ASTs and therefore no result will be found. Insert and retrieval behave overall
|
||||
/// more natural if settings related to the query result cache are erased from the AST key. Note that at this point the settings themselves
|
||||
/// more natural if settings related to the query cache are erased from the AST key. Note that at this point the settings themselves
|
||||
/// have been parsed already, they are not lost or discarded.
|
||||
ASTPtr removeQueryResultCacheSettings(ASTPtr ast)
|
||||
ASTPtr removeQueryCacheSettings(ASTPtr ast)
|
||||
{
|
||||
ASTPtr transformed_ast = ast->clone();
|
||||
|
||||
RemoveQueryResultCacheSettingsMatcher::Data visitor_data;
|
||||
RemoveQueryResultCacheSettingsVisitor(visitor_data).visit(transformed_ast);
|
||||
RemoveQueryCacheSettingsMatcher::Data visitor_data;
|
||||
RemoveQueryCacheSettingsVisitor(visitor_data).visit(transformed_ast);
|
||||
|
||||
return transformed_ast;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
QueryResultCache::Key::Key(
|
||||
QueryCache::Key::Key(
|
||||
ASTPtr ast_,
|
||||
Block header_, const std::optional<String> & username_,
|
||||
std::chrono::time_point<std::chrono::system_clock> expires_at_)
|
||||
: ast(removeQueryResultCacheSettings(ast_))
|
||||
: ast(removeQueryCacheSettings(ast_))
|
||||
, header(header_)
|
||||
, username(username_)
|
||||
, expires_at(expires_at_)
|
||||
{
|
||||
}
|
||||
|
||||
bool QueryResultCache::Key::operator==(const Key & other) const
|
||||
bool QueryCache::Key::operator==(const Key & other) const
|
||||
{
|
||||
return ast->getTreeHash() == other.ast->getTreeHash();
|
||||
}
|
||||
|
||||
String QueryResultCache::Key::queryStringFromAst() const
|
||||
String QueryCache::Key::queryStringFromAst() const
|
||||
{
|
||||
WriteBufferFromOwnString buf;
|
||||
IAST::FormatSettings format_settings(buf, /*one_line*/ true);
|
||||
@ -144,7 +144,7 @@ String QueryResultCache::Key::queryStringFromAst() const
|
||||
return buf.str();
|
||||
}
|
||||
|
||||
size_t QueryResultCache::KeyHasher::operator()(const Key & key) const
|
||||
size_t QueryCache::KeyHasher::operator()(const Key & key) const
|
||||
{
|
||||
SipHash hash;
|
||||
hash.update(key.ast->getTreeHash());
|
||||
@ -152,7 +152,7 @@ size_t QueryResultCache::KeyHasher::operator()(const Key & key) const
|
||||
return res;
|
||||
}
|
||||
|
||||
size_t QueryResultCache::QueryResult::sizeInBytes() const
|
||||
size_t QueryCache::QueryResult::sizeInBytes() const
|
||||
{
|
||||
size_t res = 0;
|
||||
for (const auto & chunk : *chunks)
|
||||
@ -163,14 +163,14 @@ size_t QueryResultCache::QueryResult::sizeInBytes() const
|
||||
namespace
|
||||
{
|
||||
|
||||
auto is_stale = [](const QueryResultCache::Key & key)
|
||||
auto is_stale = [](const QueryCache::Key & key)
|
||||
{
|
||||
return (key.expires_at < std::chrono::system_clock::now());
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
QueryResultCache::Writer::Writer(std::mutex & mutex_, Cache & cache_, const Key & key_,
|
||||
QueryCache::Writer::Writer(std::mutex & mutex_, Cache & cache_, const Key & key_,
|
||||
size_t & cache_size_in_bytes_, size_t max_cache_size_in_bytes_,
|
||||
size_t max_cache_entries_,
|
||||
size_t max_entry_size_in_bytes_, size_t max_entry_size_in_rows_,
|
||||
@ -189,7 +189,7 @@ QueryResultCache::Writer::Writer(std::mutex & mutex_, Cache & cache_, const Key
|
||||
skip_insert = true; /// Key already contained in cache and did not expire yet --> don't replace it
|
||||
}
|
||||
|
||||
void QueryResultCache::Writer::buffer(Chunk && partial_query_result)
|
||||
void QueryCache::Writer::buffer(Chunk && partial_query_result)
|
||||
{
|
||||
if (skip_insert)
|
||||
return;
|
||||
@ -208,7 +208,7 @@ void QueryResultCache::Writer::buffer(Chunk && partial_query_result)
|
||||
}
|
||||
}
|
||||
|
||||
void QueryResultCache::Writer::finalizeWrite()
|
||||
void QueryCache::Writer::finalizeWrite()
|
||||
{
|
||||
if (skip_insert)
|
||||
return;
|
||||
@ -239,7 +239,7 @@ void QueryResultCache::Writer::finalizeWrite()
|
||||
}
|
||||
else
|
||||
++it;
|
||||
LOG_TRACE(&Poco::Logger::get("QueryResultCache"), "Removed {} stale entries", removed_items);
|
||||
LOG_TRACE(&Poco::Logger::get("QueryCache"), "Removed {} stale entries", removed_items);
|
||||
}
|
||||
|
||||
/// Insert or replace if enough space
|
||||
@ -250,23 +250,23 @@ void QueryResultCache::Writer::finalizeWrite()
|
||||
cache_size_in_bytes -= it->second.sizeInBytes(); // key replacement
|
||||
|
||||
cache[key] = std::move(query_result);
|
||||
LOG_TRACE(&Poco::Logger::get("QueryResultCache"), "Stored result of query {}", key.queryStringFromAst());
|
||||
LOG_TRACE(&Poco::Logger::get("QueryCache"), "Stored result of query {}", key.queryStringFromAst());
|
||||
}
|
||||
}
|
||||
|
||||
QueryResultCache::Reader::Reader(const Cache & cache_, const Key & key, size_t & cache_size_in_bytes_, const std::lock_guard<std::mutex> &)
|
||||
QueryCache::Reader::Reader(const Cache & cache_, const Key & key, size_t & cache_size_in_bytes_, const std::lock_guard<std::mutex> &)
|
||||
{
|
||||
auto it = cache_.find(key);
|
||||
|
||||
if (it == cache_.end())
|
||||
{
|
||||
LOG_TRACE(&Poco::Logger::get("QueryResultCache"), "No entry found for query {}", key.queryStringFromAst());
|
||||
LOG_TRACE(&Poco::Logger::get("QueryCache"), "No entry found for query {}", key.queryStringFromAst());
|
||||
return;
|
||||
}
|
||||
|
||||
if (it->first.username.has_value() && it->first.username != key.username)
|
||||
{
|
||||
LOG_TRACE(&Poco::Logger::get("QueryResultCache"), "Inaccessible entry found for query {}", key.queryStringFromAst());
|
||||
LOG_TRACE(&Poco::Logger::get("QueryCache"), "Inaccessible entry found for query {}", key.queryStringFromAst());
|
||||
return;
|
||||
}
|
||||
|
||||
@ -274,33 +274,33 @@ QueryResultCache::Reader::Reader(const Cache & cache_, const Key & key, size_t &
|
||||
{
|
||||
cache_size_in_bytes_ -= it->second.sizeInBytes();
|
||||
const_cast<Cache &>(cache_).erase(it);
|
||||
LOG_TRACE(&Poco::Logger::get("QueryResultCache"), "Stale entry found and removed for query {}", key.queryStringFromAst());
|
||||
LOG_TRACE(&Poco::Logger::get("QueryCache"), "Stale entry found and removed for query {}", key.queryStringFromAst());
|
||||
return;
|
||||
}
|
||||
|
||||
pipe = Pipe(std::make_shared<SourceFromChunks>(it->first.header, it->second.chunks));
|
||||
LOG_TRACE(&Poco::Logger::get("QueryResultCache"), "Entry found for query {}", key.queryStringFromAst());
|
||||
LOG_TRACE(&Poco::Logger::get("QueryCache"), "Entry found for query {}", key.queryStringFromAst());
|
||||
}
|
||||
|
||||
bool QueryResultCache::Reader::hasCacheEntryForKey() const
|
||||
bool QueryCache::Reader::hasCacheEntryForKey() const
|
||||
{
|
||||
bool res = !pipe.empty();
|
||||
|
||||
if (res)
|
||||
ProfileEvents::increment(ProfileEvents::QueryResultCacheHits);
|
||||
ProfileEvents::increment(ProfileEvents::QueryCacheHits);
|
||||
else
|
||||
ProfileEvents::increment(ProfileEvents::QueryResultCacheMisses);
|
||||
ProfileEvents::increment(ProfileEvents::QueryCacheMisses);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
Pipe && QueryResultCache::Reader::getPipe()
|
||||
Pipe && QueryCache::Reader::getPipe()
|
||||
{
|
||||
chassert(!pipe.empty()); // cf. hasCacheEntryForKey()
|
||||
return std::move(pipe);
|
||||
}
|
||||
|
||||
QueryResultCache::QueryResultCache(size_t max_cache_size_in_bytes_, size_t max_cache_entries_, size_t max_cache_entry_size_in_bytes_, size_t max_cache_entry_size_in_rows_)
|
||||
QueryCache::QueryCache(size_t max_cache_size_in_bytes_, size_t max_cache_entries_, size_t max_cache_entry_size_in_bytes_, size_t max_cache_entry_size_in_rows_)
|
||||
: max_cache_size_in_bytes(max_cache_size_in_bytes_)
|
||||
, max_cache_entries(max_cache_entries_)
|
||||
, max_cache_entry_size_in_bytes(max_cache_entry_size_in_bytes_)
|
||||
@ -308,19 +308,19 @@ QueryResultCache::QueryResultCache(size_t max_cache_size_in_bytes_, size_t max_c
|
||||
{
|
||||
}
|
||||
|
||||
QueryResultCache::Reader QueryResultCache::createReader(const Key & key)
|
||||
QueryCache::Reader QueryCache::createReader(const Key & key)
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
return Reader(cache, key, cache_size_in_bytes, lock);
|
||||
}
|
||||
|
||||
QueryResultCache::Writer QueryResultCache::createWriter(const Key & key, std::chrono::milliseconds min_query_runtime)
|
||||
QueryCache::Writer QueryCache::createWriter(const Key & key, std::chrono::milliseconds min_query_runtime)
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
return Writer(mutex, cache, key, cache_size_in_bytes, max_cache_size_in_bytes, max_cache_entries, max_cache_entry_size_in_bytes, max_cache_entry_size_in_rows, min_query_runtime);
|
||||
}
|
||||
|
||||
void QueryResultCache::reset()
|
||||
void QueryCache::reset()
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
cache.clear();
|
||||
@ -328,7 +328,7 @@ void QueryResultCache::reset()
|
||||
cache_size_in_bytes = 0;
|
||||
}
|
||||
|
||||
size_t QueryResultCache::recordQueryRun(const Key & key)
|
||||
size_t QueryCache::recordQueryRun(const Key & key)
|
||||
{
|
||||
static constexpr size_t TIMES_EXECUTED_MAX_SIZE = 10'000;
|
||||
|
||||
@ -18,7 +18,7 @@ bool astContainsNonDeterministicFunctions(ASTPtr ast, ContextPtr context);
|
||||
/// returned. In order to still obtain sufficiently up-to-date query results, a expiry time (TTL) must be specified for each cache entry
|
||||
/// after which it becomes stale and is ignored. Stale entries are removed opportunistically from the cache, they are only evicted when a
|
||||
/// new entry is inserted and the cache has insufficient capacity.
|
||||
class QueryResultCache
|
||||
class QueryCache
|
||||
{
|
||||
public:
|
||||
/// Represents a query result in the cache.
|
||||
@ -82,9 +82,9 @@ public:
|
||||
/// Buffers multiple partial query result chunks (buffer()) and eventually stores them as cache entry (finalizeWrite()).
|
||||
///
|
||||
/// Implementation note: Queries may throw exceptions during runtime, e.g. out-of-memory errors. In this case, no query result must be
|
||||
/// written into the query result cache. Unfortunately, neither the Writer nor the special transform added on top of the query pipeline
|
||||
/// which holds the Writer know whether they are destroyed because the query ended successfully or because of an exception (otherwise,
|
||||
/// we could simply implement a check in their destructors). To handle exceptions correctly nevertheless, we do the actual insert in
|
||||
/// written into the query cache. Unfortunately, neither the Writer nor the special transform added on top of the query pipeline which
|
||||
/// holds the Writer know whether they are destroyed because the query ended successfully or because of an exception (otherwise, we
|
||||
/// could simply implement a check in their destructors). To handle exceptions correctly nevertheless, we do the actual insert in
|
||||
/// finalizeWrite() as opposed to the Writer destructor. This function is then called only for successful queries in finish_callback()
|
||||
/// which runs before the transform and the Writer are destroyed, whereas for unsuccessful queries we do nothing (the Writer is
|
||||
/// destroyed w/o inserting anything).
|
||||
@ -117,7 +117,7 @@ public:
|
||||
size_t max_entry_size_in_bytes_, size_t max_entry_size_in_rows_,
|
||||
std::chrono::milliseconds min_query_runtime_);
|
||||
|
||||
friend class QueryResultCache; /// for createWriter()
|
||||
friend class QueryCache; /// for createWriter()
|
||||
};
|
||||
|
||||
/// Looks up a query result for a key in the cache and (if found) constructs a pipe with the query result chunks as source.
|
||||
@ -129,10 +129,10 @@ public:
|
||||
private:
|
||||
Reader(const Cache & cache_, const Key & key, size_t & cache_size_in_bytes_, const std::lock_guard<std::mutex> &);
|
||||
Pipe pipe;
|
||||
friend class QueryResultCache; /// for createReader()
|
||||
friend class QueryCache; /// for createReader()
|
||||
};
|
||||
|
||||
QueryResultCache(size_t max_cache_size_in_bytes_, size_t max_cache_entries_, size_t max_cache_entry_size_in_bytes_, size_t max_cache_entry_size_in_rows_);
|
||||
QueryCache(size_t max_cache_size_in_bytes_, size_t max_cache_entries_, size_t max_cache_entry_size_in_bytes_, size_t max_cache_entry_size_in_rows_);
|
||||
|
||||
Reader createReader(const Key & key);
|
||||
Writer createWriter(const Key & key, std::chrono::milliseconds min_query_runtime);
|
||||
@ -160,9 +160,9 @@ private:
|
||||
const size_t max_cache_entry_size_in_bytes;
|
||||
const size_t max_cache_entry_size_in_rows;
|
||||
|
||||
friend class StorageSystemQueryResultCache;
|
||||
friend class StorageSystemQueryCache;
|
||||
};
|
||||
|
||||
using QueryResultCachePtr = std::shared_ptr<QueryResultCache>;
|
||||
using QueryCachePtr = std::shared_ptr<QueryCache>;
|
||||
|
||||
}
|
||||
@ -7,7 +7,6 @@
|
||||
#include <Common/ProfileEvents.h>
|
||||
#include <Common/checkStackSize.h>
|
||||
#include <TableFunctions/TableFunctionFactory.h>
|
||||
#include <IO/ConnectionTimeoutsContext.h>
|
||||
#include <Interpreters/RequiredSourceColumnsVisitor.h>
|
||||
#include <DataTypes/ObjectUtils.h>
|
||||
|
||||
|
||||
@ -40,7 +40,7 @@
|
||||
#include <Interpreters/ActionLocksManager.h>
|
||||
#include <Interpreters/ExternalLoaderXMLConfigRepository.h>
|
||||
#include <Interpreters/TemporaryDataOnDisk.h>
|
||||
#include <Interpreters/Cache/QueryResultCache.h>
|
||||
#include <Interpreters/Cache/QueryCache.h>
|
||||
#include <Core/Settings.h>
|
||||
#include <Core/SettingsQuirks.h>
|
||||
#include <Access/AccessControl.h>
|
||||
@ -236,7 +236,7 @@ struct ContextSharedPart : boost::noncopyable
|
||||
mutable std::unique_ptr<ThreadPool> load_marks_threadpool; /// Threadpool for loading marks cache.
|
||||
mutable UncompressedCachePtr index_uncompressed_cache; /// The cache of decompressed blocks for MergeTree indices.
|
||||
mutable MarkCachePtr index_mark_cache; /// Cache of marks in compressed files of MergeTree indices.
|
||||
mutable QueryResultCachePtr query_result_cache; /// Cache of query results.
|
||||
mutable QueryCachePtr query_cache; /// Cache of query results.
|
||||
mutable MMappedFileCachePtr mmap_cache; /// Cache of mmapped files to avoid frequent open/map/unmap/close and to reuse from several threads.
|
||||
ProcessList process_list; /// Executing queries at the moment.
|
||||
GlobalOvercommitTracker global_overcommit_tracker;
|
||||
@ -2041,27 +2041,27 @@ void Context::dropIndexMarkCache() const
|
||||
shared->index_mark_cache->reset();
|
||||
}
|
||||
|
||||
void Context::setQueryResultCache(size_t max_size_in_bytes, size_t max_entries, size_t max_entry_size_in_bytes, size_t max_entry_size_in_records)
|
||||
void Context::setQueryCache(size_t max_size_in_bytes, size_t max_entries, size_t max_entry_size_in_bytes, size_t max_entry_size_in_records)
|
||||
{
|
||||
auto lock = getLock();
|
||||
|
||||
if (shared->query_result_cache)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Query result cache has been already created.");
|
||||
if (shared->query_cache)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Query cache has been already created.");
|
||||
|
||||
shared->query_result_cache = std::make_shared<QueryResultCache>(max_size_in_bytes, max_entries, max_entry_size_in_bytes, max_entry_size_in_records);
|
||||
shared->query_cache = std::make_shared<QueryCache>(max_size_in_bytes, max_entries, max_entry_size_in_bytes, max_entry_size_in_records);
|
||||
}
|
||||
|
||||
QueryResultCachePtr Context::getQueryResultCache() const
|
||||
QueryCachePtr Context::getQueryCache() const
|
||||
{
|
||||
auto lock = getLock();
|
||||
return shared->query_result_cache;
|
||||
return shared->query_cache;
|
||||
}
|
||||
|
||||
void Context::dropQueryResultCache() const
|
||||
void Context::dropQueryCache() const
|
||||
{
|
||||
auto lock = getLock();
|
||||
if (shared->query_result_cache)
|
||||
shared->query_result_cache->reset();
|
||||
if (shared->query_cache)
|
||||
shared->query_cache->reset();
|
||||
}
|
||||
|
||||
void Context::setMMappedFileCache(size_t cache_size_in_num_entries)
|
||||
@ -2104,8 +2104,8 @@ void Context::dropCaches() const
|
||||
if (shared->index_mark_cache)
|
||||
shared->index_mark_cache->reset();
|
||||
|
||||
if (shared->query_result_cache)
|
||||
shared->query_result_cache->reset();
|
||||
if (shared->query_cache)
|
||||
shared->query_cache->reset();
|
||||
|
||||
if (shared->mmap_cache)
|
||||
shared->mmap_cache->reset();
|
||||
|
||||
@ -81,8 +81,8 @@ class Macros;
|
||||
struct Progress;
|
||||
struct FileProgress;
|
||||
class Clusters;
|
||||
class QueryCache;
|
||||
class QueryLog;
|
||||
class QueryResultCache;
|
||||
class QueryThreadLog;
|
||||
class QueryViewsLog;
|
||||
class PartLog;
|
||||
@ -861,9 +861,9 @@ public:
|
||||
void dropMMappedFileCache() const;
|
||||
|
||||
/// Create a cache of query results for statements which run repeatedly.
|
||||
void setQueryResultCache(size_t max_size_in_bytes, size_t max_entries, size_t max_entry_size_in_bytes, size_t max_entry_size_in_records);
|
||||
std::shared_ptr<QueryResultCache> getQueryResultCache() const;
|
||||
void dropQueryResultCache() const;
|
||||
void setQueryCache(size_t max_size_in_bytes, size_t max_entries, size_t max_entry_size_in_bytes, size_t max_entry_size_in_records);
|
||||
std::shared_ptr<QueryCache> getQueryCache() const;
|
||||
void dropQueryCache() const;
|
||||
|
||||
/** Clear the caches of the uncompressed blocks and marks.
|
||||
* This is usually done when renaming tables, changing the type of columns, deleting a table.
|
||||
|
||||
@ -147,7 +147,7 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter)
|
||||
{
|
||||
table->checkMutationIsPossible(mutation_commands, getContext()->getSettingsRef());
|
||||
MutationsInterpreter(table, metadata_snapshot, mutation_commands, getContext(), false).validate();
|
||||
table->mutate(mutation_commands, getContext(), false);
|
||||
table->mutate(mutation_commands, getContext());
|
||||
}
|
||||
|
||||
if (!partition_commands.empty())
|
||||
|
||||
@ -5,15 +5,16 @@
|
||||
#include <Databases/IDatabase.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Interpreters/FunctionNameNormalizer.h>
|
||||
#include <Interpreters/InterpreterAlterQuery.h>
|
||||
#include <Interpreters/MutationsInterpreter.h>
|
||||
#include <Parsers/parseQuery.h>
|
||||
#include <Parsers/formatAST.h>
|
||||
#include <Parsers/ParserAlterQuery.h>
|
||||
#include <Parsers/ASTDeleteQuery.h>
|
||||
#include <Parsers/ASTAssignment.h>
|
||||
#include <Parsers/ASTExpressionList.h>
|
||||
#include <Storages/AlterCommands.h>
|
||||
#include <Storages/IStorage.h>
|
||||
#include <Storages/MutationCommands.h>
|
||||
#include <Storages/LightweightDeleteDescription.h>
|
||||
#include <Storages/MergeTree/MergeTreeData.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -72,7 +73,7 @@ BlockIO InterpreterDeleteQuery::execute()
|
||||
|
||||
table->checkMutationIsPossible(mutation_commands, getContext()->getSettingsRef());
|
||||
MutationsInterpreter(table, metadata_snapshot, mutation_commands, getContext(), false).validate();
|
||||
table->mutate(mutation_commands, getContext(), false);
|
||||
table->mutate(mutation_commands, getContext());
|
||||
return {};
|
||||
}
|
||||
else if (table->supportsLightweightDelete())
|
||||
@ -82,35 +83,25 @@ BlockIO InterpreterDeleteQuery::execute()
|
||||
"Lightweight delete mutate is experimental. "
|
||||
"Set `allow_experimental_lightweight_delete` setting to enable it");
|
||||
|
||||
/// Convert to MutationCommand
|
||||
MutationCommands mutation_commands;
|
||||
MutationCommand mut_command;
|
||||
/// Build "ALTER ... UPDATE _row_exists = 0 WHERE predicate" query
|
||||
String alter_query =
|
||||
"ALTER TABLE " + table->getStorageID().getFullTableName()
|
||||
+ (delete_query.cluster.empty() ? "" : " ON CLUSTER " + backQuoteIfNeed(delete_query.cluster))
|
||||
+ " UPDATE `_row_exists` = 0 WHERE " + serializeAST(*delete_query.predicate);
|
||||
|
||||
/// Build "UPDATE _row_exists = 0 WHERE predicate" query
|
||||
mut_command.type = MutationCommand::Type::UPDATE;
|
||||
mut_command.predicate = delete_query.predicate;
|
||||
ParserAlterQuery parser;
|
||||
ASTPtr alter_ast = parseQuery(
|
||||
parser,
|
||||
alter_query.data(),
|
||||
alter_query.data() + alter_query.size(),
|
||||
"ALTER query",
|
||||
0,
|
||||
DBMS_DEFAULT_MAX_PARSER_DEPTH);
|
||||
|
||||
auto command = std::make_shared<ASTAlterCommand>();
|
||||
command->type = ASTAlterCommand::UPDATE;
|
||||
command->predicate = delete_query.predicate;
|
||||
command->update_assignments = std::make_shared<ASTExpressionList>();
|
||||
auto set_row_does_not_exist = std::make_shared<ASTAssignment>();
|
||||
set_row_does_not_exist->column_name = LightweightDeleteDescription::FILTER_COLUMN.name;
|
||||
auto zero_value = std::make_shared<ASTLiteral>(DB::Field(UInt8(0)));
|
||||
set_row_does_not_exist->children.push_back(zero_value);
|
||||
command->update_assignments->children.push_back(set_row_does_not_exist);
|
||||
command->children.push_back(command->predicate);
|
||||
command->children.push_back(command->update_assignments);
|
||||
mut_command.column_to_update_expression[set_row_does_not_exist->column_name] = zero_value;
|
||||
mut_command.ast = command->ptr();
|
||||
|
||||
mutation_commands.emplace_back(mut_command);
|
||||
|
||||
table->checkMutationIsPossible(mutation_commands, getContext()->getSettingsRef());
|
||||
MutationsInterpreter(table, metadata_snapshot, mutation_commands, getContext(), false).validate();
|
||||
table->mutate(mutation_commands, getContext(), true);
|
||||
|
||||
return {};
|
||||
auto context = Context::createCopy(getContext());
|
||||
context->setSetting("mutations_sync", 2); /// Lightweight delete is always synchronous
|
||||
InterpreterAlterQuery alter_interpreter(alter_ast, context);
|
||||
return alter_interpreter.execute();
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@ -6,7 +6,6 @@
|
||||
#include <Columns/ColumnNullable.h>
|
||||
#include <Processors/Transforms/buildPushingToViewsChain.h>
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
#include <IO/ConnectionTimeoutsContext.h>
|
||||
#include <Interpreters/InterpreterSelectWithUnionQuery.h>
|
||||
#include <Interpreters/InterpreterWatchQuery.h>
|
||||
#include <Interpreters/QueryLog.h>
|
||||
|
||||
@ -327,9 +327,9 @@ BlockIO InterpreterSystemQuery::execute()
|
||||
getContext()->checkAccess(AccessType::SYSTEM_DROP_MMAP_CACHE);
|
||||
system_context->dropMMappedFileCache();
|
||||
break;
|
||||
case Type::DROP_QUERY_RESULT_CACHE:
|
||||
getContext()->checkAccess(AccessType::SYSTEM_DROP_QUERY_RESULT_CACHE);
|
||||
getContext()->dropQueryResultCache();
|
||||
case Type::DROP_QUERY_CACHE:
|
||||
getContext()->checkAccess(AccessType::SYSTEM_DROP_QUERY_CACHE);
|
||||
getContext()->dropQueryCache();
|
||||
break;
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
case Type::DROP_COMPILED_EXPRESSION_CACHE:
|
||||
@ -969,7 +969,7 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster()
|
||||
case Type::DROP_DNS_CACHE:
|
||||
case Type::DROP_MARK_CACHE:
|
||||
case Type::DROP_MMAP_CACHE:
|
||||
case Type::DROP_QUERY_RESULT_CACHE:
|
||||
case Type::DROP_QUERY_CACHE:
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
case Type::DROP_COMPILED_EXPRESSION_CACHE:
|
||||
#endif
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
#include <QueryPipeline/BlockIO.h>
|
||||
#include <Processors/Transforms/CountingTransform.h>
|
||||
#include <Processors/Transforms/getSourceFromASTInsertQuery.h>
|
||||
#include <Processors/Transforms/StreamInQueryResultCacheTransform.h>
|
||||
#include <Processors/Transforms/StreamInQueryCacheTransform.h>
|
||||
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
#include <Parsers/ASTInsertQuery.h>
|
||||
@ -716,48 +716,48 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
|
||||
|
||||
/// If
|
||||
/// - it is a SELECT query,
|
||||
/// - passive (read) use of the query result cache is enabled, and
|
||||
/// - the query result cache knows the query result
|
||||
/// then replace the pipeline by a new pipeline with a single source that is populated from the query result cache
|
||||
auto query_result_cache = context->getQueryResultCache();
|
||||
bool read_result_from_query_result_cache = false; /// a query must not read from *and* write to the query result cache at the same time
|
||||
if (query_result_cache != nullptr
|
||||
&& (settings.allow_experimental_query_result_cache && settings.use_query_result_cache && settings.enable_reads_from_query_result_cache)
|
||||
/// - passive (read) use of the query cache is enabled, and
|
||||
/// - the query cache knows the query result
|
||||
/// then replace the pipeline by a new pipeline with a single source that is populated from the query cache
|
||||
auto query_cache = context->getQueryCache();
|
||||
bool read_result_from_query_cache = false; /// a query must not read from *and* write to the query cache at the same time
|
||||
if (query_cache != nullptr
|
||||
&& (settings.allow_experimental_query_cache && settings.use_query_cache && settings.enable_reads_from_query_cache)
|
||||
&& res.pipeline.pulling())
|
||||
{
|
||||
QueryResultCache::Key key(
|
||||
QueryCache::Key key(
|
||||
ast, res.pipeline.getHeader(),
|
||||
std::make_optional<String>(context->getUserName()),
|
||||
std::chrono::system_clock::now() + std::chrono::seconds(settings.query_result_cache_ttl));
|
||||
QueryResultCache::Reader reader = query_result_cache->createReader(key);
|
||||
std::chrono::system_clock::now() + std::chrono::seconds(settings.query_cache_ttl));
|
||||
QueryCache::Reader reader = query_cache->createReader(key);
|
||||
if (reader.hasCacheEntryForKey())
|
||||
{
|
||||
res.pipeline = QueryPipeline(reader.getPipe());
|
||||
read_result_from_query_result_cache = true;
|
||||
read_result_from_query_cache = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// If
|
||||
/// - it is a SELECT query, and
|
||||
/// - active (write) use of the query result cache is enabled
|
||||
/// then add a processor on top of the pipeline which stores the result in the query result cache.
|
||||
if (!read_result_from_query_result_cache
|
||||
&& query_result_cache != nullptr
|
||||
&& settings.allow_experimental_query_result_cache && settings.use_query_result_cache && settings.enable_writes_to_query_result_cache
|
||||
/// - active (write) use of the query cache is enabled
|
||||
/// then add a processor on top of the pipeline which stores the result in the query cache.
|
||||
if (!read_result_from_query_cache
|
||||
&& query_cache != nullptr
|
||||
&& settings.allow_experimental_query_cache && settings.use_query_cache && settings.enable_writes_to_query_cache
|
||||
&& res.pipeline.pulling()
|
||||
&& (!astContainsNonDeterministicFunctions(ast, context) || settings.query_result_cache_store_results_of_queries_with_nondeterministic_functions))
|
||||
&& (!astContainsNonDeterministicFunctions(ast, context) || settings.query_cache_store_results_of_queries_with_nondeterministic_functions))
|
||||
{
|
||||
QueryResultCache::Key key(
|
||||
QueryCache::Key key(
|
||||
ast, res.pipeline.getHeader(),
|
||||
settings.query_result_cache_share_between_users ? std::nullopt : std::make_optional<String>(context->getUserName()),
|
||||
std::chrono::system_clock::now() + std::chrono::seconds(settings.query_result_cache_ttl));
|
||||
settings.query_cache_share_between_users ? std::nullopt : std::make_optional<String>(context->getUserName()),
|
||||
std::chrono::system_clock::now() + std::chrono::seconds(settings.query_cache_ttl));
|
||||
|
||||
const size_t num_query_runs = query_result_cache->recordQueryRun(key);
|
||||
if (num_query_runs > settings.query_result_cache_min_query_runs)
|
||||
const size_t num_query_runs = query_cache->recordQueryRun(key);
|
||||
if (num_query_runs > settings.query_cache_min_query_runs)
|
||||
{
|
||||
auto stream_in_query_result_cache_transform = std::make_shared<StreamInQueryResultCacheTransform>(res.pipeline.getHeader(), query_result_cache, key,
|
||||
std::chrono::milliseconds(context->getSettings().query_result_cache_min_query_duration.totalMilliseconds()));
|
||||
res.pipeline.streamIntoQueryResultCache(stream_in_query_result_cache_transform);
|
||||
auto stream_in_query_cache_transform = std::make_shared<StreamInQueryCacheTransform>(res.pipeline.getHeader(), query_cache, key,
|
||||
std::chrono::milliseconds(context->getSettings().query_cache_min_query_duration.totalMilliseconds()));
|
||||
res.pipeline.streamIntoQueryCache(stream_in_query_cache_transform);
|
||||
}
|
||||
}
|
||||
|
||||
@ -908,10 +908,10 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
|
||||
auto finish_callback = [elem,
|
||||
context,
|
||||
ast,
|
||||
allow_experimental_query_result_cache = settings.allow_experimental_query_result_cache,
|
||||
use_query_result_cache = settings.use_query_result_cache,
|
||||
enable_writes_to_query_result_cache = settings.enable_writes_to_query_result_cache,
|
||||
query_result_cache_store_results_of_queries_with_nondeterministic_functions = settings.query_result_cache_store_results_of_queries_with_nondeterministic_functions,
|
||||
allow_experimental_query_cache = settings.allow_experimental_query_cache,
|
||||
use_query_cache = settings.use_query_cache,
|
||||
enable_writes_to_query_cache = settings.enable_writes_to_query_cache,
|
||||
query_cache_store_results_of_queries_with_nondeterministic_functions = settings.query_cache_store_results_of_queries_with_nondeterministic_functions,
|
||||
log_queries,
|
||||
log_queries_min_type = settings.log_queries_min_type,
|
||||
log_queries_min_query_duration_ms = settings.log_queries_min_query_duration_ms.totalMilliseconds(),
|
||||
@ -921,15 +921,15 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
|
||||
pulling_pipeline = pipeline.pulling(),
|
||||
query_span](QueryPipeline & query_pipeline) mutable
|
||||
{
|
||||
/// If active (write) use of the query result cache is enabled and the query is eligible for result caching, then store the
|
||||
/// query result buffered in the special-purpose cache processor (added on top of the pipeline) into the cache.
|
||||
auto query_result_cache = context->getQueryResultCache();
|
||||
if (query_result_cache != nullptr
|
||||
/// If active (write) use of the query cache is enabled and the query is eligible for result caching, then store the query
|
||||
/// result buffered in the special-purpose cache processor (added on top of the pipeline) into the cache.
|
||||
auto query_cache = context->getQueryCache();
|
||||
if (query_cache != nullptr
|
||||
&& pulling_pipeline
|
||||
&& allow_experimental_query_result_cache && use_query_result_cache && enable_writes_to_query_result_cache
|
||||
&& (!astContainsNonDeterministicFunctions(ast, context) || query_result_cache_store_results_of_queries_with_nondeterministic_functions))
|
||||
&& allow_experimental_query_cache && use_query_cache && enable_writes_to_query_cache
|
||||
&& (!astContainsNonDeterministicFunctions(ast, context) || query_cache_store_results_of_queries_with_nondeterministic_functions))
|
||||
{
|
||||
query_pipeline.finalizeWriteInQueryResultCache();
|
||||
query_pipeline.finalizeWriteInQueryCache();
|
||||
}
|
||||
|
||||
QueryStatusPtr process_list_elem = context->getProcessListElement();
|
||||
|
||||
@ -41,6 +41,8 @@ void ASTDeleteQuery::formatQueryImpl(const FormatSettings & settings, FormatStat
|
||||
}
|
||||
settings.ostr << backQuoteIfNeed(getTable());
|
||||
|
||||
formatOnCluster(settings);
|
||||
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : "");
|
||||
predicate->formatImpl(settings, state, frame);
|
||||
}
|
||||
|
||||
@ -2,15 +2,20 @@
|
||||
|
||||
#include <Parsers/IAST.h>
|
||||
#include <Parsers/ASTQueryWithTableAndOutput.h>
|
||||
#include <Parsers/ASTQueryWithOnCluster.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
/// DELETE FROM [db.]name WHERE ...
|
||||
class ASTDeleteQuery : public ASTQueryWithTableAndOutput
|
||||
class ASTDeleteQuery : public ASTQueryWithTableAndOutput, public ASTQueryWithOnCluster
|
||||
{
|
||||
public:
|
||||
String getID(char delim) const final;
|
||||
ASTPtr clone() const final;
|
||||
ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams & params) const override
|
||||
{
|
||||
return removeOnCluster<ASTDeleteQuery>(clone(), params.default_database);
|
||||
}
|
||||
|
||||
ASTPtr predicate;
|
||||
|
||||
|
||||
@ -37,81 +37,118 @@ namespace
|
||||
{
|
||||
/// Finds arguments of a specified function which should not be displayed for most users for security reasons.
|
||||
/// That involves passwords and secret keys.
|
||||
/// The member function getRange() returns a pair of numbers [first, last) specifying arguments
|
||||
/// which must be hidden. If the function returns {-1, -1} that means no arguments must be hidden.
|
||||
class FunctionSecretArgumentsFinder
|
||||
{
|
||||
public:
|
||||
explicit FunctionSecretArgumentsFinder(const ASTFunction & function_) : function(function_)
|
||||
{
|
||||
if (function.arguments)
|
||||
{
|
||||
if (const auto * expr_list = function.arguments->as<ASTExpressionList>())
|
||||
arguments = &expr_list->children;
|
||||
}
|
||||
}
|
||||
if (!function.arguments)
|
||||
return;
|
||||
|
||||
std::pair<size_t, size_t> getRange() const
|
||||
{
|
||||
if (!arguments)
|
||||
return npos;
|
||||
const auto * expr_list = function.arguments->as<ASTExpressionList>();
|
||||
if (!expr_list)
|
||||
return;
|
||||
|
||||
arguments = &expr_list->children;
|
||||
switch (function.kind)
|
||||
{
|
||||
case ASTFunction::Kind::ORDINARY_FUNCTION: return findOrdinaryFunctionSecretArguments();
|
||||
case ASTFunction::Kind::WINDOW_FUNCTION: return npos;
|
||||
case ASTFunction::Kind::LAMBDA_FUNCTION: return npos;
|
||||
case ASTFunction::Kind::TABLE_ENGINE: return findTableEngineSecretArguments();
|
||||
case ASTFunction::Kind::DATABASE_ENGINE: return findDatabaseEngineSecretArguments();
|
||||
case ASTFunction::Kind::BACKUP_NAME: return findBackupNameSecretArguments();
|
||||
case ASTFunction::Kind::ORDINARY_FUNCTION: findOrdinaryFunctionSecretArguments(); break;
|
||||
case ASTFunction::Kind::WINDOW_FUNCTION: break;
|
||||
case ASTFunction::Kind::LAMBDA_FUNCTION: break;
|
||||
case ASTFunction::Kind::TABLE_ENGINE: findTableEngineSecretArguments(); break;
|
||||
case ASTFunction::Kind::DATABASE_ENGINE: findDatabaseEngineSecretArguments(); break;
|
||||
case ASTFunction::Kind::BACKUP_NAME: findBackupNameSecretArguments(); break;
|
||||
}
|
||||
}
|
||||
|
||||
static const constexpr std::pair<size_t, size_t> npos{static_cast<size_t>(-1), static_cast<size_t>(-1)};
|
||||
struct Result
|
||||
{
|
||||
/// Result constructed by default means no arguments will be hidden.
|
||||
size_t start = static_cast<size_t>(-1);
|
||||
size_t count = 0; /// Mostly it's either 0 or 1. There are only a few cases where `count` can be greater than 1 (e.g. see `encrypt`).
|
||||
/// In all known cases secret arguments are consecutive
|
||||
bool are_named = false; /// Arguments like `password = 'password'` are considered as named arguments.
|
||||
};
|
||||
|
||||
Result getResult() const { return result; }
|
||||
|
||||
private:
|
||||
std::pair<size_t, size_t> findOrdinaryFunctionSecretArguments() const
|
||||
const ASTFunction & function;
|
||||
const ASTs * arguments = nullptr;
|
||||
Result result;
|
||||
|
||||
void markSecretArgument(size_t index, bool argument_is_named = false)
|
||||
{
|
||||
if (!result.count)
|
||||
{
|
||||
result.start = index;
|
||||
result.are_named = argument_is_named;
|
||||
}
|
||||
chassert(index >= result.start); /// We always check arguments consecutively
|
||||
result.count = index + 1 - result.start;
|
||||
if (!argument_is_named)
|
||||
result.are_named = false;
|
||||
}
|
||||
|
||||
void findOrdinaryFunctionSecretArguments()
|
||||
{
|
||||
if ((function.name == "mysql") || (function.name == "postgresql") || (function.name == "mongodb"))
|
||||
{
|
||||
/// mysql('host:port', 'database', 'table', 'user', 'password', ...)
|
||||
/// postgresql('host:port', 'database', 'table', 'user', 'password', ...)
|
||||
/// mongodb('host:port', 'database', 'collection', 'user', 'password', ...)
|
||||
return {4, 5};
|
||||
findMySQLFunctionSecretArguments();
|
||||
}
|
||||
else if ((function.name == "s3") || (function.name == "cosn") || (function.name == "oss"))
|
||||
{
|
||||
/// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...)
|
||||
return findS3FunctionSecretArguments(/* is_cluster_function= */ false);
|
||||
findS3FunctionSecretArguments(/* is_cluster_function= */ false);
|
||||
}
|
||||
else if (function.name == "s3Cluster")
|
||||
{
|
||||
/// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', ...)
|
||||
return findS3FunctionSecretArguments(/* is_cluster_function= */ true);
|
||||
findS3FunctionSecretArguments(/* is_cluster_function= */ true);
|
||||
}
|
||||
else if ((function.name == "remote") || (function.name == "remoteSecure"))
|
||||
{
|
||||
/// remote('addresses_expr', 'db', 'table', 'user', 'password', ...)
|
||||
return findRemoteFunctionSecretArguments();
|
||||
findRemoteFunctionSecretArguments();
|
||||
}
|
||||
else if ((function.name == "encrypt") || (function.name == "decrypt") ||
|
||||
(function.name == "aes_encrypt_mysql") || (function.name == "aes_decrypt_mysql") ||
|
||||
(function.name == "tryDecrypt"))
|
||||
{
|
||||
/// encrypt('mode', 'plaintext', 'key' [, iv, aad])
|
||||
return findEncryptionFunctionSecretArguments();
|
||||
}
|
||||
else
|
||||
{
|
||||
return npos;
|
||||
findEncryptionFunctionSecretArguments();
|
||||
}
|
||||
}
|
||||
|
||||
std::pair<size_t, size_t> findS3FunctionSecretArguments(bool is_cluster_function) const
|
||||
void findMySQLFunctionSecretArguments()
|
||||
{
|
||||
if (isNamedCollectionName(0))
|
||||
{
|
||||
/// mysql(named_collection, ..., password = 'password', ...)
|
||||
findSecretNamedArgument("password", 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
/// mysql('host:port', 'database', 'table', 'user', 'password', ...)
|
||||
markSecretArgument(4);
|
||||
}
|
||||
}
|
||||
|
||||
void findS3FunctionSecretArguments(bool is_cluster_function)
|
||||
{
|
||||
/// s3Cluster('cluster_name', 'url', ...) has 'url' as its second argument.
|
||||
size_t url_arg_idx = is_cluster_function ? 1 : 0;
|
||||
|
||||
if (!is_cluster_function && isNamedCollectionName(0))
|
||||
{
|
||||
/// s3(named_collection, ..., secret_access_key = 'secret_access_key', ...)
|
||||
findSecretNamedArgument("secret_access_key", 1);
|
||||
return;
|
||||
}
|
||||
|
||||
/// We're going to replace 'aws_secret_access_key' with '[HIDDEN'] for the following signatures:
|
||||
/// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...)
|
||||
/// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression')
|
||||
@ -119,12 +156,12 @@ namespace
|
||||
/// But we should check the number of arguments first because we don't need to do any replacements in case of
|
||||
/// s3('url' [, 'format']) or s3Cluster('cluster_name', 'url' [, 'format'])
|
||||
if (arguments->size() < url_arg_idx + 3)
|
||||
return npos;
|
||||
return;
|
||||
|
||||
if (arguments->size() >= url_arg_idx + 5)
|
||||
{
|
||||
/// s3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'structure', ...)
|
||||
return {url_arg_idx + 2, url_arg_idx + 3};
|
||||
markSecretArgument(url_arg_idx + 2);
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -136,15 +173,16 @@ namespace
|
||||
{
|
||||
/// We couldn't evaluate the argument after 'url' so we don't know whether it is a format or `aws_access_key_id`.
|
||||
/// So it's safer to wipe the next argument just in case.
|
||||
return {url_arg_idx + 2, url_arg_idx + 3}; /// Wipe either `aws_secret_access_key` or `structure`.
|
||||
markSecretArgument(url_arg_idx + 2); /// Wipe either `aws_secret_access_key` or `structure`.
|
||||
return;
|
||||
}
|
||||
|
||||
if (KnownFormatNames::instance().exists(format))
|
||||
return npos; /// The argument after 'url' is a format: s3('url', 'format', ...)
|
||||
return; /// The argument after 'url' is a format: s3('url', 'format', ...)
|
||||
|
||||
/// The argument after 'url' is not a format so we do our replacement:
|
||||
/// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...) -> s3('url', 'aws_access_key_id', '[HIDDEN]', ...)
|
||||
return {url_arg_idx + 2, url_arg_idx + 3};
|
||||
markSecretArgument(url_arg_idx + 2);
|
||||
}
|
||||
}
|
||||
|
||||
@ -153,8 +191,12 @@ namespace
|
||||
if (arg_idx >= arguments->size())
|
||||
return false;
|
||||
|
||||
ASTPtr argument = (*arguments)[arg_idx];
|
||||
if (const auto * literal = argument->as<ASTLiteral>())
|
||||
return tryGetStringFromArgument(*(*arguments)[arg_idx], res, allow_identifier);
|
||||
}
|
||||
|
||||
static bool tryGetStringFromArgument(const IAST & argument, String * res, bool allow_identifier = true)
|
||||
{
|
||||
if (const auto * literal = argument.as<ASTLiteral>())
|
||||
{
|
||||
if (literal->value.getType() != Field::Types::String)
|
||||
return false;
|
||||
@ -165,7 +207,7 @@ namespace
|
||||
|
||||
if (allow_identifier)
|
||||
{
|
||||
if (const auto * id = argument->as<ASTIdentifier>())
|
||||
if (const auto * id = argument.as<ASTIdentifier>())
|
||||
{
|
||||
if (res)
|
||||
*res = id->name();
|
||||
@ -176,8 +218,15 @@ namespace
|
||||
return false;
|
||||
}
|
||||
|
||||
std::pair<size_t, size_t> findRemoteFunctionSecretArguments() const
|
||||
void findRemoteFunctionSecretArguments()
|
||||
{
|
||||
if (isNamedCollectionName(0))
|
||||
{
|
||||
/// remote(named_collection, ..., password = 'password', ...)
|
||||
findSecretNamedArgument("password", 1);
|
||||
return;
|
||||
}
|
||||
|
||||
/// We're going to replace 'password' with '[HIDDEN'] for the following signatures:
|
||||
/// remote('addresses_expr', db.table, 'user' [, 'password'] [, sharding_key])
|
||||
/// remote('addresses_expr', 'db', 'table', 'user' [, 'password'] [, sharding_key])
|
||||
@ -186,7 +235,7 @@ namespace
|
||||
/// But we should check the number of arguments first because we don't need to do any replacements in case of
|
||||
/// remote('addresses_expr', db.table)
|
||||
if (arguments->size() < 3)
|
||||
return npos;
|
||||
return;
|
||||
|
||||
size_t arg_num = 1;
|
||||
|
||||
@ -207,20 +256,17 @@ namespace
|
||||
/// before the argument 'password'. So it's safer to wipe two arguments just in case.
|
||||
/// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string
|
||||
/// before wiping it (because the `password` argument is always a literal string).
|
||||
auto res = npos;
|
||||
if (tryGetStringFromArgument(arg_num + 2, nullptr, /* allow_identifier= */ false))
|
||||
{
|
||||
/// Wipe either `password` or `user`.
|
||||
res = {arg_num + 2, arg_num + 3};
|
||||
markSecretArgument(arg_num + 2);
|
||||
}
|
||||
if (tryGetStringFromArgument(arg_num + 3, nullptr, /* allow_identifier= */ false))
|
||||
{
|
||||
/// Wipe either `password` or `sharding_key`.
|
||||
if (res == npos)
|
||||
res.first = arg_num + 3;
|
||||
res.second = arg_num + 4;
|
||||
markSecretArgument(arg_num + 3);
|
||||
}
|
||||
return res;
|
||||
return;
|
||||
}
|
||||
|
||||
/// Skip the current argument (which is either a database name or a qualified table name).
|
||||
@ -241,9 +287,7 @@ namespace
|
||||
/// before wiping it (because the `password` argument is always a literal string).
|
||||
bool can_be_password = tryGetStringFromArgument(arg_num, nullptr, /* allow_identifier= */ false);
|
||||
if (can_be_password)
|
||||
return {arg_num, arg_num + 1};
|
||||
|
||||
return npos;
|
||||
markSecretArgument(arg_num);
|
||||
}
|
||||
|
||||
/// Tries to get either a database name or a qualified table name from an argument.
|
||||
@ -278,20 +322,24 @@ namespace
|
||||
return true;
|
||||
}
|
||||
|
||||
std::pair<size_t, size_t> findEncryptionFunctionSecretArguments() const
|
||||
void findEncryptionFunctionSecretArguments()
|
||||
{
|
||||
if (arguments->empty())
|
||||
return;
|
||||
|
||||
/// We replace all arguments after 'mode' with '[HIDDEN]':
|
||||
/// encrypt('mode', 'plaintext', 'key' [, iv, aad]) -> encrypt('mode', '[HIDDEN]')
|
||||
return {1, arguments->size()};
|
||||
result.start = 1;
|
||||
result.count = arguments->size() - 1;
|
||||
}
|
||||
|
||||
std::pair<size_t, size_t> findTableEngineSecretArguments() const
|
||||
void findTableEngineSecretArguments()
|
||||
{
|
||||
const String & engine_name = function.name;
|
||||
if (engine_name == "ExternalDistributed")
|
||||
{
|
||||
/// ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password')
|
||||
return {5, 6};
|
||||
findExternalDistributedTableEngineSecretArguments();
|
||||
}
|
||||
else if ((engine_name == "MySQL") || (engine_name == "PostgreSQL") ||
|
||||
(engine_name == "MaterializedPostgreSQL") || (engine_name == "MongoDB"))
|
||||
@ -300,21 +348,38 @@ namespace
|
||||
/// PostgreSQL('host:port', 'database', 'table', 'user', 'password', ...)
|
||||
/// MaterializedPostgreSQL('host:port', 'database', 'table', 'user', 'password', ...)
|
||||
/// MongoDB('host:port', 'database', 'collection', 'user', 'password', ...)
|
||||
return {4, 5};
|
||||
findMySQLFunctionSecretArguments();
|
||||
}
|
||||
else if ((engine_name == "S3") || (engine_name == "COSN") || (engine_name == "OSS"))
|
||||
{
|
||||
/// S3('url', ['aws_access_key_id', 'aws_secret_access_key',] ...)
|
||||
return findS3TableEngineSecretArguments();
|
||||
}
|
||||
else
|
||||
{
|
||||
return npos;
|
||||
findS3TableEngineSecretArguments();
|
||||
}
|
||||
}
|
||||
|
||||
std::pair<size_t, size_t> findS3TableEngineSecretArguments() const
|
||||
void findExternalDistributedTableEngineSecretArguments()
|
||||
{
|
||||
if (isNamedCollectionName(1))
|
||||
{
|
||||
/// ExternalDistributed('engine', named_collection, ..., password = 'password', ...)
|
||||
findSecretNamedArgument("password", 2);
|
||||
}
|
||||
else
|
||||
{
|
||||
/// ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password')
|
||||
markSecretArgument(5);
|
||||
}
|
||||
}
|
||||
|
||||
void findS3TableEngineSecretArguments()
|
||||
{
|
||||
if (isNamedCollectionName(0))
|
||||
{
|
||||
/// S3(named_collection, ..., secret_access_key = 'secret_access_key')
|
||||
findSecretNamedArgument("secret_access_key", 1);
|
||||
return;
|
||||
}
|
||||
|
||||
/// We replace 'aws_secret_access_key' with '[HIDDEN'] for the following signatures:
|
||||
/// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format')
|
||||
/// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression')
|
||||
@ -322,12 +387,12 @@ namespace
|
||||
/// But we should check the number of arguments first because we don't need to do that replacements in case of
|
||||
/// S3('url' [, 'format' [, 'compression']])
|
||||
if (arguments->size() < 4)
|
||||
return npos;
|
||||
return;
|
||||
|
||||
return {2, 3};
|
||||
markSecretArgument(2);
|
||||
}
|
||||
|
||||
std::pair<size_t, size_t> findDatabaseEngineSecretArguments() const
|
||||
void findDatabaseEngineSecretArguments()
|
||||
{
|
||||
const String & engine_name = function.name;
|
||||
if ((engine_name == "MySQL") || (engine_name == "MaterializeMySQL") ||
|
||||
@ -335,31 +400,71 @@ namespace
|
||||
(engine_name == "MaterializedPostgreSQL"))
|
||||
{
|
||||
/// MySQL('host:port', 'database', 'user', 'password')
|
||||
/// PostgreSQL('host:port', 'database', 'user', 'password', ...)
|
||||
return {3, 4};
|
||||
}
|
||||
else
|
||||
{
|
||||
return npos;
|
||||
/// PostgreSQL('host:port', 'database', 'user', 'password')
|
||||
findMySQLDatabaseSecretArguments();
|
||||
}
|
||||
}
|
||||
|
||||
std::pair<size_t, size_t> findBackupNameSecretArguments() const
|
||||
void findMySQLDatabaseSecretArguments()
|
||||
{
|
||||
if (isNamedCollectionName(0))
|
||||
{
|
||||
/// MySQL(named_collection, ..., password = 'password', ...)
|
||||
findSecretNamedArgument("password", 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
/// MySQL('host:port', 'database', 'user', 'password')
|
||||
markSecretArgument(3);
|
||||
}
|
||||
}
|
||||
|
||||
void findBackupNameSecretArguments()
|
||||
{
|
||||
const String & engine_name = function.name;
|
||||
if (engine_name == "S3")
|
||||
{
|
||||
/// BACKUP ... TO S3(url, [aws_access_key_id, aws_secret_access_key])
|
||||
return {2, 3};
|
||||
}
|
||||
else
|
||||
{
|
||||
return npos;
|
||||
markSecretArgument(2);
|
||||
}
|
||||
}
|
||||
|
||||
const ASTFunction & function;
|
||||
const ASTs * arguments = nullptr;
|
||||
/// Whether a specified argument can be the name of a named collection?
|
||||
bool isNamedCollectionName(size_t arg_idx) const
|
||||
{
|
||||
if (arguments->size() <= arg_idx)
|
||||
return false;
|
||||
|
||||
const auto * identifier = (*arguments)[arg_idx]->as<ASTIdentifier>();
|
||||
return identifier != nullptr;
|
||||
}
|
||||
|
||||
/// Looks for a secret argument with a specified name. This function looks for arguments in format `key=value` where the key is specified.
|
||||
void findSecretNamedArgument(const std::string_view & key, size_t start = 0)
|
||||
{
|
||||
for (size_t i = start; i < arguments->size(); ++i)
|
||||
{
|
||||
const auto & argument = (*arguments)[i];
|
||||
const auto * equals_func = argument->as<ASTFunction>();
|
||||
if (!equals_func || (equals_func->name != "equals"))
|
||||
continue;
|
||||
|
||||
const auto * expr_list = equals_func->arguments->as<ASTExpressionList>();
|
||||
if (!expr_list)
|
||||
continue;
|
||||
|
||||
const auto & equal_args = expr_list->children;
|
||||
if (equal_args.size() != 2)
|
||||
continue;
|
||||
|
||||
String found_key;
|
||||
if (!tryGetStringFromArgument(*equal_args[0], &found_key))
|
||||
continue;
|
||||
|
||||
if (found_key == key)
|
||||
markSecretArgument(i, /* argument_is_named= */ true);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@ -966,32 +1071,39 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format
|
||||
&& (name == "match" || name == "extract" || name == "extractAll" || name == "replaceRegexpOne"
|
||||
|| name == "replaceRegexpAll");
|
||||
|
||||
auto secret_arguments = std::make_pair(static_cast<size_t>(-1), static_cast<size_t>(-1));
|
||||
FunctionSecretArgumentsFinder::Result secret_arguments;
|
||||
if (!settings.show_secrets)
|
||||
secret_arguments = FunctionSecretArgumentsFinder(*this).getRange();
|
||||
secret_arguments = FunctionSecretArgumentsFinder{*this}.getResult();
|
||||
|
||||
for (size_t i = 0, size = arguments->children.size(); i < size; ++i)
|
||||
{
|
||||
if (i != 0)
|
||||
settings.ostr << ", ";
|
||||
if (arguments->children[i]->as<ASTSetQuery>())
|
||||
|
||||
const auto & argument = arguments->children[i];
|
||||
if (argument->as<ASTSetQuery>())
|
||||
settings.ostr << "SETTINGS ";
|
||||
|
||||
if (!settings.show_secrets && (secret_arguments.first <= i) && (i < secret_arguments.second))
|
||||
if (!settings.show_secrets && (secret_arguments.start <= i) && (i < secret_arguments.start + secret_arguments.count))
|
||||
{
|
||||
if (secret_arguments.are_named)
|
||||
{
|
||||
assert_cast<const ASTFunction *>(argument.get())->arguments->children[0]->formatImpl(settings, state, nested_dont_need_parens);
|
||||
settings.ostr << (settings.hilite ? hilite_operator : "") << " = " << (settings.hilite ? hilite_none : "");
|
||||
}
|
||||
settings.ostr << "'[HIDDEN]'";
|
||||
if (size - 1 < secret_arguments.second)
|
||||
if (size <= secret_arguments.start + secret_arguments.count && !secret_arguments.are_named)
|
||||
break; /// All other arguments should also be hidden.
|
||||
continue;
|
||||
}
|
||||
|
||||
if ((i == 1) && special_hilite_regexp
|
||||
&& highlightStringLiteralWithMetacharacters(arguments->children[i], settings, "|()^$.[]?*+{:-"))
|
||||
&& highlightStringLiteralWithMetacharacters(argument, settings, "|()^$.[]?*+{:-"))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
arguments->children[i]->formatImpl(settings, state, nested_dont_need_parens);
|
||||
argument->formatImpl(settings, state, nested_dont_need_parens);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1005,14 +1117,7 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format
|
||||
|
||||
bool ASTFunction::hasSecretParts() const
|
||||
{
|
||||
if (arguments)
|
||||
{
|
||||
size_t num_arguments = arguments->children.size();
|
||||
auto secret_arguments = FunctionSecretArgumentsFinder(*this).getRange();
|
||||
if ((secret_arguments.first < num_arguments) && (secret_arguments.first < secret_arguments.second))
|
||||
return true;
|
||||
}
|
||||
return childrenHaveSecretParts();
|
||||
return (FunctionSecretArgumentsFinder{*this}.getResult().count > 0) || childrenHaveSecretParts();
|
||||
}
|
||||
|
||||
String getFunctionName(const IAST * ast)
|
||||
|
||||
@ -17,8 +17,13 @@ public:
|
||||
ASTPtr clone() const override
|
||||
{
|
||||
auto clone = std::make_shared<ASTQualifiedAsterisk>(*this);
|
||||
clone->children.clear();
|
||||
|
||||
if (transformers) { clone->transformers = transformers->clone(); clone->children.push_back(clone->transformers); }
|
||||
if (transformers)
|
||||
{
|
||||
clone->transformers = transformers->clone();
|
||||
clone->children.push_back(clone->transformers);
|
||||
}
|
||||
|
||||
clone->qualifier = qualifier->clone();
|
||||
clone->children.push_back(clone->qualifier);
|
||||
|
||||
@ -25,7 +25,7 @@ public:
|
||||
DROP_INDEX_MARK_CACHE,
|
||||
DROP_INDEX_UNCOMPRESSED_CACHE,
|
||||
DROP_MMAP_CACHE,
|
||||
DROP_QUERY_RESULT_CACHE,
|
||||
DROP_QUERY_CACHE,
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
DROP_COMPILED_EXPRESSION_CACHE,
|
||||
#endif
|
||||
|
||||
@ -18,6 +18,7 @@ bool ParserDeleteQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
ParserKeyword s_where("WHERE");
|
||||
ParserExpression parser_exp_elem;
|
||||
ParserKeyword s_settings("SETTINGS");
|
||||
ParserKeyword s_on{"ON"};
|
||||
|
||||
if (s_delete.ignore(pos, expected))
|
||||
{
|
||||
@ -27,6 +28,14 @@ bool ParserDeleteQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
if (!parseDatabaseAndTableAsAST(pos, expected, query->database, query->table))
|
||||
return false;
|
||||
|
||||
if (s_on.ignore(pos, expected))
|
||||
{
|
||||
String cluster_str;
|
||||
if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected))
|
||||
return false;
|
||||
query->cluster = cluster_str;
|
||||
}
|
||||
|
||||
if (!s_where.ignore(pos, expected))
|
||||
return false;
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1377,8 +1377,7 @@ void Planner::buildPlanForQueryNode()
|
||||
*/
|
||||
if (query_node.hasLimit() && apply_limit && !limit_applied && apply_offset)
|
||||
addLimitStep(query_plan, query_analysis_result, planner_context, query_node);
|
||||
|
||||
if (apply_offset && query_node.hasOffset())
|
||||
else if (!limit_applied && apply_offset && query_node.hasOffset())
|
||||
addOffsetStep(query_plan, query_analysis_result);
|
||||
|
||||
const auto & projection_analysis_result = expression_analysis_result.getProjection();
|
||||
|
||||
24
src/Processors/Transforms/StreamInQueryCacheTransform.cpp
Normal file
24
src/Processors/Transforms/StreamInQueryCacheTransform.cpp
Normal file
@ -0,0 +1,24 @@
|
||||
#include <Processors/Transforms/StreamInQueryCacheTransform.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
StreamInQueryCacheTransform::StreamInQueryCacheTransform(
|
||||
const Block & header_, QueryCachePtr cache, const QueryCache::Key & cache_key, std::chrono::milliseconds min_query_duration)
|
||||
: ISimpleTransform(header_, header_, false)
|
||||
, cache_writer(cache->createWriter(cache_key, min_query_duration))
|
||||
{
|
||||
}
|
||||
|
||||
void StreamInQueryCacheTransform::transform(Chunk & chunk)
|
||||
{
|
||||
cache_writer.buffer(chunk.clone());
|
||||
}
|
||||
|
||||
void StreamInQueryCacheTransform::finalizeWriteInQueryCache()
|
||||
{
|
||||
if (!isCancelled())
|
||||
cache_writer.finalizeWrite();
|
||||
}
|
||||
|
||||
};
|
||||
26
src/Processors/Transforms/StreamInQueryCacheTransform.h
Normal file
26
src/Processors/Transforms/StreamInQueryCacheTransform.h
Normal file
@ -0,0 +1,26 @@
|
||||
#pragma once
|
||||
|
||||
#include <Processors/ISimpleTransform.h>
|
||||
#include <Interpreters/Cache/QueryCache.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class StreamInQueryCacheTransform : public ISimpleTransform
|
||||
{
|
||||
public:
|
||||
StreamInQueryCacheTransform(
|
||||
const Block & header_, QueryCachePtr cache, const QueryCache::Key & cache_key, std::chrono::milliseconds min_query_duration);
|
||||
|
||||
protected:
|
||||
void transform(Chunk & chunk) override;
|
||||
|
||||
public:
|
||||
void finalizeWriteInQueryCache();
|
||||
String getName() const override { return "StreamInQueryCacheTransform"; }
|
||||
|
||||
private:
|
||||
QueryCache::Writer cache_writer;
|
||||
};
|
||||
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
#include <Processors/Transforms/StreamInQueryResultCacheTransform.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
StreamInQueryResultCacheTransform::StreamInQueryResultCacheTransform(
|
||||
const Block & header_, QueryResultCachePtr cache, const QueryResultCache::Key & cache_key, std::chrono::milliseconds min_query_duration)
|
||||
: ISimpleTransform(header_, header_, false)
|
||||
, cache_writer(cache->createWriter(cache_key, min_query_duration))
|
||||
{
|
||||
}
|
||||
|
||||
void StreamInQueryResultCacheTransform::transform(Chunk & chunk)
|
||||
{
|
||||
cache_writer.buffer(chunk.clone());
|
||||
}
|
||||
|
||||
void StreamInQueryResultCacheTransform::finalizeWriteInQueryResultCache()
|
||||
{
|
||||
if (!isCancelled())
|
||||
cache_writer.finalizeWrite();
|
||||
}
|
||||
|
||||
};
|
||||
@ -1,26 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <Processors/ISimpleTransform.h>
|
||||
#include <Interpreters/Cache/QueryResultCache.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class StreamInQueryResultCacheTransform : public ISimpleTransform
|
||||
{
|
||||
public:
|
||||
StreamInQueryResultCacheTransform(
|
||||
const Block & header_, QueryResultCachePtr cache, const QueryResultCache::Key & cache_key, std::chrono::milliseconds min_query_duration);
|
||||
|
||||
protected:
|
||||
void transform(Chunk & chunk) override;
|
||||
|
||||
public:
|
||||
void finalizeWriteInQueryResultCache();
|
||||
String getName() const override { return "StreamInQueryResultCacheTransform"; }
|
||||
|
||||
private:
|
||||
QueryResultCache::Writer cache_writer;
|
||||
};
|
||||
|
||||
}
|
||||
@ -18,7 +18,7 @@
|
||||
#include <Processors/Transforms/LimitsCheckingTransform.h>
|
||||
#include <Processors/Transforms/MaterializingTransform.h>
|
||||
#include <Processors/Transforms/PartialSortingTransform.h>
|
||||
#include <Processors/Transforms/StreamInQueryResultCacheTransform.h>
|
||||
#include <Processors/Transforms/StreamInQueryCacheTransform.h>
|
||||
#include <Processors/Transforms/ExpressionTransform.h>
|
||||
#include <Processors/QueryPlan/ReadFromPreparedSource.h>
|
||||
|
||||
@ -525,7 +525,7 @@ bool QueryPipeline::tryGetResultRowsAndBytes(UInt64 & result_rows, UInt64 & resu
|
||||
return true;
|
||||
}
|
||||
|
||||
void QueryPipeline::streamIntoQueryResultCache(std::shared_ptr<StreamInQueryResultCacheTransform> transform)
|
||||
void QueryPipeline::streamIntoQueryCache(std::shared_ptr<StreamInQueryCacheTransform> transform)
|
||||
{
|
||||
assert(pulling());
|
||||
|
||||
@ -534,16 +534,16 @@ void QueryPipeline::streamIntoQueryResultCache(std::shared_ptr<StreamInQueryResu
|
||||
processors->emplace_back(transform);
|
||||
}
|
||||
|
||||
void QueryPipeline::finalizeWriteInQueryResultCache()
|
||||
void QueryPipeline::finalizeWriteInQueryCache()
|
||||
{
|
||||
auto it = std::find_if(
|
||||
processors->begin(), processors->end(),
|
||||
[](ProcessorPtr processor){ return dynamic_cast<StreamInQueryResultCacheTransform *>(&*processor); });
|
||||
[](ProcessorPtr processor){ return dynamic_cast<StreamInQueryCacheTransform *>(&*processor); });
|
||||
|
||||
/// the pipeline should theoretically contain just one StreamInQueryResultCacheTransform
|
||||
/// the pipeline should theoretically contain just one StreamInQueryCacheTransform
|
||||
|
||||
if (it != processors->end())
|
||||
dynamic_cast<StreamInQueryResultCacheTransform &>(**it).finalizeWriteInQueryResultCache();
|
||||
dynamic_cast<StreamInQueryCacheTransform &>(**it).finalizeWriteInQueryCache();
|
||||
}
|
||||
|
||||
void QueryPipeline::addStorageHolder(StoragePtr storage)
|
||||
|
||||
@ -31,7 +31,7 @@ class SinkToStorage;
|
||||
class ISource;
|
||||
class ISink;
|
||||
class ReadProgressCallback;
|
||||
class StreamInQueryResultCacheTransform;
|
||||
class StreamInQueryCacheTransform;
|
||||
|
||||
struct ColumnWithTypeAndName;
|
||||
using ColumnsWithTypeAndName = std::vector<ColumnWithTypeAndName>;
|
||||
@ -105,8 +105,8 @@ public:
|
||||
void setLimitsAndQuota(const StreamLocalLimits & limits, std::shared_ptr<const EnabledQuota> quota_);
|
||||
bool tryGetResultRowsAndBytes(UInt64 & result_rows, UInt64 & result_bytes) const;
|
||||
|
||||
void streamIntoQueryResultCache(std::shared_ptr<StreamInQueryResultCacheTransform> transform);
|
||||
void finalizeWriteInQueryResultCache();
|
||||
void streamIntoQueryCache(std::shared_ptr<StreamInQueryCacheTransform> transform);
|
||||
void finalizeWriteInQueryCache();
|
||||
|
||||
void setQuota(std::shared_ptr<const EnabledQuota> quota_);
|
||||
|
||||
|
||||
@ -8,6 +8,7 @@
|
||||
#include <Client/Connection.h>
|
||||
#include <Core/QueryProcessingStage.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <IO/ConnectionTimeoutsContext.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Interpreters/getHeaderForProcessingStage.h>
|
||||
#include <Interpreters/SelectQueryOptions.h>
|
||||
|
||||
@ -487,7 +487,7 @@ public:
|
||||
}
|
||||
|
||||
/// Mutate the table contents
|
||||
virtual void mutate(const MutationCommands &, ContextPtr, bool /*force_wait*/)
|
||||
virtual void mutate(const MutationCommands &, ContextPtr)
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Mutations are not supported by storage {}", getName());
|
||||
}
|
||||
|
||||
@ -165,7 +165,7 @@ MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withPartFormatFromVolume()
|
||||
if (!storage || !mark_type)
|
||||
{
|
||||
/// Didn't find any data or mark file, suppose that part is empty.
|
||||
return withBytesAndRows(0, 0);
|
||||
return withBytesAndRowsOnDisk(0, 0);
|
||||
}
|
||||
|
||||
part_storage = std::move(storage);
|
||||
@ -181,7 +181,7 @@ MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withPartFormatFromStorage()
|
||||
if (!mark_type)
|
||||
{
|
||||
/// Didn't find any mark file, suppose that part is empty.
|
||||
return withBytesAndRows(0, 0);
|
||||
return withBytesAndRowsOnDisk(0, 0);
|
||||
}
|
||||
|
||||
part_type = mark_type->part_type;
|
||||
|
||||
@ -217,7 +217,7 @@ void StorageEmbeddedRocksDB::checkMutationIsPossible(const MutationCommands & co
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Only DELETE and UPDATE mutation supported for EmbeddedRocksDB");
|
||||
}
|
||||
|
||||
void StorageEmbeddedRocksDB::mutate(const MutationCommands & commands, ContextPtr context_, bool /*force_wait*/)
|
||||
void StorageEmbeddedRocksDB::mutate(const MutationCommands & commands, ContextPtr context_)
|
||||
{
|
||||
if (commands.empty())
|
||||
return;
|
||||
|
||||
@ -52,7 +52,7 @@ public:
|
||||
void truncate(const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, ContextPtr, TableExclusiveLockHolder &) override;
|
||||
|
||||
void checkMutationIsPossible(const MutationCommands & commands, const Settings & settings) const override;
|
||||
void mutate(const MutationCommands &, ContextPtr, bool) override;
|
||||
void mutate(const MutationCommands &, ContextPtr) override;
|
||||
|
||||
bool supportsParallelInsert() const override { return true; }
|
||||
bool supportsIndexForIn() const override { return true; }
|
||||
|
||||
@ -108,7 +108,7 @@ void StorageJoin::checkMutationIsPossible(const MutationCommands & commands, con
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Table engine Join supports only DELETE mutations");
|
||||
}
|
||||
|
||||
void StorageJoin::mutate(const MutationCommands & commands, ContextPtr context, bool /*force_wait*/)
|
||||
void StorageJoin::mutate(const MutationCommands & commands, ContextPtr context)
|
||||
{
|
||||
/// Firstly acquire lock for mutation, that locks changes of data.
|
||||
/// We cannot acquire rwlock here, because read lock is needed
|
||||
|
||||
@ -45,7 +45,7 @@ public:
|
||||
|
||||
/// Only delete is supported.
|
||||
void checkMutationIsPossible(const MutationCommands & commands, const Settings & settings) const override;
|
||||
void mutate(const MutationCommands & commands, ContextPtr context, bool force_wait) override;
|
||||
void mutate(const MutationCommands & commands, ContextPtr context) override;
|
||||
|
||||
/// Return instance of HashJoin holding lock that protects from insertions to StorageJoin.
|
||||
/// HashJoin relies on structure of hash table that's why we need to return it with locked mutex.
|
||||
|
||||
@ -319,10 +319,10 @@ void StorageMaterializedView::checkAlterPartitionIsPossible(
|
||||
getTargetTable()->checkAlterPartitionIsPossible(commands, metadata_snapshot, settings);
|
||||
}
|
||||
|
||||
void StorageMaterializedView::mutate(const MutationCommands & commands, ContextPtr local_context, bool force_wait)
|
||||
void StorageMaterializedView::mutate(const MutationCommands & commands, ContextPtr local_context)
|
||||
{
|
||||
checkStatementCanBeForwarded();
|
||||
getTargetTable()->mutate(commands, local_context, force_wait);
|
||||
getTargetTable()->mutate(commands, local_context);
|
||||
}
|
||||
|
||||
void StorageMaterializedView::renameInMemory(const StorageID & new_table_id)
|
||||
|
||||
@ -65,7 +65,7 @@ public:
|
||||
|
||||
void checkAlterPartitionIsPossible(const PartitionCommands & commands, const StorageMetadataPtr & metadata_snapshot, const Settings & settings) const override;
|
||||
|
||||
void mutate(const MutationCommands & commands, ContextPtr context, bool force_wait) override;
|
||||
void mutate(const MutationCommands & commands, ContextPtr context) override;
|
||||
|
||||
void renameInMemory(const StorageID & new_table_id) override;
|
||||
|
||||
|
||||
@ -305,7 +305,7 @@ void StorageMemory::checkMutationIsPossible(const MutationCommands & /*commands*
|
||||
/// Some validation will be added
|
||||
}
|
||||
|
||||
void StorageMemory::mutate(const MutationCommands & commands, ContextPtr context, bool /*force_wait*/)
|
||||
void StorageMemory::mutate(const MutationCommands & commands, ContextPtr context)
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
auto metadata_snapshot = getInMemoryMetadataPtr();
|
||||
|
||||
@ -67,7 +67,7 @@ public:
|
||||
void drop() override;
|
||||
|
||||
void checkMutationIsPossible(const MutationCommands & commands, const Settings & settings) const override;
|
||||
void mutate(const MutationCommands & commands, ContextPtr context, bool force_wait) override;
|
||||
void mutate(const MutationCommands & commands, ContextPtr context) override;
|
||||
|
||||
void truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder &) override;
|
||||
|
||||
|
||||
@ -532,14 +532,14 @@ void StorageMergeTree::setMutationCSN(const String & mutation_id, CSN csn)
|
||||
it->second.writeCSN(csn);
|
||||
}
|
||||
|
||||
void StorageMergeTree::mutate(const MutationCommands & commands, ContextPtr query_context, bool force_wait)
|
||||
void StorageMergeTree::mutate(const MutationCommands & commands, ContextPtr query_context)
|
||||
{
|
||||
/// Validate partition IDs (if any) before starting mutation
|
||||
getPartitionIdsAffectedByCommands(commands, query_context);
|
||||
|
||||
Int64 version = startMutation(commands, query_context);
|
||||
|
||||
if (force_wait || query_context->getSettingsRef().mutations_sync > 0 || query_context->getCurrentTransaction())
|
||||
if (query_context->getSettingsRef().mutations_sync > 0 || query_context->getCurrentTransaction())
|
||||
waitForMutation(version);
|
||||
}
|
||||
|
||||
|
||||
@ -85,7 +85,7 @@ public:
|
||||
const Names & deduplicate_by_columns,
|
||||
ContextPtr context) override;
|
||||
|
||||
void mutate(const MutationCommands & commands, ContextPtr context, bool force_wait) override;
|
||||
void mutate(const MutationCommands & commands, ContextPtr context) override;
|
||||
|
||||
bool hasLightweightDeletedMask() const override;
|
||||
|
||||
|
||||
@ -132,7 +132,7 @@ public:
|
||||
return getNested()->optimize(query, metadata_snapshot, partition, final, deduplicate, deduplicate_by_columns, context);
|
||||
}
|
||||
|
||||
void mutate(const MutationCommands & commands, ContextPtr context, bool force_wait) override { getNested()->mutate(commands, context, force_wait); }
|
||||
void mutate(const MutationCommands & commands, ContextPtr context) override { getNested()->mutate(commands, context); }
|
||||
|
||||
CancellationCode killMutation(const String & mutation_id) override { return getNested()->killMutation(mutation_id); }
|
||||
|
||||
|
||||
@ -6276,7 +6276,7 @@ void StorageReplicatedMergeTree::fetchPartition(
|
||||
}
|
||||
|
||||
|
||||
void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, ContextPtr query_context, bool force_wait)
|
||||
void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, ContextPtr query_context)
|
||||
{
|
||||
/// Overview of the mutation algorithm.
|
||||
///
|
||||
@ -6390,8 +6390,7 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, Conte
|
||||
throw Coordination::Exception("Unable to create a mutation znode", rc);
|
||||
}
|
||||
|
||||
const size_t mutations_sync = force_wait ? 2 : query_context->getSettingsRef().mutations_sync;
|
||||
waitMutation(mutation_entry.znode_name, mutations_sync);
|
||||
waitMutation(mutation_entry.znode_name, query_context->getSettingsRef().mutations_sync);
|
||||
}
|
||||
|
||||
void StorageReplicatedMergeTree::waitMutation(const String & znode_name, size_t mutations_sync) const
|
||||
|
||||
@ -153,7 +153,7 @@ public:
|
||||
|
||||
void alter(const AlterCommands & commands, ContextPtr query_context, AlterLockHolder & table_lock_holder) override;
|
||||
|
||||
void mutate(const MutationCommands & commands, ContextPtr context, bool force_wait) override;
|
||||
void mutate(const MutationCommands & commands, ContextPtr context) override;
|
||||
void waitMutation(const String & znode_name, size_t mutations_sync) const;
|
||||
std::vector<MergeTreeMutationStatus> getMutationsStatus() const override;
|
||||
CancellationCode killMutation(const String & mutation_id) override;
|
||||
|
||||
@ -8,6 +8,7 @@
|
||||
#include "Client/Connection.h"
|
||||
#include "Core/QueryProcessingStage.h"
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <IO/ConnectionTimeoutsContext.h>
|
||||
#include <IO/WriteBufferFromS3.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
@ -1,15 +1,15 @@
|
||||
#include "StorageSystemQueryResultCache.h"
|
||||
#include "StorageSystemQueryCache.h"
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Interpreters/Cache/QueryResultCache.h>
|
||||
#include <Interpreters/Cache/QueryCache.h>
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
NamesAndTypesList StorageSystemQueryResultCache::getNamesAndTypes()
|
||||
NamesAndTypesList StorageSystemQueryCache::getNamesAndTypes()
|
||||
{
|
||||
return {
|
||||
{"query", std::make_shared<DataTypeString>()},
|
||||
@ -21,23 +21,23 @@ NamesAndTypesList StorageSystemQueryResultCache::getNamesAndTypes()
|
||||
};
|
||||
}
|
||||
|
||||
StorageSystemQueryResultCache::StorageSystemQueryResultCache(const StorageID & table_id_)
|
||||
StorageSystemQueryCache::StorageSystemQueryCache(const StorageID & table_id_)
|
||||
: IStorageSystemOneBlock(table_id_)
|
||||
{
|
||||
}
|
||||
|
||||
void StorageSystemQueryResultCache::fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo &) const
|
||||
void StorageSystemQueryCache::fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo &) const
|
||||
{
|
||||
auto query_result_cache = context->getQueryResultCache();
|
||||
auto query_cache = context->getQueryCache();
|
||||
|
||||
if (!query_result_cache)
|
||||
if (!query_cache)
|
||||
return;
|
||||
|
||||
const String & username = context->getUserName();
|
||||
|
||||
std::lock_guard lock(query_result_cache->mutex);
|
||||
std::lock_guard lock(query_cache->mutex);
|
||||
|
||||
for (const auto & [key, result] : query_result_cache->cache)
|
||||
for (const auto & [key, result] : query_cache->cache)
|
||||
{
|
||||
/// Showing other user's queries is considered a security risk
|
||||
if (key.username.has_value() && key.username != username)
|
||||
@ -5,12 +5,12 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class StorageSystemQueryResultCache final : public IStorageSystemOneBlock<StorageSystemQueryResultCache>
|
||||
class StorageSystemQueryCache final : public IStorageSystemOneBlock<StorageSystemQueryCache>
|
||||
{
|
||||
public:
|
||||
explicit StorageSystemQueryResultCache(const StorageID & table_id_);
|
||||
explicit StorageSystemQueryCache(const StorageID & table_id_);
|
||||
|
||||
std::string getName() const override { return "SystemQueryResultCache"; }
|
||||
std::string getName() const override { return "SystemQueryCache"; }
|
||||
|
||||
static NamesAndTypesList getNamesAndTypes();
|
||||
|
||||
@ -73,7 +73,7 @@
|
||||
#include <Storages/System/StorageSystemAsynchronousInserts.h>
|
||||
#include <Storages/System/StorageSystemTransactions.h>
|
||||
#include <Storages/System/StorageSystemFilesystemCache.h>
|
||||
#include <Storages/System/StorageSystemQueryResultCache.h>
|
||||
#include <Storages/System/StorageSystemQueryCache.h>
|
||||
#include <Storages/System/StorageSystemNamedCollections.h>
|
||||
#include <Storages/System/StorageSystemRemoteDataPaths.h>
|
||||
#include <Storages/System/StorageSystemCertificates.h>
|
||||
@ -176,7 +176,7 @@ void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, b
|
||||
attach<StorageSystemPartMovesBetweenShards>(context, system_database, "part_moves_between_shards");
|
||||
attach<StorageSystemAsynchronousInserts>(context, system_database, "asynchronous_inserts");
|
||||
attach<StorageSystemFilesystemCache>(context, system_database, "filesystem_cache");
|
||||
attach<StorageSystemQueryResultCache>(context, system_database, "query_result_cache");
|
||||
attach<StorageSystemQueryCache>(context, system_database, "query_cache");
|
||||
attach<StorageSystemRemoteDataPaths>(context, system_database, "remote_data_paths");
|
||||
attach<StorageSystemCertificates>(context, system_database, "certificates");
|
||||
attach<StorageSystemNamedCollections>(context, system_database, "named_collections");
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Interpreters/evaluateConstantExpression.h>
|
||||
#include <Interpreters/parseColumnsListForTableFunction.h>
|
||||
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
|
||||
@ -38,23 +39,29 @@ void TableFunctionFormat::parseArguments(const ASTPtr & ast_function, ContextPtr
|
||||
|
||||
ASTs & args = args_func.at(0)->children;
|
||||
|
||||
if (args.size() != 2)
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' requires 2 arguments: format and data", getName());
|
||||
if (args.size() != 2 && args.size() != 3)
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' requires 2 or 3 arguments: format, [structure], data", getName());
|
||||
|
||||
for (auto & arg : args)
|
||||
arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context);
|
||||
|
||||
format = checkAndGetLiteralArgument<String>(args[0], "format");
|
||||
data = checkAndGetLiteralArgument<String>(args[1], "data");
|
||||
data = checkAndGetLiteralArgument<String>(args.back(), "data");
|
||||
if (args.size() == 3)
|
||||
structure = checkAndGetLiteralArgument<String>(args[1], "structure");
|
||||
}
|
||||
|
||||
ColumnsDescription TableFunctionFormat::getActualTableStructure(ContextPtr context) const
|
||||
{
|
||||
ReadBufferIterator read_buffer_iterator = [&](ColumnsDescription &)
|
||||
if (structure == "auto")
|
||||
{
|
||||
return std::make_unique<ReadBufferFromString>(data);
|
||||
};
|
||||
return readSchemaFromFormat(format, std::nullopt, read_buffer_iterator, false, context);
|
||||
ReadBufferIterator read_buffer_iterator = [&](ColumnsDescription &)
|
||||
{
|
||||
return std::make_unique<ReadBufferFromString>(data);
|
||||
};
|
||||
return readSchemaFromFormat(format, std::nullopt, read_buffer_iterator, false, context);
|
||||
}
|
||||
return parseColumnsListFromString(structure, context);
|
||||
}
|
||||
|
||||
Block TableFunctionFormat::parseData(ColumnsDescription columns, ContextPtr context) const
|
||||
|
||||
@ -28,6 +28,7 @@ private:
|
||||
|
||||
String format;
|
||||
String data;
|
||||
String structure = "auto";
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -183,6 +183,7 @@ def prepare_tests_results_for_clickhouse(
|
||||
current_row["test_duration_ms"] = int(test_time * 1000)
|
||||
current_row["test_name"] = test_name
|
||||
current_row["test_status"] = test_status
|
||||
current_row["test_context_raw"] = test_result.raw_logs or ""
|
||||
result.append(current_row)
|
||||
|
||||
return result
|
||||
|
||||
@ -211,7 +211,7 @@ def read_test_results(results_path: Path, with_raw_logs: bool = True) -> TestRes
|
||||
name = line[0]
|
||||
status = line[1]
|
||||
time = None
|
||||
if len(line) >= 3 and line[2]:
|
||||
if len(line) >= 3 and line[2] and line[2] != "\\N":
|
||||
# The value can be emtpy, but when it's not,
|
||||
# it's the time spent on the test
|
||||
try:
|
||||
|
||||
@ -95,7 +95,7 @@ def process_results(
|
||||
|
||||
try:
|
||||
results_path = Path(result_folder) / "test_results.tsv"
|
||||
test_results = read_test_results(results_path, False)
|
||||
test_results = read_test_results(results_path, True)
|
||||
if len(test_results) == 0:
|
||||
raise Exception("Empty results")
|
||||
except Exception as e:
|
||||
|
||||
@ -0,0 +1,10 @@
|
||||
<clickhouse>
|
||||
<named_collections>
|
||||
<named_collection_1/>
|
||||
<named_collection_2/>
|
||||
<named_collection_3/>
|
||||
<named_collection_4/>
|
||||
<named_collection_5/>
|
||||
<named_collection_6/>
|
||||
</named_collections>
|
||||
</clickhouse>
|
||||
@ -4,7 +4,13 @@ from helpers.cluster import ClickHouseCluster
|
||||
from helpers.test_tools import TSV
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
node = cluster.add_instance("node", with_zookeeper=True)
|
||||
node = cluster.add_instance(
|
||||
"node",
|
||||
main_configs=[
|
||||
"configs/named_collections.xml",
|
||||
],
|
||||
with_zookeeper=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
@ -116,6 +122,12 @@ def test_create_table():
|
||||
f"S3('http://minio1:9001/root/data/test3.csv.gz', 'CSV', 'gzip')",
|
||||
f"S3('http://minio1:9001/root/data/test4.csv', 'minio', '{password}', 'CSV')",
|
||||
f"S3('http://minio1:9001/root/data/test5.csv.gz', 'minio', '{password}', 'CSV', 'gzip')",
|
||||
f"MySQL(named_collection_1, host = 'mysql57', port = 3306, database = 'mysql_db', table = 'mysql_table', user = 'mysql_user', password = '{password}')",
|
||||
f"MySQL(named_collection_2, database = 'mysql_db', host = 'mysql57', port = 3306, password = '{password}', table = 'mysql_table', user = 'mysql_user')",
|
||||
f"MySQL(named_collection_3, database = 'mysql_db', host = 'mysql57', port = 3306, table = 'mysql_table')",
|
||||
f"PostgreSQL(named_collection_4, host = 'postgres1', port = 5432, database = 'postgres_db', table = 'postgres_table', user = 'postgres_user', password = '{password}')",
|
||||
f"MongoDB(named_collection_5, host = 'mongo1', port = 5432, database = 'mongo_db', collection = 'mongo_col', user = 'mongo_user', password = '{password}')",
|
||||
f"S3(named_collection_6, url = 'http://minio1:9001/root/data/test8.csv', access_key_id = 'minio', secret_access_key = '{password}', format = 'CSV')",
|
||||
]
|
||||
|
||||
for i, table_engine in enumerate(table_engines):
|
||||
@ -147,6 +159,12 @@ def test_create_table():
|
||||
"CREATE TABLE table5 (x int) ENGINE = S3('http://minio1:9001/root/data/test3.csv.gz', 'CSV', 'gzip')",
|
||||
"CREATE TABLE table6 (`x` int) ENGINE = S3('http://minio1:9001/root/data/test4.csv', 'minio', '[HIDDEN]', 'CSV')",
|
||||
"CREATE TABLE table7 (`x` int) ENGINE = S3('http://minio1:9001/root/data/test5.csv.gz', 'minio', '[HIDDEN]', 'CSV', 'gzip')",
|
||||
"CREATE TABLE table8 (`x` int) ENGINE = MySQL(named_collection_1, host = 'mysql57', port = 3306, database = 'mysql_db', table = 'mysql_table', user = 'mysql_user', password = '[HIDDEN]')",
|
||||
"CREATE TABLE table9 (`x` int) ENGINE = MySQL(named_collection_2, database = 'mysql_db', host = 'mysql57', port = 3306, password = '[HIDDEN]', table = 'mysql_table', user = 'mysql_user')",
|
||||
"CREATE TABLE table10 (x int) ENGINE = MySQL(named_collection_3, database = 'mysql_db', host = 'mysql57', port = 3306, table = 'mysql_table')",
|
||||
"CREATE TABLE table11 (`x` int) ENGINE = PostgreSQL(named_collection_4, host = 'postgres1', port = 5432, database = 'postgres_db', table = 'postgres_table', user = 'postgres_user', password = '[HIDDEN]')",
|
||||
"CREATE TABLE table12 (`x` int) ENGINE = MongoDB(named_collection_5, host = 'mongo1', port = 5432, database = 'mongo_db', collection = 'mongo_col', user = 'mongo_user', password = '[HIDDEN]'",
|
||||
"CREATE TABLE table13 (`x` int) ENGINE = S3(named_collection_6, url = 'http://minio1:9001/root/data/test8.csv', access_key_id = 'minio', secret_access_key = '[HIDDEN]', format = 'CSV')",
|
||||
],
|
||||
must_not_contain=[password],
|
||||
)
|
||||
@ -160,6 +178,7 @@ def test_create_database():
|
||||
|
||||
database_engines = [
|
||||
f"MySQL('localhost:3306', 'mysql_db', 'mysql_user', '{password}') SETTINGS connect_timeout=1, connection_max_tries=1",
|
||||
f"MySQL(named_collection_1, host = 'localhost', port = 3306, database = 'mysql_db', user = 'mysql_user', password = '{password}') SETTINGS connect_timeout=1, connection_max_tries=1",
|
||||
# f"PostgreSQL('localhost:5432', 'postgres_db', 'postgres_user', '{password}')",
|
||||
]
|
||||
|
||||
@ -173,7 +192,8 @@ def test_create_database():
|
||||
check_logs(
|
||||
must_contain=[
|
||||
"CREATE DATABASE database0 ENGINE = MySQL('localhost:3306', 'mysql_db', 'mysql_user', '[HIDDEN]')",
|
||||
# "CREATE DATABASE database1 ENGINE = PostgreSQL('localhost:5432', 'postgres_db', 'postgres_user', '[HIDDEN]')",
|
||||
"CREATE DATABASE database1 ENGINE = MySQL(named_collection_1, host = 'localhost', port = 3306, database = 'mysql_db', user = 'mysql_user', password = '[HIDDEN]')",
|
||||
# "CREATE DATABASE database2 ENGINE = PostgreSQL('localhost:5432', 'postgres_db', 'postgres_user', '[HIDDEN]')",
|
||||
],
|
||||
must_not_contain=[password],
|
||||
)
|
||||
@ -211,6 +231,11 @@ def test_table_functions():
|
||||
f"remote('127.{{2..11}}', numbers(10), 'remote_user', '{password}', rand())",
|
||||
f"remoteSecure('127.{{2..11}}', 'default', 'remote_table', 'remote_user', '{password}')",
|
||||
f"remoteSecure('127.{{2..11}}', 'default', 'remote_table', 'remote_user', rand())",
|
||||
f"mysql(named_collection_1, host = 'mysql57', port = 3306, database = 'mysql_db', table = 'mysql_table', user = 'mysql_user', password = '{password}')",
|
||||
f"postgresql(named_collection_2, password = '{password}', host = 'postgres1', port = 5432, database = 'postgres_db', table = 'postgres_table', user = 'postgres_user')",
|
||||
f"s3(named_collection_3, url = 'http://minio1:9001/root/data/test4.csv', access_key_id = 'minio', secret_access_key = '{password}')",
|
||||
f"remote(named_collection_4, addresses_expr = '127.{{2..11}}', database = 'default', table = 'remote_table', user = 'remote_user', password = '{password}', sharding_key = rand())",
|
||||
f"remoteSecure(named_collection_5, addresses_expr = '127.{{2..11}}', database = 'default', table = 'remote_table', user = 'remote_user', password = '{password}')",
|
||||
]
|
||||
|
||||
for i, table_function in enumerate(table_functions):
|
||||
@ -259,6 +284,11 @@ def test_table_functions():
|
||||
"CREATE TABLE tablefunc22 (`x` int) AS remote('127.{2..11}', numbers(10), 'remote_user', '[HIDDEN]', rand())",
|
||||
"CREATE TABLE tablefunc23 (`x` int) AS remoteSecure('127.{2..11}', 'default', 'remote_table', 'remote_user', '[HIDDEN]')",
|
||||
"CREATE TABLE tablefunc24 (x int) AS remoteSecure('127.{2..11}', 'default', 'remote_table', 'remote_user', rand())",
|
||||
"CREATE TABLE tablefunc25 (`x` int) AS mysql(named_collection_1, host = 'mysql57', port = 3306, database = 'mysql_db', table = 'mysql_table', user = 'mysql_user', password = '[HIDDEN]')",
|
||||
"CREATE TABLE tablefunc26 (`x` int) AS postgresql(named_collection_2, password = '[HIDDEN]', host = 'postgres1', port = 5432, database = 'postgres_db', table = 'postgres_table', user = 'postgres_user')",
|
||||
"CREATE TABLE tablefunc27 (`x` int) AS s3(named_collection_3, url = 'http://minio1:9001/root/data/test4.csv', access_key_id = 'minio', secret_access_key = '[HIDDEN]')",
|
||||
"CREATE TABLE tablefunc28 (`x` int) AS remote(named_collection_4, addresses_expr = '127.{2..11}', database = 'default', table = 'remote_table', user = 'remote_user', password = '[HIDDEN]', sharding_key = rand())",
|
||||
"CREATE TABLE tablefunc29 (`x` int) AS remoteSecure(named_collection_5, addresses_expr = '127.{2..11}', database = 'default', table = 'remote_table', user = 'remote_user', password = '[HIDDEN]')",
|
||||
],
|
||||
must_not_contain=[password],
|
||||
)
|
||||
|
||||
@ -212,6 +212,48 @@ def test_simple_alter_table(started_cluster, engine):
|
||||
competing_node.query("DROP DATABASE testdb SYNC")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"])
|
||||
def test_delete_from_table(started_cluster, engine):
|
||||
main_node.query(
|
||||
"CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');"
|
||||
)
|
||||
dummy_node.query(
|
||||
"CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard2', 'replica1');"
|
||||
)
|
||||
|
||||
name = "testdb.delete_test_{}".format(engine)
|
||||
main_node.query(
|
||||
"CREATE TABLE {} "
|
||||
"(id UInt64, value String) "
|
||||
"ENGINE = {} PARTITION BY id%2 ORDER BY (id);".format(name, engine)
|
||||
)
|
||||
main_node.query("INSERT INTO TABLE {} VALUES(1, 'aaaa');".format(name))
|
||||
main_node.query("INSERT INTO TABLE {} VALUES(2, 'aaaa');".format(name))
|
||||
dummy_node.query("INSERT INTO TABLE {} VALUES(1, 'bbbb');".format(name))
|
||||
dummy_node.query("INSERT INTO TABLE {} VALUES(2, 'bbbb');".format(name))
|
||||
|
||||
main_node.query(
|
||||
"SET allow_experimental_lightweight_delete=1; DELETE FROM {} WHERE id=2;".format(
|
||||
name
|
||||
)
|
||||
)
|
||||
|
||||
expected = "1\taaaa\n1\tbbbb"
|
||||
|
||||
table_for_select = name
|
||||
if not "Replicated" in engine:
|
||||
table_for_select = "cluster('testdb', {})".format(name)
|
||||
for node in [main_node, dummy_node]:
|
||||
assert_eq_with_retry(
|
||||
node,
|
||||
"SELECT * FROM {} ORDER BY id, value;".format(table_for_select),
|
||||
expected,
|
||||
)
|
||||
|
||||
main_node.query("DROP DATABASE testdb SYNC")
|
||||
dummy_node.query("DROP DATABASE testdb SYNC")
|
||||
|
||||
|
||||
def get_table_uuid(database, name):
|
||||
return main_node.query(
|
||||
f"SELECT uuid FROM system.tables WHERE database = '{database}' and name = '{name}'"
|
||||
|
||||
@ -6,4 +6,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
|
||||
echo -ne '1,Hello\n2,World\n' | ${CLICKHOUSE_CURL} -sSF 'file=@-' "${CLICKHOUSE_URL}&query=SELECT+*+FROM+file&file_format=CSV&file_types=UInt8,String";
|
||||
echo -ne '1@Hello\n2@World\n' | ${CLICKHOUSE_CURL} -sSF 'file=@-' "${CLICKHOUSE_URL}&query=SELECT+*+FROM+file&file_format=CSV&file_types=UInt8,String&format_csv_delimiter=@";
|
||||
echo -ne '\x01\x00\x00\x00\x02\x00\x00\x00' | ${CLICKHOUSE_CURL} -sSF "tmp=@-" "${CLICKHOUSE_URL}&query=SELECT+*+FROM+tmp&tmp_structure=TaskID+UInt32&tmp_format=RowBinary";
|
||||
|
||||
# use big-endian version of binary data for s390x
|
||||
if [[ $(uname -a | grep s390x) ]]; then
|
||||
echo -ne '\x00\x00\x00\x01\x00\x00\x00\x02' | ${CLICKHOUSE_CURL} -sSF "tmp=@-" "${CLICKHOUSE_URL}&query=SELECT+*+FROM+tmp&tmp_structure=TaskID+UInt32&tmp_format=RowBinary";
|
||||
else
|
||||
echo -ne '\x01\x00\x00\x00\x02\x00\x00\x00' | ${CLICKHOUSE_CURL} -sSF "tmp=@-" "${CLICKHOUSE_URL}&query=SELECT+*+FROM+tmp&tmp_structure=TaskID+UInt32&tmp_format=RowBinary";
|
||||
fi
|
||||
|
||||
@ -96,7 +96,7 @@ SYSTEM DROP DNS CACHE ['SYSTEM DROP DNS','DROP DNS CACHE','DROP DNS'] GLOBAL SYS
|
||||
SYSTEM DROP MARK CACHE ['SYSTEM DROP MARK','DROP MARK CACHE','DROP MARKS'] GLOBAL SYSTEM DROP CACHE
|
||||
SYSTEM DROP UNCOMPRESSED CACHE ['SYSTEM DROP UNCOMPRESSED','DROP UNCOMPRESSED CACHE','DROP UNCOMPRESSED'] GLOBAL SYSTEM DROP CACHE
|
||||
SYSTEM DROP MMAP CACHE ['SYSTEM DROP MMAP','DROP MMAP CACHE','DROP MMAP'] GLOBAL SYSTEM DROP CACHE
|
||||
SYSTEM DROP QUERY RESULT CACHE ['SYSTEM DROP QUERY RESULT','DROP QUERY RESULT CACHE','DROP QUERY RESULT'] GLOBAL SYSTEM DROP CACHE
|
||||
SYSTEM DROP QUERY CACHE ['SYSTEM DROP QUERY','DROP QUERY CACHE','DROP QUERY'] GLOBAL SYSTEM DROP CACHE
|
||||
SYSTEM DROP COMPILED EXPRESSION CACHE ['SYSTEM DROP COMPILED EXPRESSION','DROP COMPILED EXPRESSION CACHE','DROP COMPILED EXPRESSIONS'] GLOBAL SYSTEM DROP CACHE
|
||||
SYSTEM DROP FILESYSTEM CACHE ['SYSTEM DROP FILESYSTEM CACHE','DROP FILESYSTEM CACHE'] GLOBAL SYSTEM DROP CACHE
|
||||
SYSTEM DROP SCHEMA CACHE ['SYSTEM DROP SCHEMA CACHE','DROP SCHEMA CACHE'] GLOBAL SYSTEM DROP CACHE
|
||||
|
||||
@ -1,16 +1,16 @@
|
||||
SELECT 116, 'Qqfu://2020-02-10isqkc1203 sp 2000-05-27T18:38:01', 13e100, Obsidian_id_diverTeam, sweets(Workplace), avgIf(remote('128.0.0.1'))
|
||||
SELECT treasury_mammoth_hazelnut between nutmeg and span, case when chive >= 116 then switching else null end
|
||||
SELECT 116, 'Qqfu://2020-02-10isqkc1203 sp 2000-05-27T18:38:01', 13e100, Jewelry_id_studyBeast, algebra(Stable), avgIf(remote('128.0.0.1'))
|
||||
SELECT surfboard_solitaire_crunch between understanding and populist, case when instrument >= 116 then poverty else null end
|
||||
|
||||
SELECT
|
||||
EarthquakeID,
|
||||
Workout.ID, Workout.CoupleThrill,
|
||||
MedalEMPIRE,
|
||||
HOPE.ListingName, HOPE.ListingBomb, HOPE.ListingRamen, HOPE.ListingResult, HOPE.CoupleThrill, HOPE.Smile
|
||||
FROM merge.marsh_agreement
|
||||
BugleID,
|
||||
Reliability.ID, Reliability.ExperiencePrevalence,
|
||||
DepressiveTURKEY,
|
||||
SPARK.RainmakerName, SPARK.RainmakerReligion, SPARK.RainmakerMisfit, SPARK.RainmakerAardvark, SPARK.ExperiencePrevalence, SPARK.Type
|
||||
FROM merge.invader_schizophrenic
|
||||
WHERE
|
||||
RecapitulationLeaver >= '2020-10-13' AND RecapitulationLeaver <= '2020-10-21'
|
||||
AND MasonryID = 30750384
|
||||
AND intHash32(EyeballID) = 448362928 AND intHash64(EyeballID) = 12572659331310383983
|
||||
AND EarthquakeID IN (8195672321757027078, 7079643623150622129, 5057006826979676478, 7886875230160484653, 7494974311229040743)
|
||||
AND Aide = 1
|
||||
PortraitInvasion >= '2020-10-13' AND PortraitInvasion <= '2020-10-21'
|
||||
AND FrownID = 30750384
|
||||
AND intHash32(HaversackID) = 448362928 AND intHash64(HaversackID) = 12572659331310383983
|
||||
AND BugleID IN (8195672321757027078, 7079643623150622129, 5057006826979676478, 7886875230160484653, 7494974311229040743)
|
||||
AND Hostel = 1
|
||||
|
||||
|
||||
@ -35,3 +35,10 @@ SELECT sparkbar(5,toDate('2020-01-01'),toDate('2020-01-10'))(event_date,cnt) FRO
|
||||
DROP TABLE IF EXISTS spark_bar_test;
|
||||
|
||||
WITH number DIV 50 AS k, number % 50 AS value SELECT k, sparkbar(50, 0, 99)(number, value) FROM numbers(100) GROUP BY k ORDER BY k;
|
||||
|
||||
-- OOM guard
|
||||
DROP TABLE IF EXISTS spark_bar_oom;
|
||||
CREATE TABLE spark_bar_oom (x UInt64, y UInt8) Engine=MergeTree ORDER BY tuple();
|
||||
INSERT INTO spark_bar_oom VALUES (18446744073709551615,255),(0,0),(0,0),(4036797895307271799,163);
|
||||
SELECT sparkbar(9)(x,y) FROM spark_bar_oom SETTINGS max_memory_usage = 100000000; -- { serverError 241 }
|
||||
DROP TABLE IF EXISTS spark_bar_oom;
|
||||
|
||||
@ -288,7 +288,7 @@ CREATE TABLE system.grants
|
||||
(
|
||||
`user_name` Nullable(String),
|
||||
`role_name` Nullable(String),
|
||||
`access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY RESULT CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP CACHE' = 102, 'SYSTEM RELOAD CONFIG' = 103, 'SYSTEM RELOAD USERS' = 104, 'SYSTEM RELOAD SYMBOLS' = 105, 'SYSTEM RELOAD DICTIONARY' = 106, 'SYSTEM RELOAD MODEL' = 107, 'SYSTEM RELOAD FUNCTION' = 108, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 109, 'SYSTEM RELOAD' = 110, 'SYSTEM RESTART DISK' = 111, 'SYSTEM MERGES' = 112, 'SYSTEM TTL MERGES' = 113, 'SYSTEM FETCHES' = 114, 'SYSTEM MOVES' = 115, 'SYSTEM DISTRIBUTED SENDS' = 116, 'SYSTEM REPLICATED SENDS' = 117, 'SYSTEM SENDS' = 118, 'SYSTEM REPLICATION QUEUES' = 119, 'SYSTEM DROP REPLICA' = 120, 'SYSTEM SYNC REPLICA' = 121, 'SYSTEM RESTART REPLICA' = 122, 'SYSTEM RESTORE REPLICA' = 123, 'SYSTEM WAIT LOADING PARTS' = 124, 'SYSTEM SYNC DATABASE REPLICA' = 125, 'SYSTEM SYNC TRANSACTION LOG' = 126, 'SYSTEM SYNC FILE CACHE' = 127, 'SYSTEM FLUSH DISTRIBUTED' = 128, 'SYSTEM FLUSH LOGS' = 129, 'SYSTEM FLUSH' = 130, 'SYSTEM THREAD FUZZER' = 131, 'SYSTEM UNFREEZE' = 132, 'SYSTEM' = 133, 'dictGet' = 134, 'addressToLine' = 135, 'addressToLineWithInlines' = 136, 'addressToSymbol' = 137, 'demangle' = 138, 'INTROSPECTION' = 139, 'FILE' = 140, 'URL' = 141, 'REMOTE' = 142, 'MONGO' = 143, 'MEILISEARCH' = 144, 'MYSQL' = 145, 'POSTGRES' = 146, 'SQLITE' = 147, 'ODBC' = 148, 'JDBC' = 149, 'HDFS' = 150, 'S3' = 151, 'HIVE' = 152, 'SOURCES' = 153, 'CLUSTER' = 154, 'ALL' = 155, 'NONE' = 156),
|
||||
`access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP CACHE' = 102, 'SYSTEM RELOAD CONFIG' = 103, 'SYSTEM RELOAD USERS' = 104, 'SYSTEM RELOAD SYMBOLS' = 105, 'SYSTEM RELOAD DICTIONARY' = 106, 'SYSTEM RELOAD MODEL' = 107, 'SYSTEM RELOAD FUNCTION' = 108, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 109, 'SYSTEM RELOAD' = 110, 'SYSTEM RESTART DISK' = 111, 'SYSTEM MERGES' = 112, 'SYSTEM TTL MERGES' = 113, 'SYSTEM FETCHES' = 114, 'SYSTEM MOVES' = 115, 'SYSTEM DISTRIBUTED SENDS' = 116, 'SYSTEM REPLICATED SENDS' = 117, 'SYSTEM SENDS' = 118, 'SYSTEM REPLICATION QUEUES' = 119, 'SYSTEM DROP REPLICA' = 120, 'SYSTEM SYNC REPLICA' = 121, 'SYSTEM RESTART REPLICA' = 122, 'SYSTEM RESTORE REPLICA' = 123, 'SYSTEM WAIT LOADING PARTS' = 124, 'SYSTEM SYNC DATABASE REPLICA' = 125, 'SYSTEM SYNC TRANSACTION LOG' = 126, 'SYSTEM SYNC FILE CACHE' = 127, 'SYSTEM FLUSH DISTRIBUTED' = 128, 'SYSTEM FLUSH LOGS' = 129, 'SYSTEM FLUSH' = 130, 'SYSTEM THREAD FUZZER' = 131, 'SYSTEM UNFREEZE' = 132, 'SYSTEM' = 133, 'dictGet' = 134, 'addressToLine' = 135, 'addressToLineWithInlines' = 136, 'addressToSymbol' = 137, 'demangle' = 138, 'INTROSPECTION' = 139, 'FILE' = 140, 'URL' = 141, 'REMOTE' = 142, 'MONGO' = 143, 'MEILISEARCH' = 144, 'MYSQL' = 145, 'POSTGRES' = 146, 'SQLITE' = 147, 'ODBC' = 148, 'JDBC' = 149, 'HDFS' = 150, 'S3' = 151, 'HIVE' = 152, 'SOURCES' = 153, 'CLUSTER' = 154, 'ALL' = 155, 'NONE' = 156),
|
||||
`database` Nullable(String),
|
||||
`table` Nullable(String),
|
||||
`column` Nullable(String),
|
||||
@ -569,10 +569,10 @@ ENGINE = SystemPartsColumns
|
||||
COMMENT 'SYSTEM TABLE is built on the fly.'
|
||||
CREATE TABLE system.privileges
|
||||
(
|
||||
`privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY RESULT CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP CACHE' = 102, 'SYSTEM RELOAD CONFIG' = 103, 'SYSTEM RELOAD USERS' = 104, 'SYSTEM RELOAD SYMBOLS' = 105, 'SYSTEM RELOAD DICTIONARY' = 106, 'SYSTEM RELOAD MODEL' = 107, 'SYSTEM RELOAD FUNCTION' = 108, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 109, 'SYSTEM RELOAD' = 110, 'SYSTEM RESTART DISK' = 111, 'SYSTEM MERGES' = 112, 'SYSTEM TTL MERGES' = 113, 'SYSTEM FETCHES' = 114, 'SYSTEM MOVES' = 115, 'SYSTEM DISTRIBUTED SENDS' = 116, 'SYSTEM REPLICATED SENDS' = 117, 'SYSTEM SENDS' = 118, 'SYSTEM REPLICATION QUEUES' = 119, 'SYSTEM DROP REPLICA' = 120, 'SYSTEM SYNC REPLICA' = 121, 'SYSTEM RESTART REPLICA' = 122, 'SYSTEM RESTORE REPLICA' = 123, 'SYSTEM WAIT LOADING PARTS' = 124, 'SYSTEM SYNC DATABASE REPLICA' = 125, 'SYSTEM SYNC TRANSACTION LOG' = 126, 'SYSTEM SYNC FILE CACHE' = 127, 'SYSTEM FLUSH DISTRIBUTED' = 128, 'SYSTEM FLUSH LOGS' = 129, 'SYSTEM FLUSH' = 130, 'SYSTEM THREAD FUZZER' = 131, 'SYSTEM UNFREEZE' = 132, 'SYSTEM' = 133, 'dictGet' = 134, 'addressToLine' = 135, 'addressToLineWithInlines' = 136, 'addressToSymbol' = 137, 'demangle' = 138, 'INTROSPECTION' = 139, 'FILE' = 140, 'URL' = 141, 'REMOTE' = 142, 'MONGO' = 143, 'MEILISEARCH' = 144, 'MYSQL' = 145, 'POSTGRES' = 146, 'SQLITE' = 147, 'ODBC' = 148, 'JDBC' = 149, 'HDFS' = 150, 'S3' = 151, 'HIVE' = 152, 'SOURCES' = 153, 'CLUSTER' = 154, 'ALL' = 155, 'NONE' = 156),
|
||||
`privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP CACHE' = 102, 'SYSTEM RELOAD CONFIG' = 103, 'SYSTEM RELOAD USERS' = 104, 'SYSTEM RELOAD SYMBOLS' = 105, 'SYSTEM RELOAD DICTIONARY' = 106, 'SYSTEM RELOAD MODEL' = 107, 'SYSTEM RELOAD FUNCTION' = 108, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 109, 'SYSTEM RELOAD' = 110, 'SYSTEM RESTART DISK' = 111, 'SYSTEM MERGES' = 112, 'SYSTEM TTL MERGES' = 113, 'SYSTEM FETCHES' = 114, 'SYSTEM MOVES' = 115, 'SYSTEM DISTRIBUTED SENDS' = 116, 'SYSTEM REPLICATED SENDS' = 117, 'SYSTEM SENDS' = 118, 'SYSTEM REPLICATION QUEUES' = 119, 'SYSTEM DROP REPLICA' = 120, 'SYSTEM SYNC REPLICA' = 121, 'SYSTEM RESTART REPLICA' = 122, 'SYSTEM RESTORE REPLICA' = 123, 'SYSTEM WAIT LOADING PARTS' = 124, 'SYSTEM SYNC DATABASE REPLICA' = 125, 'SYSTEM SYNC TRANSACTION LOG' = 126, 'SYSTEM SYNC FILE CACHE' = 127, 'SYSTEM FLUSH DISTRIBUTED' = 128, 'SYSTEM FLUSH LOGS' = 129, 'SYSTEM FLUSH' = 130, 'SYSTEM THREAD FUZZER' = 131, 'SYSTEM UNFREEZE' = 132, 'SYSTEM' = 133, 'dictGet' = 134, 'addressToLine' = 135, 'addressToLineWithInlines' = 136, 'addressToSymbol' = 137, 'demangle' = 138, 'INTROSPECTION' = 139, 'FILE' = 140, 'URL' = 141, 'REMOTE' = 142, 'MONGO' = 143, 'MEILISEARCH' = 144, 'MYSQL' = 145, 'POSTGRES' = 146, 'SQLITE' = 147, 'ODBC' = 148, 'JDBC' = 149, 'HDFS' = 150, 'S3' = 151, 'HIVE' = 152, 'SOURCES' = 153, 'CLUSTER' = 154, 'ALL' = 155, 'NONE' = 156),
|
||||
`aliases` Array(String),
|
||||
`level` Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2, 'DICTIONARY' = 3, 'VIEW' = 4, 'COLUMN' = 5)),
|
||||
`parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY RESULT CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP CACHE' = 102, 'SYSTEM RELOAD CONFIG' = 103, 'SYSTEM RELOAD USERS' = 104, 'SYSTEM RELOAD SYMBOLS' = 105, 'SYSTEM RELOAD DICTIONARY' = 106, 'SYSTEM RELOAD MODEL' = 107, 'SYSTEM RELOAD FUNCTION' = 108, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 109, 'SYSTEM RELOAD' = 110, 'SYSTEM RESTART DISK' = 111, 'SYSTEM MERGES' = 112, 'SYSTEM TTL MERGES' = 113, 'SYSTEM FETCHES' = 114, 'SYSTEM MOVES' = 115, 'SYSTEM DISTRIBUTED SENDS' = 116, 'SYSTEM REPLICATED SENDS' = 117, 'SYSTEM SENDS' = 118, 'SYSTEM REPLICATION QUEUES' = 119, 'SYSTEM DROP REPLICA' = 120, 'SYSTEM SYNC REPLICA' = 121, 'SYSTEM RESTART REPLICA' = 122, 'SYSTEM RESTORE REPLICA' = 123, 'SYSTEM WAIT LOADING PARTS' = 124, 'SYSTEM SYNC DATABASE REPLICA' = 125, 'SYSTEM SYNC TRANSACTION LOG' = 126, 'SYSTEM SYNC FILE CACHE' = 127, 'SYSTEM FLUSH DISTRIBUTED' = 128, 'SYSTEM FLUSH LOGS' = 129, 'SYSTEM FLUSH' = 130, 'SYSTEM THREAD FUZZER' = 131, 'SYSTEM UNFREEZE' = 132, 'SYSTEM' = 133, 'dictGet' = 134, 'addressToLine' = 135, 'addressToLineWithInlines' = 136, 'addressToSymbol' = 137, 'demangle' = 138, 'INTROSPECTION' = 139, 'FILE' = 140, 'URL' = 141, 'REMOTE' = 142, 'MONGO' = 143, 'MEILISEARCH' = 144, 'MYSQL' = 145, 'POSTGRES' = 146, 'SQLITE' = 147, 'ODBC' = 148, 'JDBC' = 149, 'HDFS' = 150, 'S3' = 151, 'HIVE' = 152, 'SOURCES' = 153, 'CLUSTER' = 154, 'ALL' = 155, 'NONE' = 156))
|
||||
`parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP CACHE' = 102, 'SYSTEM RELOAD CONFIG' = 103, 'SYSTEM RELOAD USERS' = 104, 'SYSTEM RELOAD SYMBOLS' = 105, 'SYSTEM RELOAD DICTIONARY' = 106, 'SYSTEM RELOAD MODEL' = 107, 'SYSTEM RELOAD FUNCTION' = 108, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 109, 'SYSTEM RELOAD' = 110, 'SYSTEM RESTART DISK' = 111, 'SYSTEM MERGES' = 112, 'SYSTEM TTL MERGES' = 113, 'SYSTEM FETCHES' = 114, 'SYSTEM MOVES' = 115, 'SYSTEM DISTRIBUTED SENDS' = 116, 'SYSTEM REPLICATED SENDS' = 117, 'SYSTEM SENDS' = 118, 'SYSTEM REPLICATION QUEUES' = 119, 'SYSTEM DROP REPLICA' = 120, 'SYSTEM SYNC REPLICA' = 121, 'SYSTEM RESTART REPLICA' = 122, 'SYSTEM RESTORE REPLICA' = 123, 'SYSTEM WAIT LOADING PARTS' = 124, 'SYSTEM SYNC DATABASE REPLICA' = 125, 'SYSTEM SYNC TRANSACTION LOG' = 126, 'SYSTEM SYNC FILE CACHE' = 127, 'SYSTEM FLUSH DISTRIBUTED' = 128, 'SYSTEM FLUSH LOGS' = 129, 'SYSTEM FLUSH' = 130, 'SYSTEM THREAD FUZZER' = 131, 'SYSTEM UNFREEZE' = 132, 'SYSTEM' = 133, 'dictGet' = 134, 'addressToLine' = 135, 'addressToLineWithInlines' = 136, 'addressToSymbol' = 137, 'demangle' = 138, 'INTROSPECTION' = 139, 'FILE' = 140, 'URL' = 141, 'REMOTE' = 142, 'MONGO' = 143, 'MEILISEARCH' = 144, 'MYSQL' = 145, 'POSTGRES' = 146, 'SQLITE' = 147, 'ODBC' = 148, 'JDBC' = 149, 'HDFS' = 150, 'S3' = 151, 'HIVE' = 152, 'SOURCES' = 153, 'CLUSTER' = 154, 'ALL' = 155, 'NONE' = 156))
|
||||
)
|
||||
ENGINE = SystemPrivileges
|
||||
COMMENT 'SYSTEM TABLE is built on the fly.'
|
||||
|
||||
@ -31,6 +31,9 @@ IPv6 functions
|
||||
::ffff:127.0.0.1
|
||||
::ffff:127.0.0.1
|
||||
::ffff:127.0.0.1
|
||||
::
|
||||
\N
|
||||
100000000
|
||||
--
|
||||
::ffff:127.0.0.1
|
||||
--
|
||||
|
||||
@ -56,6 +56,12 @@ SELECT toIPv6('::ffff:127.0.0.1');
|
||||
SELECT toIPv6OrDefault('::ffff:127.0.0.1');
|
||||
SELECT toIPv6OrNull('::ffff:127.0.0.1');
|
||||
|
||||
SELECT toIPv6('::.1.2.3'); --{serverError CANNOT_PARSE_IPV6}
|
||||
SELECT toIPv6OrDefault('::.1.2.3');
|
||||
SELECT toIPv6OrNull('::.1.2.3');
|
||||
|
||||
SELECT count() FROM numbers_mt(100000000) WHERE NOT ignore(toIPv6OrZero(randomString(8)));
|
||||
|
||||
SELECT '--';
|
||||
|
||||
SELECT cast('test' , 'IPv6'); --{serverError CANNOT_PARSE_IPV6}
|
||||
|
||||
@ -0,0 +1,29 @@
|
||||
-- Tags: no-parallel
|
||||
-- Tag no-parallel: Messes with internal cache
|
||||
|
||||
SET allow_experimental_query_cache = true;
|
||||
|
||||
-- Start with empty query cache (QC) and query log
|
||||
SYSTEM DROP QUERY CACHE;
|
||||
DROP TABLE system.query_log SYNC;
|
||||
|
||||
-- Insert an entry into the query cache.
|
||||
SELECT 1 SETTINGS use_query_cache = true;
|
||||
-- Check that entry in QC exists
|
||||
SELECT COUNT(*) FROM system.query_cache;
|
||||
|
||||
-- Run the same SELECT but with different case (--> select). We want its result to be served from the QC.
|
||||
SELECT '---';
|
||||
select 1 SETTINGS use_query_cache = true;
|
||||
|
||||
-- There should still be just one entry in the QC
|
||||
SELECT COUNT(*) FROM system.query_cache;
|
||||
|
||||
-- The second query should cause a QC hit.
|
||||
SYSTEM FLUSH LOGS;
|
||||
SELECT ProfileEvents['QueryCacheHits'], ProfileEvents['QueryCacheMisses']
|
||||
FROM system.query_log
|
||||
WHERE type = 'QueryFinish'
|
||||
AND query = 'select 1 SETTINGS use_query_cache = true;';
|
||||
|
||||
SYSTEM DROP QUERY CACHE;
|
||||
12
tests/queries/0_stateless/02494_query_cache_drop_cache.sql
Normal file
12
tests/queries/0_stateless/02494_query_cache_drop_cache.sql
Normal file
@ -0,0 +1,12 @@
|
||||
-- Tags: no-parallel
|
||||
-- Tag no-parallel: Messes with internal cache
|
||||
|
||||
SET allow_experimental_query_cache = true;
|
||||
|
||||
-- Cache query result in query cache
|
||||
SELECT 1 SETTINGS use_query_cache = true;
|
||||
SELECT count(*) FROM system.query_cache;
|
||||
|
||||
-- No query results are cached after DROP
|
||||
SYSTEM DROP QUERY CACHE;
|
||||
SELECT count(*) FROM system.query_cache;
|
||||
@ -0,0 +1,68 @@
|
||||
-- Tags: no-parallel
|
||||
-- Tag no-parallel: Messes with internal cache
|
||||
|
||||
SET allow_experimental_query_cache = true;
|
||||
|
||||
SYSTEM DROP QUERY CACHE;
|
||||
DROP TABLE IF EXISTS eligible_test;
|
||||
DROP TABLE IF EXISTS eligible_test2;
|
||||
|
||||
-- enable query cache session-wide but also force it individually in each of below statements
|
||||
SET use_query_cache = true;
|
||||
|
||||
-- check that SELECT statements create entries in the query cache ...
|
||||
SELECT 1 SETTINGS use_query_cache = true;
|
||||
SELECT COUNT(*) FROM system.query_cache;
|
||||
|
||||
SYSTEM DROP QUERY CACHE;
|
||||
|
||||
-- ... and all other statements also should not create entries:
|
||||
|
||||
-- CREATE
|
||||
CREATE TABLE eligible_test (a String) ENGINE=MergeTree ORDER BY a; -- SETTINGS use_query_cache = true; -- SETTINGS rejected as unknown
|
||||
SELECT COUNT(*) FROM system.query_cache;
|
||||
|
||||
-- ALTER
|
||||
ALTER TABLE eligible_test ADD COLUMN b String SETTINGS use_query_cache = true;
|
||||
SELECT COUNT(*) FROM system.query_cache;
|
||||
|
||||
-- INSERT
|
||||
INSERT INTO eligible_test VALUES('a', 'b'); -- SETTINGS use_query_cache = true; -- SETTINGS rejected as unknown
|
||||
SELECT COUNT(*) FROM system.query_cache;
|
||||
INSERT INTO eligible_test SELECT * FROM eligible_test SETTINGS use_query_cache = true;
|
||||
SELECT COUNT(*) FROM system.query_cache;
|
||||
|
||||
-- SHOW
|
||||
SHOW TABLES SETTINGS use_query_cache = true;
|
||||
SELECT COUNT(*) FROM system.query_cache;
|
||||
|
||||
-- CHECK
|
||||
CHECK TABLE eligible_test SETTINGS use_query_cache = true;
|
||||
SELECT COUNT(*) FROM system.query_cache;
|
||||
|
||||
-- DESCRIBE
|
||||
DESCRIBE TABLE eligible_test SETTINGS use_query_cache = true;
|
||||
SELECT COUNT(*) FROM system.query_cache;
|
||||
|
||||
-- EXISTS
|
||||
EXISTS TABLE eligible_test SETTINGS use_query_cache = true;
|
||||
SELECT COUNT(*) FROM system.query_cache;
|
||||
|
||||
-- KILL
|
||||
KILL QUERY WHERE query_id='3-857d-4a57-9ee0-3c7da5d60a90' SETTINGS use_query_cache = true;
|
||||
SELECT COUNT(*) FROM system.query_cache;
|
||||
|
||||
-- OPTIMIZE
|
||||
OPTIMIZE TABLE eligible_test FINAL SETTINGS use_query_cache = true;
|
||||
SELECT COUNT(*) FROM system.query_cache;
|
||||
|
||||
-- TRUNCATE
|
||||
TRUNCATE TABLE eligible_test SETTINGS use_query_cache = true;
|
||||
SELECT COUNT(*) FROM system.query_cache;
|
||||
|
||||
-- RENAME
|
||||
RENAME TABLE eligible_test TO eligible_test2 SETTINGS use_query_cache = true;
|
||||
SELECT COUNT(*) FROM system.query_cache;
|
||||
|
||||
SYSTEM DROP QUERY CACHE;
|
||||
DROP TABLE eligible_test2;
|
||||
32
tests/queries/0_stateless/02494_query_cache_events.sql
Normal file
32
tests/queries/0_stateless/02494_query_cache_events.sql
Normal file
@ -0,0 +1,32 @@
|
||||
-- Tags: no-parallel
|
||||
-- Tag no-parallel: Messes with internal cache
|
||||
|
||||
SET allow_experimental_query_cache = true;
|
||||
|
||||
-- Start with empty query cache QC and query log
|
||||
SYSTEM DROP QUERY CACHE;
|
||||
DROP TABLE system.query_log SYNC;
|
||||
|
||||
-- Run a query with QC on. The first execution is a QC miss.
|
||||
SELECT '---';
|
||||
SELECT 1 SETTINGS use_query_cache = true;
|
||||
|
||||
SYSTEM FLUSH LOGS;
|
||||
SELECT ProfileEvents['QueryCacheHits'], ProfileEvents['QueryCacheMisses']
|
||||
FROM system.query_log
|
||||
WHERE type = 'QueryFinish'
|
||||
AND query = 'SELECT 1 SETTINGS use_query_cache = true;';
|
||||
|
||||
|
||||
-- Run previous query again with query cache on
|
||||
SELECT '---';
|
||||
SELECT 1 SETTINGS use_query_cache = true;
|
||||
|
||||
DROP TABLE system.query_log SYNC;
|
||||
SYSTEM FLUSH LOGS;
|
||||
SELECT ProfileEvents['QueryCacheHits'], ProfileEvents['QueryCacheMisses']
|
||||
FROM system.query_log
|
||||
WHERE type = 'QueryFinish'
|
||||
AND query = 'SELECT 1 SETTINGS use_query_cache = true;';
|
||||
|
||||
SYSTEM DROP QUERY CACHE;
|
||||
@ -0,0 +1,12 @@
|
||||
-- Tags: no-parallel
|
||||
-- Tag no-parallel: Messes with internal cache
|
||||
|
||||
SET allow_experimental_query_cache = true;
|
||||
|
||||
SYSTEM DROP QUERY CACHE;
|
||||
|
||||
-- If an exception is thrown during query execution, no entry must be created in the query cache
|
||||
SELECT throwIf(1) SETTINGS use_query_cache = true; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
|
||||
SELECT COUNT(*) FROM system.query_cache;
|
||||
|
||||
SYSTEM DROP QUERY CACHE;
|
||||
@ -1,23 +1,23 @@
|
||||
-- Tags: no-parallel
|
||||
-- Tag no-parallel: Messes with internal cache
|
||||
|
||||
SET allow_experimental_query_result_cache = true;
|
||||
SET allow_experimental_query_cache = true;
|
||||
|
||||
SYSTEM DROP QUERY RESULT CACHE;
|
||||
SYSTEM DROP QUERY CACHE;
|
||||
|
||||
-- Run a silly query with a non-trivial plan and put the result into the query result cache (QRC)
|
||||
SELECT 1 + number from system.numbers LIMIT 1 SETTINGS use_query_result_cache = true;
|
||||
SELECT count(*) FROM system.query_result_cache;
|
||||
-- Run a silly query with a non-trivial plan and put the result into the query cache QC
|
||||
SELECT 1 + number from system.numbers LIMIT 1 SETTINGS use_query_cache = true;
|
||||
SELECT count(*) FROM system.query_cache;
|
||||
|
||||
-- EXPLAIN PLAN should show the same regardless if the result is calculated or read from the QRC
|
||||
-- EXPLAIN PLAN should show the same regardless if the result is calculated or read from the QC
|
||||
EXPLAIN PLAN SELECT 1 + number from system.numbers LIMIT 1;
|
||||
EXPLAIN PLAN SELECT 1 + number from system.numbers LIMIT 1 SETTINGS use_query_result_cache = true; -- (*)
|
||||
EXPLAIN PLAN SELECT 1 + number from system.numbers LIMIT 1 SETTINGS use_query_cache = true; -- (*)
|
||||
|
||||
-- EXPLAIN PIPELINE should show the same regardless if the result is calculated or read from the QRC
|
||||
-- EXPLAIN PIPELINE should show the same regardless if the result is calculated or read from the QC
|
||||
EXPLAIN PIPELINE SELECT 1 + number from system.numbers LIMIT 1;
|
||||
EXPLAIN PIPELINE SELECT 1 + number from system.numbers LIMIT 1 SETTINGS use_query_result_cache = true; -- (*)
|
||||
EXPLAIN PIPELINE SELECT 1 + number from system.numbers LIMIT 1 SETTINGS use_query_cache = true; -- (*)
|
||||
|
||||
-- Statements (*) must not cache their results into the QRC
|
||||
SELECT count(*) FROM system.query_result_cache;
|
||||
-- Statements (*) must not cache their results into the QC
|
||||
SELECT count(*) FROM system.query_cache;
|
||||
|
||||
SYSTEM DROP QUERY RESULT CACHE;
|
||||
SYSTEM DROP QUERY CACHE;
|
||||
@ -0,0 +1,20 @@
|
||||
-- Tags: no-parallel
|
||||
-- Tag no-parallel: Messes with internal cache
|
||||
|
||||
SET allow_experimental_query_cache = true;
|
||||
|
||||
SYSTEM DROP QUERY CACHE;
|
||||
|
||||
-- This creates an entry in the query cache ...
|
||||
SELECT 1 SETTINGS use_query_cache = true;
|
||||
SELECT COUNT(*) FROM system.query_cache;
|
||||
|
||||
SYSTEM DROP QUERY CACHE;
|
||||
|
||||
SELECT '---';
|
||||
|
||||
-- ... but this does not because the query executes much faster than the specified minumum query duration for caching the result
|
||||
SELECT 1 SETTINGS use_query_cache = true, query_cache_min_query_duration = 10000;
|
||||
SELECT COUNT(*) FROM system.query_cache;
|
||||
|
||||
SYSTEM DROP QUERY CACHE;
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user