thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 01:25:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into fasttest-tsv

Browse Source
This commit is contained in:
mergify[bot] 2021-09-27 07:48:49 +00:00 committed by GitHub
commit dbb924171d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
173 changed files with 1261 additions and 6501 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.arcignore
View File

@ -1,12 +0,0 @@
# .arcignore is the same as .gitignore but for Arc VCS.
# Arc VCS is a proprietary VCS in Yandex that is very similar to Git
# from the user perspective but with the following differences:
# 1. Data is stored in distributed object storage.
# 2. Local copy works via FUSE without downloading all the objects.
# For this reason, it is better suited for huge monorepositories that can be found in large companies (e.g. Yandex, Google).
# As ClickHouse developers, we don't use Arc as a VCS (we use Git).
# But the ClickHouse source code is also mirrored into internal monorepository and our collegues are using Arc.
# You can read more about Arc here: https://habr.com/en/company/yandex/blog/482926/
# Repository is synchronized without 3rd-party submodules.
contrib

1
CHANGELOG.md
View File

@ -34,7 +34,6 @@
* New functions `currentProfiles()`, `enabledProfiles()`, `defaultProfiles()`. [#26714](https://github.com/ClickHouse/ClickHouse/pull/26714) ([Vitaly Baranov](https://github.com/vitlibar)).
* Add functions that return (initial_)query_id of the current query. This closes [#23682](https://github.com/ClickHouse/ClickHouse/issues/23682). [#26410](https://github.com/ClickHouse/ClickHouse/pull/26410) ([Alexey Boykov](https://github.com/mathalex)).
* Add `REPLACE GRANT` feature. [#26384](https://github.com/ClickHouse/ClickHouse/pull/26384) ([Caspian](https://github.com/Cas-pian)).
* Implement window function `nth_value(expr, N)` that returns the value of the Nth row of the window frame. [#26334](https://github.com/ClickHouse/ClickHouse/pull/26334) ([Zuo, RuoYu](https://github.com/ryzuo)).
* `EXPLAIN` query now has `EXPLAIN ESTIMATE ...` mode that will show information about read rows, marks and parts from MergeTree tables. Closes [#23941](https://github.com/ClickHouse/ClickHouse/issues/23941). [#26131](https://github.com/ClickHouse/ClickHouse/pull/26131) ([fastio](https://github.com/fastio)).
* Added `system.zookeeper_log` table. All actions of ZooKeeper client are logged into this table. Implements [#25449](https://github.com/ClickHouse/ClickHouse/issues/25449). [#26129](https://github.com/ClickHouse/ClickHouse/pull/26129) ([tavplubix](https://github.com/tavplubix)).
* Zero-copy replication for `ReplicatedMergeTree` over `HDFS` storage. [#25918](https://github.com/ClickHouse/ClickHouse/pull/25918) ([Zhichang Yu](https://github.com/yuzhichang)).

63
base/common/ya.make
View File

@ -1,63 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
ADDINCL(
GLOBAL clickhouse/base
)
CFLAGS (GLOBAL -DARCADIA_BUILD)
CFLAGS (GLOBAL -DUSE_CPUID=1)
CFLAGS (GLOBAL -DUSE_JEMALLOC=0)
CFLAGS (GLOBAL -DUSE_RAPIDJSON=1)
CFLAGS (GLOBAL -DUSE_SSL=1)
IF (OS_DARWIN)
CFLAGS (GLOBAL -DOS_DARWIN)
ELSEIF (OS_FREEBSD)
CFLAGS (GLOBAL -DOS_FREEBSD)
ELSEIF (OS_LINUX)
CFLAGS (GLOBAL -DOS_LINUX)
ENDIF ()
PEERDIR(
contrib/libs/cctz
contrib/libs/cxxsupp/libcxx-filesystem
contrib/libs/poco/Net
contrib/libs/poco/Util
contrib/libs/poco/NetSSL_OpenSSL
contrib/libs/fmt
contrib/restricted/boost
contrib/restricted/cityhash-1.0.2
)
CFLAGS(-g0)
SRCS(
DateLUT.cpp
DateLUTImpl.cpp
JSON.cpp
LineReader.cpp
StringRef.cpp
argsToConfig.cpp
coverage.cpp
demangle.cpp
errnoToString.cpp
getFQDNOrHostName.cpp
getMemoryAmount.cpp
getPageSize.cpp
getResource.cpp
getThreadId.cpp
mremap.cpp
phdr_cache.cpp
preciseExp10.cpp
setTerminalEcho.cpp
shift10.cpp
sleep.cpp
terminalColors.cpp
)
END()

41
base/common/ya.make.in
View File

@ -1,41 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
ADDINCL(
GLOBAL clickhouse/base
)
CFLAGS (GLOBAL -DARCADIA_BUILD)
CFLAGS (GLOBAL -DUSE_CPUID=1)
CFLAGS (GLOBAL -DUSE_JEMALLOC=0)
CFLAGS (GLOBAL -DUSE_RAPIDJSON=1)
CFLAGS (GLOBAL -DUSE_SSL=1)
IF (OS_DARWIN)
CFLAGS (GLOBAL -DOS_DARWIN)
ELSEIF (OS_FREEBSD)
CFLAGS (GLOBAL -DOS_FREEBSD)
ELSEIF (OS_LINUX)
CFLAGS (GLOBAL -DOS_LINUX)
ENDIF ()
PEERDIR(
contrib/libs/cctz
contrib/libs/cxxsupp/libcxx-filesystem
contrib/libs/poco/Net
contrib/libs/poco/Util
contrib/libs/poco/NetSSL_OpenSSL
contrib/libs/fmt
contrib/restricted/boost
contrib/restricted/cityhash-1.0.2
)
CFLAGS(-g0)
SRCS(
<? find . -name '*.cpp' | grep -v -F tests/ | grep -v -F examples | grep -v -F Replxx | grep -v -F Readline | sed 's/^\.\// /' | sort ?>
)
END()

19
base/daemon/ya.make
View File

@ -1,19 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
NO_COMPILER_WARNINGS()
PEERDIR(
clickhouse/src/Common
)
CFLAGS(-g0)
SRCS(
BaseDaemon.cpp
GraphiteWriter.cpp
SentryWriter.cpp
)
END()

19
base/loggers/ya.make
View File

@ -1,19 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
CFLAGS(-g0)
SRCS(
ExtendedLogChannel.cpp
Loggers.cpp
OwnFormattingChannel.cpp
OwnPatternFormatter.cpp
OwnSplitChannel.cpp
)
END()

39
base/mysqlxx/ya.make
View File

@ -1,39 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
LIBRARY()
OWNER(g:clickhouse)
CFLAGS(-g0)
PEERDIR(
contrib/restricted/boost/libs
contrib/libs/libmysql_r
contrib/libs/poco/Foundation
contrib/libs/poco/Util
)
ADDINCL(
GLOBAL clickhouse/base
clickhouse/base
contrib/libs/libmysql_r
)
NO_COMPILER_WARNINGS()
NO_UTIL()
SRCS(
Connection.cpp
Exception.cpp
Pool.cpp
PoolFactory.cpp
PoolWithFailover.cpp
Query.cpp
ResultBase.cpp
Row.cpp
UseQueryResult.cpp
Value.cpp
)
END()

28
base/mysqlxx/ya.make.in
View File

@ -1,28 +0,0 @@
LIBRARY()
OWNER(g:clickhouse)
CFLAGS(-g0)
PEERDIR(
contrib/restricted/boost/libs
contrib/libs/libmysql_r
contrib/libs/poco/Foundation
contrib/libs/poco/Util
)
ADDINCL(
GLOBAL clickhouse/base
clickhouse/base
contrib/libs/libmysql_r
)
NO_COMPILER_WARNINGS()
NO_UTIL()
SRCS(
<? find . -name '*.cpp' | grep -v -F tests/ | grep -v -F examples | sed 's/^\.\// /' | sort ?>
)
END()

7
base/pcg-random/ya.make
View File

@ -1,7 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
ADDINCL (GLOBAL clickhouse/base/pcg-random)
END()

11
base/readpassphrase/ya.make
View File

@ -1,11 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
CFLAGS(-g0)
SRCS(
readpassphrase.c
)
END()

13
base/widechar_width/ya.make
View File

@ -1,13 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
ADDINCL(GLOBAL clickhouse/base/widechar_width)
CFLAGS(-g0)
SRCS(
widechar_width.cpp
)
END()

11
base/ya.make
View File

@ -1,11 +0,0 @@
OWNER(g:clickhouse)
RECURSE(
common
daemon
loggers
mysqlxx
pcg-random
widechar_width
readpassphrase
)

25
cmake/yandex/ya.make.versions.inc
View File

@ -1,25 +0,0 @@
INCLUDE(${ARCADIA_ROOT}/clickhouse/cmake/autogenerated_versions.txt)
# TODO: not sure if this is customizable per-binary
SET(VERSION_NAME "ClickHouse")
# TODO: not quite sure how to replace dash with space in ya.make
SET(VERSION_FULL "${VERSION_NAME}-${VERSION_STRING}")
CFLAGS (GLOBAL -DDBMS_NAME=\"ClickHouse\")
CFLAGS (GLOBAL -DDBMS_VERSION_MAJOR=${VERSION_MAJOR})
CFLAGS (GLOBAL -DDBMS_VERSION_MINOR=${VERSION_MINOR})
CFLAGS (GLOBAL -DDBMS_VERSION_PATCH=${VERSION_PATCH})
CFLAGS (GLOBAL -DVERSION_FULL=\"\\\"${VERSION_FULL}\\\"\")
CFLAGS (GLOBAL -DVERSION_MAJOR=${VERSION_MAJOR})
CFLAGS (GLOBAL -DVERSION_MINOR=${VERSION_MINOR})
CFLAGS (GLOBAL -DVERSION_PATCH=${VERSION_PATCH})
# TODO: not supported yet, not sure if ya.make supports arithmetic.
CFLAGS (GLOBAL -DVERSION_INTEGER=0)
CFLAGS (GLOBAL -DVERSION_NAME=\"\\\"${VERSION_NAME}\\\"\")
CFLAGS (GLOBAL -DVERSION_OFFICIAL=\"-arcadia\")
CFLAGS (GLOBAL -DVERSION_REVISION=${VERSION_REVISION})
CFLAGS (GLOBAL -DVERSION_STRING=\"\\\"${VERSION_STRING}\\\"\")

2
contrib/libhdfs3 vendored

@ -1 +1 @@
Subproject commit 095b9d48b400abb72d967cb0539af13b1e3d90cf
Subproject commit 082e55f17d1c58bf124290fb044fea40e985ec11

2
docker/test/coverage/s3downloader
View File

@ -92,7 +92,7 @@ if __name__ == "__main__":
logging.info("Some exception occured %s", str(ex))
raise
finally:
logging.info("Will remove dowloaded file %s from filesystem if it exists", temp_archive_path)
logging.info("Will remove downloaded file %s from filesystem if it exists", temp_archive_path)
if os.path.exists(temp_archive_path):
os.remove(temp_archive_path)
logging.info("Processing of %s finished", dataset)

2
docker/test/stateful/s3downloader
View File

@ -92,7 +92,7 @@ if __name__ == "__main__":
logging.info("Some exception occured %s", str(ex))
raise
finally:
logging.info("Will remove dowloaded file %s from filesystem if it exists", temp_archive_path)
logging.info("Will remove downloaded file %s from filesystem if it exists", temp_archive_path)
if os.path.exists(temp_archive_path):
os.remove(temp_archive_path)
logging.info("Processing of %s finished", dataset)

4
docs/en/getting-started/example-datasets/metrica.md
View File

@ -42,7 +42,11 @@ md5sum hits_v1.tsv
# Checksum should be equal to: f3631b6295bf06989c1437491f7592cb
# now create table
clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets"
# for hits_v1
clickhouse-client --query "CREATE TABLE datasets.hits_v1 ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192"
# for hits_100m_obfuscated
clickhouse-client --query="CREATE TABLE hits_100m_obfuscated (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, Refresh UInt8, RefererCategoryID UInt16, RefererRegionID UInt32, URLCategoryID UInt16, URLRegionID UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, OriginalURL String, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), LocalEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, RemoteIP UInt32, WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming UInt32, DNSTiming UInt32, ConnectTiming UInt32, ResponseStartTiming UInt32, ResponseEndTiming UInt32, FetchTiming UInt32, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192"
# import data
cat hits_v1.tsv | clickhouse-client --query "INSERT INTO datasets.hits_v1 FORMAT TSV" --max_insert_block_size=100000
# optionally you can optimize table

1
docs/en/introduction/adopters.md
View File

@ -56,6 +56,7 @@ toc_title: Adopters
| <a href="https://geniee.co.jp" class="favicon">Geniee</a> | Ad network | Main product | — | — | [Blog post in Japanese, July 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) |
| <a href="https://www.genotek.ru/" class="favicon">Genotek</a> | Bioinformatics | Main product | — | — | [Video, August 2020](https://youtu.be/v3KyZbz9lEE) |
| <a href="https://glaber.io/" class="favicon">Glaber</a> | Monitoring | Main product | — | — | [Website](https://glaber.io/) |
| <a href="https://graphcdn.io/" class="favicon">GraphCDN</a> | CDN | Traffic Analytics | — | — | [Blog Post in English, August 2021](https://altinity.com/blog/delivering-insight-on-graphql-apis-with-clickhouse-at-graphcdn/) |
| <a href="https://www.huya.com/" class="favicon">HUYA</a> | Video Streaming | Analytics | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) |
| <a href="https://www.the-ica.com/" class="favicon">ICA</a> | FinTech | Risk Management | — | — | [Blog Post in English, Sep 2020](https://altinity.com/blog/clickhouse-vs-redshift-performance-for-fintech-risk-management?utm_campaign=ClickHouse%20vs%20RedShift&utm_content=143520807&utm_medium=social&utm_source=twitter&hss_channel=tw-3894792263) |
| <a href="https://www.idealista.com" class="favicon">Idealista</a> | Real Estate | Analytics | — | — | [Blog Post in English, April 2019](https://clickhouse.com/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) |

6
docs/en/sql-reference/functions/geo/h3.md
View File

@ -594,14 +594,14 @@ Result:
└─────┘
```
## h3ResIsClassIII {#h3resisclassIII}
## h3IsResClassIII {#h3isresclassIII}
Returns whether [H3](#h3index) index has a resolution with Class III orientation.
**Syntax**
``` sql
h3ResIsClassIII(index)
h3IsResClassIII(index)
```
**Parameter**
@ -620,7 +620,7 @@ Type: [UInt8](../../../sql-reference/data-types/int-uint.md).
Query:
``` sql
SELECT h3ResIsClassIII(617420388352917503) as res;
SELECT h3IsResClassIII(617420388352917503) as res;
```
Result:

13
programs/format/Format.cpp
View File

@ -44,6 +44,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
desc.add_options()
("query", po::value<std::string>(), "query to format")
("help,h", "produce help message")
("hilite", "add syntax highlight with ANSI terminal escape sequences")
("oneline", "format in single line")
@ -86,8 +87,16 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
}
String query;
ReadBufferFromFileDescriptor in(STDIN_FILENO);
readStringUntilEOF(query, in);
if (options.count("query"))
{
query = options["query"].as<std::string>();
}
else
{
ReadBufferFromFileDescriptor in(STDIN_FILENO);
readStringUntilEOF(query, in);
}
if (obfuscate)
{

2
programs/install/Install.cpp
View File

@ -962,7 +962,7 @@ namespace
if (isRunning(pid_file))
{
throw Exception(ErrorCodes::CANNOT_KILL,
"The server process still exists after %zu ms",
"The server process still exists after {} tries (delay: {} ms)",
num_kill_check_tries, kill_check_delay_ms);
}
}

22
programs/server/ya.make
View File

@ -1,22 +0,0 @@
OWNER(g:clickhouse)
PROGRAM(clickhouse-server)
PEERDIR(
clickhouse/base/common
clickhouse/base/daemon
clickhouse/base/loggers
clickhouse/src
contrib/libs/poco/NetSSL_OpenSSL
)
CFLAGS(-g0)
SRCS(
clickhouse-server.cpp
MetricsTransmitter.cpp
Server.cpp
)
END()

32
programs/ya.make
View File

@ -1,32 +0,0 @@
OWNER(g:clickhouse)
PROGRAM(clickhouse)
CFLAGS(
-DENABLE_CLICKHOUSE_CLIENT
-DENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG
-DENABLE_CLICKHOUSE_SERVER
)
PEERDIR(
clickhouse/base/daemon
clickhouse/base/loggers
clickhouse/src
)
CFLAGS(-g0)
SRCS(
main.cpp
client/Client.cpp
client/QueryFuzzer.cpp
client/ConnectionParameters.cpp
client/Suggest.cpp
client/TestHint.cpp
extract-from-config/ExtractFromConfig.cpp
server/Server.cpp
server/MetricsTransmitter.cpp
)
END()

54
src/Access/ya.make
View File

@ -1,54 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
SRCS(
AccessControlManager.cpp
AccessEntityIO.cpp
AccessRights.cpp
AccessRightsElement.cpp
AllowedClientHosts.cpp
Authentication.cpp
ContextAccess.cpp
Credentials.cpp
DiskAccessStorage.cpp
EnabledQuota.cpp
EnabledRoles.cpp
EnabledRolesInfo.cpp
EnabledRowPolicies.cpp
EnabledSettings.cpp
ExternalAuthenticators.cpp
GSSAcceptor.cpp
GrantedRoles.cpp
IAccessEntity.cpp
IAccessStorage.cpp
LDAPAccessStorage.cpp
LDAPClient.cpp
MemoryAccessStorage.cpp
MultipleAccessStorage.cpp
Quota.cpp
QuotaCache.cpp
QuotaUsage.cpp
ReplicatedAccessStorage.cpp
Role.cpp
RoleCache.cpp
RolesOrUsersSet.cpp
RowPolicy.cpp
RowPolicyCache.cpp
SettingsConstraints.cpp
SettingsProfile.cpp
SettingsProfileElement.cpp
SettingsProfilesCache.cpp
SettingsProfilesInfo.cpp
User.cpp
UsersConfigAccessStorage.cpp
)
END()

14
src/Access/ya.make.in
View File

@ -1,14 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | sed 's/^\.\// /' | sort ?>
)
END()

13
src/AggregateFunctions/AggregateFunctionFactory.cpp
View File

@ -155,7 +155,7 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl(
}
/// Combinators of aggregate functions.
/// For every aggregate function 'agg' and combiner '-Comb' there is combined aggregate function with name 'aggComb',
/// For every aggregate function 'agg' and combiner '-Comb' there is a combined aggregate function with the name 'aggComb',
/// that can have different number and/or types of arguments, different result type and different behaviour.
if (AggregateFunctionCombinatorPtr combinator = AggregateFunctionCombinatorFactory::instance().tryFindSuffix(name))
@ -172,13 +172,12 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl(
String nested_name = name.substr(0, name.size() - combinator_name.size());
/// Nested identical combinators (i.e. uniqCombinedIfIf) is not
/// supported (since they even don't work -- silently).
/// supported (since they don't work -- silently).
///
/// But non-identical does supported and works, for example
/// uniqCombinedIfMergeIf, it is useful in case when the underlying
/// But non-identical is supported and works. For example,
/// uniqCombinedIfMergeIf is useful in cases when the underlying
/// storage stores AggregateFunction(uniqCombinedIf) and in SELECT you
/// need to filter aggregation result based on another column for
/// example.
/// need to filter aggregation result based on another column.
if (!combinator->supportsNesting() && nested_name.ends_with(combinator_name))
{
throw Exception(ErrorCodes::ILLEGAL_AGGREGATION,
@ -234,7 +233,7 @@ std::optional<AggregateFunctionProperties> AggregateFunctionFactory::tryGetPrope
return found.properties;
/// Combinators of aggregate functions.
/// For every aggregate function 'agg' and combiner '-Comb' there is combined aggregate function with name 'aggComb',
/// For every aggregate function 'agg' and combiner '-Comb' there is a combined aggregate function with the name 'aggComb',
/// that can have different number and/or types of arguments, different result type and different behaviour.
if (AggregateFunctionCombinatorPtr combinator = AggregateFunctionCombinatorFactory::instance().tryFindSuffix(name))

25
src/AggregateFunctions/QuantileTDigest.h
View File

@ -1,8 +1,10 @@
#pragma once
#include <cmath>
#include <Common/Exception.h>
#include <Common/RadixSort.h>
#include <Common/PODArray.h>
#include <Core/AccurateComparison.h>
#include <IO/WriteBuffer.h>
#include <IO/ReadBuffer.h>
#include <IO/VarInt.h>
@ -14,8 +16,9 @@ struct Settings;
namespace ErrorCodes
{
extern const int TOO_LARGE_ARRAY_SIZE;
extern const int CANNOT_PARSE_INPUT_ASSERTION_FAILED;
extern const int DECIMAL_OVERFLOW;
extern const int TOO_LARGE_ARRAY_SIZE;
}
@ -314,7 +317,7 @@ public:
compress();
if (centroids.size() == 1)
return centroids.front().mean;
return checkOverflow<ResultType>(centroids.front().mean);
Float64 x = level * count;
Float64 prev_x = 0;
@ -333,11 +336,11 @@ public:
Float64 right = current_x - 0.5 * (c.count == 1);
if (x <= left)
return prev_mean;
return checkOverflow<ResultType>(prev_mean);
else if (x >= right)
return c.mean;
return checkOverflow<ResultType>(c.mean);
else
return interpolate(x, left, prev_mean, right, c.mean);
return checkOverflow<ResultType>(interpolate(x, left, prev_mean, right, c.mean));
}
sum += c.count;
@ -346,7 +349,7 @@ public:
prev_x = current_x;
}
return centroids.back().mean;
return checkOverflow<ResultType>(centroids.back().mean);
}
/** Get multiple quantiles (`size` parts).
@ -438,6 +441,16 @@ public:
{
getManyImpl(levels, indices, size, result);
}
private:
template <typename ResultType>
static ResultType checkOverflow(Value val)
{
ResultType result;
if (accurate::convertNumeric(val, result))
return result;
throw DB::Exception("Numeric overflow", ErrorCodes::DECIMAL_OVERFLOW);
}
};
}

74
src/AggregateFunctions/ya.make
View File

@ -1,74 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
SRCS(
AggregateFunctionAggThrow.cpp
AggregateFunctionAny.cpp
AggregateFunctionArray.cpp
AggregateFunctionAvg.cpp
AggregateFunctionAvgWeighted.cpp
AggregateFunctionBitwise.cpp
AggregateFunctionBoundingRatio.cpp
AggregateFunctionCategoricalInformationValue.cpp
AggregateFunctionCombinatorFactory.cpp
AggregateFunctionCount.cpp
AggregateFunctionDeltaSum.cpp
AggregateFunctionDeltaSumTimestamp.cpp
AggregateFunctionDistinct.cpp
AggregateFunctionEntropy.cpp
AggregateFunctionFactory.cpp
AggregateFunctionForEach.cpp
AggregateFunctionGroupArray.cpp
AggregateFunctionGroupArrayInsertAt.cpp
AggregateFunctionGroupArrayMoving.cpp
AggregateFunctionGroupUniqArray.cpp
AggregateFunctionHistogram.cpp
AggregateFunctionIf.cpp
AggregateFunctionIntervalLengthSum.cpp
AggregateFunctionMLMethod.cpp
AggregateFunctionMannWhitney.cpp
AggregateFunctionMax.cpp
AggregateFunctionMaxIntersections.cpp
AggregateFunctionMerge.cpp
AggregateFunctionMin.cpp
AggregateFunctionNull.cpp
AggregateFunctionOrFill.cpp
AggregateFunctionQuantile.cpp
AggregateFunctionRankCorrelation.cpp
AggregateFunctionResample.cpp
AggregateFunctionRetention.cpp
AggregateFunctionSequenceMatch.cpp
AggregateFunctionSequenceNextNode.cpp
AggregateFunctionSimpleLinearRegression.cpp
AggregateFunctionSimpleState.cpp
AggregateFunctionSingleValueOrNull.cpp
AggregateFunctionSparkbar.cpp
AggregateFunctionState.cpp
AggregateFunctionStatistics.cpp
AggregateFunctionStatisticsSimple.cpp
AggregateFunctionStudentTTest.cpp
AggregateFunctionSum.cpp
AggregateFunctionSumCount.cpp
AggregateFunctionSumMap.cpp
AggregateFunctionTopK.cpp
AggregateFunctionUniq.cpp
AggregateFunctionUniqCombined.cpp
AggregateFunctionUniqUpTo.cpp
AggregateFunctionWelchTTest.cpp
AggregateFunctionWindowFunnel.cpp
IAggregateFunction.cpp
UniqCombinedBiasData.cpp
UniqVariadicHash.cpp
parseAggregateFunctionParameters.cpp
registerAggregateFunctions.cpp
)
END()

14
src/AggregateFunctions/ya.make.in
View File

@ -1,14 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | grep -v -F GroupBitmap | sed 's/^\.\// /' | sort ?>
)
END()

27
src/Backups/ya.make
View File

@ -1,27 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
SRCS(
BackupEntryConcat.cpp
BackupEntryFromAppendOnlyFile.cpp
BackupEntryFromImmutableFile.cpp
BackupEntryFromMemory.cpp
BackupEntryFromSmallFile.cpp
BackupFactory.cpp
BackupInDirectory.cpp
BackupRenamingConfig.cpp
BackupSettings.cpp
BackupUtils.cpp
hasCompatibleDataToRestoreTable.cpp
renameInCreateQuery.cpp
)
END()

14
src/Backups/ya.make.in
View File

@ -1,14 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | sed 's/^\.\// /' | sort ?>
)
END()

17
src/Bridge/ya.make
View File

@ -1,17 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
SRCS(
IBridgeHelper.cpp
LibraryBridgeHelper.cpp
)
END()

14
src/Bridge/ya.make.in
View File

@ -1,14 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | sed 's/^\.\// /' | sort ?>
)
END()

24
src/Client/ya.make
View File

@ -1,24 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
contrib/libs/poco/NetSSL_OpenSSL
)
SRCS(
Connection.cpp
ConnectionEstablisher.cpp
ConnectionPool.cpp
ConnectionPoolWithFailover.cpp
HedgedConnections.cpp
HedgedConnectionsFactory.cpp
IConnections.cpp
MultiplexedConnections.cpp
)
END()

15
src/Client/ya.make.in
View File

@ -1,15 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
contrib/libs/poco/NetSSL_OpenSSL
)
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | sed 's/^\.\// /' | sort ?>
)
END()

43
src/Columns/ya.make
View File

@ -1,43 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
ADDINCL(
contrib/libs/icu/common
contrib/libs/icu/i18n
contrib/libs/pdqsort
contrib/libs/lz4
)
PEERDIR(
clickhouse/src/Common
contrib/libs/icu
contrib/libs/pdqsort
contrib/libs/lz4
)
SRCS(
Collator.cpp
ColumnAggregateFunction.cpp
ColumnArray.cpp
ColumnCompressed.cpp
ColumnConst.cpp
ColumnDecimal.cpp
ColumnFixedString.cpp
ColumnFunction.cpp
ColumnLowCardinality.cpp
ColumnMap.cpp
ColumnNullable.cpp
ColumnString.cpp
ColumnTuple.cpp
ColumnVector.cpp
ColumnsCommon.cpp
FilterDescription.cpp
IColumn.cpp
MaskOperations.cpp
getLeastSuperColumn.cpp
)
END()

23
src/Columns/ya.make.in
View File

@ -1,23 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
ADDINCL(
contrib/libs/icu/common
contrib/libs/icu/i18n
contrib/libs/pdqsort
contrib/libs/lz4
)
PEERDIR(
clickhouse/src/Common
contrib/libs/icu
contrib/libs/pdqsort
contrib/libs/lz4
)
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | sed 's/^\.\// /' | sort ?>
)
END()

133
src/Common/ya.make
View File

@ -1,133 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
ADDINCL (
GLOBAL clickhouse/src
contrib/libs/libcpuid
contrib/libs/libunwind/include
GLOBAL contrib/restricted/dragonbox
)
PEERDIR(
clickhouse/base/common
clickhouse/base/pcg-random
clickhouse/base/widechar_width
contrib/libs/libcpuid
contrib/libs/openssl
contrib/libs/poco/NetSSL_OpenSSL
contrib/libs/re2
contrib/restricted/dragonbox
)
INCLUDE(${ARCADIA_ROOT}/clickhouse/cmake/yandex/ya.make.versions.inc)
SRCS(
ActionLock.cpp
AlignedBuffer.cpp
Allocator.cpp
ClickHouseRevision.cpp
Config/AbstractConfigurationComparison.cpp
Config/ConfigProcessor.cpp
Config/ConfigReloader.cpp
Config/YAMLParser.cpp
Config/configReadClient.cpp
CurrentMemoryTracker.cpp
CurrentMetrics.cpp
CurrentThread.cpp
DNSResolver.cpp
Dwarf.cpp
Elf.cpp
Epoll.cpp
ErrorCodes.cpp
Exception.cpp
FieldVisitorDump.cpp
FieldVisitorHash.cpp
FieldVisitorSum.cpp
FieldVisitorToString.cpp
FieldVisitorWriteBinary.cpp
FileChecker.cpp
IO.cpp
IPv6ToBinary.cpp
IntervalKind.cpp
JSONBuilder.cpp
Macros.cpp
MemoryStatisticsOS.cpp
MemoryTracker.cpp
OpenSSLHelpers.cpp
OptimizedRegularExpression.cpp
PODArray.cpp
PipeFDs.cpp
ProcfsMetricsProvider.cpp
ProfileEvents.cpp
ProgressIndication.cpp
QueryProfiler.cpp
RWLock.cpp
RemoteHostFilter.cpp
SensitiveDataMasker.cpp
SettingsChanges.cpp
SharedLibrary.cpp
ShellCommand.cpp
StackTrace.cpp
StatusFile.cpp
StatusInfo.cpp
Stopwatch.cpp
StringUtils/StringUtils.cpp
StudentTTest.cpp
SymbolIndex.cpp
TLDListsHolder.cpp
TaskStatsInfoGetter.cpp
TerminalSize.cpp
ThreadFuzzer.cpp
ThreadPool.cpp
ThreadProfileEvents.cpp
ThreadStatus.cpp
Throttler.cpp
TimerDescriptor.cpp
TraceCollector.cpp
UTF8Helpers.cpp
UnicodeBar.cpp
VersionNumber.cpp
WeakHash.cpp
ZooKeeper/IKeeper.cpp
ZooKeeper/TestKeeper.cpp
ZooKeeper/ZooKeeper.cpp
ZooKeeper/ZooKeeperCommon.cpp
ZooKeeper/ZooKeeperConstants.cpp
ZooKeeper/ZooKeeperIO.cpp
ZooKeeper/ZooKeeperImpl.cpp
ZooKeeper/ZooKeeperNodeCache.cpp
checkStackSize.cpp
clearPasswordFromCommandLine.cpp
clickhouse_malloc.cpp
createHardLink.cpp
escapeForFileName.cpp
filesystemHelpers.cpp
formatIPv6.cpp
formatReadable.cpp
getExecutablePath.cpp
getHashOfLoadedBinary.cpp
getMappedArea.cpp
getMultipleKeysFromConfig.cpp
getNumberOfPhysicalCPUCores.cpp
hasLinuxCapability.cpp
hex.cpp
isLocalAddress.cpp
isValidUTF8.cpp
malloc.cpp
new_delete.cpp
parseAddress.cpp
parseGlobs.cpp
parseRemoteDescription.cpp
quoteString.cpp
randomSeed.cpp
remapExecutable.cpp
renameat2.cpp
setThreadName.cpp
thread_local_rng.cpp
)
END()

30
src/Common/ya.make.in
View File

@ -1,30 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
ADDINCL (
GLOBAL clickhouse/src
contrib/libs/libcpuid
contrib/libs/libunwind/include
GLOBAL contrib/restricted/dragonbox
)
PEERDIR(
clickhouse/base/common
clickhouse/base/pcg-random
clickhouse/base/widechar_width
contrib/libs/libcpuid
contrib/libs/openssl
contrib/libs/poco/NetSSL_OpenSSL
contrib/libs/re2
contrib/restricted/dragonbox
)
INCLUDE(${ARCADIA_ROOT}/clickhouse/cmake/yandex/ya.make.versions.inc)
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | sed 's/^\.\// /' | sort ?>
)
END()

42
src/Compression/ya.make
View File

@ -1,42 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
ADDINCL(
contrib/libs/lz4
contrib/libs/zstd/include
)
PEERDIR(
clickhouse/src/Common
contrib/libs/lz4
contrib/libs/zstd
)
SRCS(
CachedCompressedReadBuffer.cpp
CheckingCompressedReadBuffer.cpp
CompressedReadBuffer.cpp
CompressedReadBufferBase.cpp
CompressedReadBufferFromFile.cpp
CompressedWriteBuffer.cpp
CompressionCodecDelta.cpp
CompressionCodecDoubleDelta.cpp
CompressionCodecEncrypted.cpp
CompressionCodecGorilla.cpp
CompressionCodecLZ4.cpp
CompressionCodecMultiple.cpp
CompressionCodecNone.cpp
CompressionCodecT64.cpp
CompressionCodecZSTD.cpp
CompressionFactory.cpp
CompressionFactoryAdditions.cpp
ICompressionCodec.cpp
LZ4_decompress_faster.cpp
getCompressionCodecForFile.cpp
)
END()

21
src/Compression/ya.make.in
View File

@ -1,21 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
ADDINCL(
contrib/libs/lz4
contrib/libs/zstd/include
)
PEERDIR(
clickhouse/src/Common
contrib/libs/lz4
contrib/libs/zstd
)
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | sed 's/^\.\// /' | sort ?>
)
END()

226
src/Coordination/Changelog.cpp
View File

@ -2,6 +2,7 @@
#include <IO/WriteHelpers.h>
#include <IO/ReadHelpers.h>
#include <IO/ReadBufferFromFile.h>
#include <IO/ZstdDeflatingAppendableWriteBuffer.h>
#include <filesystem>
#include <boost/algorithm/string/split.hpp>
#include <boost/algorithm/string/join.hpp>
@ -10,6 +11,7 @@
#include <Common/SipHash.h>
#include <common/logger_useful.h>
namespace DB
{
@ -17,6 +19,7 @@ namespace ErrorCodes
{
extern const int CHECKSUM_DOESNT_MATCH;
extern const int CORRUPTED_DATA;
extern const int UNSUPPORTED_METHOD;
extern const int UNKNOWN_FORMAT_VERSION;
extern const int LOGICAL_ERROR;
}
@ -29,7 +32,7 @@ constexpr auto DEFAULT_PREFIX = "changelog";
std::string formatChangelogPath(const std::string & prefix, const ChangelogFileDescription & name)
{
std::filesystem::path path(prefix);
path /= std::filesystem::path(name.prefix + "_" + std::to_string(name.from_log_index) + "_" + std::to_string(name.to_log_index) + ".bin");
path /= std::filesystem::path(name.prefix + "_" + std::to_string(name.from_log_index) + "_" + std::to_string(name.to_log_index) + "." + name.extension);
return path;
}
@ -46,6 +49,7 @@ ChangelogFileDescription getChangelogFileDescription(const std::string & path_st
result.prefix = filename_parts[0];
result.from_log_index = parse<uint64_t>(filename_parts[1]);
result.to_log_index = parse<uint64_t>(filename_parts[2]);
result.extension = path.extension();
result.path = path_str;
return result;
}
@ -75,52 +79,53 @@ class ChangelogWriter
public:
ChangelogWriter(const std::string & filepath_, WriteMode mode, uint64_t start_index_)
: filepath(filepath_)
, plain_buf(filepath, DBMS_DEFAULT_BUFFER_SIZE, mode == WriteMode::Rewrite ? -1 : (O_APPEND | O_CREAT | O_WRONLY))
, file_buf(filepath, DBMS_DEFAULT_BUFFER_SIZE, mode == WriteMode::Rewrite ? -1 : (O_APPEND | O_CREAT | O_WRONLY))
, start_index(start_index_)
{}
off_t appendRecord(ChangelogRecord && record)
{
off_t result = plain_buf.count();
writeIntBinary(computeRecordChecksum(record), plain_buf);
writeIntBinary(record.header.version, plain_buf);
writeIntBinary(record.header.index, plain_buf);
writeIntBinary(record.header.term, plain_buf);
writeIntBinary(record.header.value_type, plain_buf);
writeIntBinary(record.header.blob_size, plain_buf);
if (record.header.blob_size != 0)
plain_buf.write(reinterpret_cast<char *>(record.blob->data_begin()), record.blob->size());
entries_written++;
return result;
auto compression_method = chooseCompressionMethod(filepath_, "");
if (compression_method != CompressionMethod::Zstd && compression_method != CompressionMethod::None)
{
throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Unsupported coordination log serialization format {}", toContentEncodingName(compression_method));
}
else if (compression_method == CompressionMethod::Zstd)
{
compressed_buffer = std::make_unique<ZstdDeflatingAppendableWriteBuffer>(file_buf, /* compression level = */ 3, /* append_to_existing_stream = */ mode == WriteMode::Append);
}
else
{
/// no compression, only file buffer
}
}
void truncateToLength(off_t new_length)
void appendRecord(ChangelogRecord && record)
{
plain_buf.next();
plain_buf.truncate(new_length);
plain_buf.seek(new_length, SEEK_SET);
writeIntBinary(computeRecordChecksum(record), getBuffer());
writeIntBinary(record.header.version, getBuffer());
writeIntBinary(record.header.index, getBuffer());
writeIntBinary(record.header.term, getBuffer());
writeIntBinary(record.header.value_type, getBuffer());
writeIntBinary(record.header.blob_size, getBuffer());
if (record.header.blob_size != 0)
getBuffer().write(reinterpret_cast<char *>(record.blob->data_begin()), record.blob->size());
}
void flush(bool force_fsync)
{
plain_buf.next();
if (compressed_buffer)
{
/// Flush compressed data to WriteBufferFromFile working_buffer
compressed_buffer->next();
}
/// Flush working buffer to file system
file_buf.next();
/// Fsync file system if needed
if (force_fsync)
plain_buf.sync();
}
uint64_t getEntriesWritten() const
{
return entries_written;
}
void setEntriesWritten(uint64_t entries_written_)
{
entries_written = entries_written_;
file_buf.sync();
}
uint64_t getStartIndex() const
@ -128,15 +133,17 @@ public:
return start_index;
}
void setStartIndex(uint64_t start_index_)
private:
WriteBuffer & getBuffer()
{
start_index = start_index_;
if (compressed_buffer)
return *compressed_buffer;
return file_buf;
}
private:
std::string filepath;
WriteBufferFromFile plain_buf;
uint64_t entries_written = 0;
WriteBufferFromFile file_buf;
std::unique_ptr<WriteBuffer> compressed_buffer;
uint64_t start_index;
};
@ -167,30 +174,32 @@ class ChangelogReader
public:
explicit ChangelogReader(const std::string & filepath_)
: filepath(filepath_)
, read_buf(filepath)
{}
{
auto compression_method = chooseCompressionMethod(filepath, "");
auto read_buffer_from_file = std::make_unique<ReadBufferFromFile>(filepath);
read_buf = wrapReadBufferWithCompressionMethod(std::move(read_buffer_from_file), compression_method);
}
/// start_log_index -- all entries with index < start_log_index will be skipped, but accounted into total_entries_read_from_log
ChangelogReadResult readChangelog(IndexToLogEntry & logs, uint64_t start_log_index, IndexToOffset & index_to_offset, Poco::Logger * log)
ChangelogReadResult readChangelog(IndexToLogEntry & logs, uint64_t start_log_index, Poco::Logger * log)
{
uint64_t previous_index = 0;
ChangelogReadResult result{};
try
{
while (!read_buf.eof())
while (!read_buf->eof())
{
result.last_position = read_buf.count();
result.last_position = read_buf->count();
/// Read checksum
Checksum record_checksum;
readIntBinary(record_checksum, read_buf);
readIntBinary(record_checksum, *read_buf);
/// Read header
ChangelogRecord record;
readIntBinary(record.header.version, read_buf);
readIntBinary(record.header.index, read_buf);
readIntBinary(record.header.term, read_buf);
readIntBinary(record.header.value_type, read_buf);
readIntBinary(record.header.blob_size, read_buf);
readIntBinary(record.header.version, *read_buf);
readIntBinary(record.header.index, *read_buf);
readIntBinary(record.header.term, *read_buf);
readIntBinary(record.header.value_type, *read_buf);
readIntBinary(record.header.blob_size, *read_buf);
if (record.header.version > CURRENT_CHANGELOG_VERSION)
throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, "Unsupported changelog version {} on path {}", record.header.version, filepath);
@ -200,18 +209,12 @@ public:
{
auto buffer = nuraft::buffer::alloc(record.header.blob_size);
auto * buffer_begin = reinterpret_cast<char *>(buffer->data_begin());
read_buf.readStrict(buffer_begin, record.header.blob_size);
read_buf->readStrict(buffer_begin, record.header.blob_size);
record.blob = buffer;
}
else
record.blob = nullptr;
/// Check changelog integrity
if (previous_index != 0 && previous_index + 1 != record.header.index)
throw Exception(ErrorCodes::CORRUPTED_DATA, "Previous log entry {}, next log entry {}, seems like some entries skipped", previous_index, record.header.index);
previous_index = record.header.index;
/// Compare checksums
Checksum checksum = computeRecordChecksum(record);
if (checksum != record_checksum)
@ -223,7 +226,7 @@ public:
/// Check for duplicated changelog ids
if (logs.count(record.header.index) != 0)
throw Exception(ErrorCodes::CORRUPTED_DATA, "Duplicated index id {} in log {}", record.header.index, filepath);
std::erase_if(logs, [record] (const auto & item) { return item.first >= record.header.index; });
result.total_entries_read_from_log += 1;
@ -238,7 +241,6 @@ public:
/// Put it into in memory structure
logs.emplace(record.header.index, log_entry);
index_to_offset[record.header.index] = result.last_position;
result.last_read_index = record.header.index;
if (result.total_entries_read_from_log % 50000 == 0)
@ -266,18 +268,20 @@ public:
private:
std::string filepath;
ReadBufferFromFile read_buf;
std::unique_ptr<ReadBuffer> read_buf;
};
Changelog::Changelog(
const std::string & changelogs_dir_,
uint64_t rotate_interval_,
bool force_sync_,
Poco::Logger * log_)
Poco::Logger * log_,
bool compress_logs_)
: changelogs_dir(changelogs_dir_)
, rotate_interval(rotate_interval_)
, force_sync(force_sync_)
, log(log_)
, compress_logs(compress_logs_)
{
/// Load all files in changelog directory
namespace fs = std::filesystem;
@ -339,10 +343,15 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin
}
}
ChangelogReader reader(changelog_description.path);
last_log_read_result = reader.readChangelog(logs, start_to_read_from, index_to_start_pos, log);
last_log_read_result = reader.readChangelog(logs, start_to_read_from, log);
last_log_read_result->log_start_index = changelog_description.from_log_index;
if (last_log_read_result->error)
{
last_log_is_not_complete = true;
break;
}
/// Otherwise we have already initialized it
if (min_log_id == 0)
min_log_id = last_log_read_result->first_read_index;
@ -350,13 +359,11 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin
if (last_log_read_result->last_read_index != 0)
max_log_id = last_log_read_result->last_read_index;
last_log_read_result->log_start_index = changelog_description.from_log_index;
/// How many entries we have in the last changelog
uint64_t expected_entries_in_log = changelog_description.expectedEntriesCountInLog();
/// May happen after truncate, crash or simply unfinished log
if (last_log_read_result->total_entries_read_from_log < expected_entries_in_log)
/// Unfinished log
if (last_log_read_result->error || last_log_read_result->total_entries_read_from_log < expected_entries_in_log)
{
last_log_is_not_complete = true;
break;
@ -376,19 +383,29 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin
else if (last_log_is_not_complete) /// if it's complete just start new one
{
assert(last_log_read_result != std::nullopt);
assert(!existing_changelogs.empty());
/// Actually they shouldn't exist, but to be sure we remove them
removeAllLogsAfter(last_log_read_result->log_start_index);
assert(!existing_changelogs.empty());
/// This log, even if it finished with error shouldn't be removed
assert(existing_changelogs.find(last_log_read_result->log_start_index) != existing_changelogs.end());
assert(existing_changelogs.find(last_log_read_result->log_start_index)->first == existing_changelogs.rbegin()->first);
/// Continue to write into incomplete existing log
/// Continue to write into incomplete existing log if it doesn't finished with error
auto description = existing_changelogs[last_log_read_result->log_start_index];
if (last_log_read_result->error)
initWriter(description, last_log_read_result->total_entries_read_from_log, /* truncate_to_offset = */ last_log_read_result->last_position);
if (last_log_read_result->last_read_index == 0 || last_log_read_result->error) /// If it's broken log then remove it
{
LOG_INFO(log, "Removing log {} because it's empty or read finished with error", description.path);
std::filesystem::remove(description.path);
existing_changelogs.erase(last_log_read_result->log_start_index);
std::erase_if(logs, [last_log_read_result] (const auto & item) { return item.first >= last_log_read_result->log_start_index; });
}
else
initWriter(description, last_log_read_result->total_entries_read_from_log);
{
initWriter(description);
}
}
/// Start new log if we don't initialize writer from previous log. All logs can be "complete".
@ -397,33 +414,32 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin
}
void Changelog::initWriter(const ChangelogFileDescription & description, uint64_t entries_already_written, std::optional<uint64_t> truncate_to_offset)
void Changelog::initWriter(const ChangelogFileDescription & description)
{
if (description.expectedEntriesCountInLog() != rotate_interval)
LOG_TRACE(log, "Looks like rotate_logs_interval was changed, current {}, expected entries in last log {}", rotate_interval, description.expectedEntriesCountInLog());
LOG_TRACE(log, "Continue to write into {}", description.path);
current_writer = std::make_unique<ChangelogWriter>(description.path, WriteMode::Append, description.from_log_index);
current_writer->setEntriesWritten(entries_already_written);
if (truncate_to_offset)
{
LOG_WARNING(log, "Changelog {} contain broken enties, truncating all broken log entries", description.path);
current_writer->truncateToLength(*truncate_to_offset);
}
}
void Changelog::removeAllLogsAfter(uint64_t start_to_remove_from_id)
void Changelog::removeAllLogsAfter(uint64_t remove_after_log_start_index)
{
auto start_to_remove_from = existing_changelogs.upper_bound(start_to_remove_from_id);
auto start_to_remove_from_itr = existing_changelogs.upper_bound(remove_after_log_start_index);
if (start_to_remove_from_itr == existing_changelogs.end())
return;
size_t start_to_remove_from_log_id = start_to_remove_from_itr->first;
/// All subsequent logs shouldn't exist. But they may exist if we crashed after writeAt started. Remove them.
for (auto itr = start_to_remove_from; itr != existing_changelogs.end();)
for (auto itr = start_to_remove_from_itr; itr != existing_changelogs.end();)
{
LOG_WARNING(log, "Removing changelog {}, because it's goes after broken changelog entry", itr->second.path);
std::filesystem::remove(itr->second.path);
itr = existing_changelogs.erase(itr);
}
std::erase_if(logs, [start_to_remove_from_log_id] (const auto & item) { return item.first >= start_to_remove_from_log_id; });
}
void Changelog::removeAllLogs()
@ -435,6 +451,7 @@ void Changelog::removeAllLogs()
std::filesystem::remove(itr->second.path);
itr = existing_changelogs.erase(itr);
}
logs.clear();
}
void Changelog::rotate(uint64_t new_start_log_index)
@ -447,6 +464,10 @@ void Changelog::rotate(uint64_t new_start_log_index)
new_description.prefix = DEFAULT_PREFIX;
new_description.from_log_index = new_start_log_index;
new_description.to_log_index = new_start_log_index + rotate_interval - 1;
new_description.extension = "bin";
if (compress_logs)
new_description.extension += "." + toContentEncodingName(CompressionMethod::Zstd);
new_description.path = formatChangelogPath(changelogs_dir, new_description);
@ -482,24 +503,18 @@ void Changelog::appendEntry(uint64_t index, const LogEntryPtr & log_entry)
min_log_id = index;
const auto & current_changelog_description = existing_changelogs[current_writer->getStartIndex()];
const bool log_is_complete = current_writer->getEntriesWritten() == current_changelog_description.expectedEntriesCountInLog();
const bool log_is_complete = index - current_writer->getStartIndex() == current_changelog_description.expectedEntriesCountInLog();
if (log_is_complete)
rotate(index);
const auto offset = current_writer->appendRecord(buildRecord(index, log_entry));
if (!index_to_start_pos.try_emplace(index, offset).second)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Record with index {} already exists", index);
current_writer->appendRecord(buildRecord(index, log_entry));
logs[index] = makeClone(log_entry);
max_log_id = index;
}
void Changelog::writeAt(uint64_t index, const LogEntryPtr & log_entry)
{
if (index_to_start_pos.count(index) == 0)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot write at index {} because changelog doesn't contain it", index);
/// This write_at require to overwrite everything in this file and also in previous file(s)
const bool go_to_previous_file = index < current_writer->getStartIndex();
@ -516,14 +531,7 @@ void Changelog::writeAt(uint64_t index, const LogEntryPtr & log_entry)
/// Initialize writer from this log file
current_writer = std::make_unique<ChangelogWriter>(description.path, WriteMode::Append, index_changelog->first);
current_writer->setEntriesWritten(description.to_log_index - description.from_log_index + 1);
}
/// Truncate current file
current_writer->truncateToLength(index_to_start_pos[index]);
if (go_to_previous_file)
{
/// Remove all subsequent files if overwritten something in previous one
auto to_remove_itr = existing_changelogs.upper_bound(index);
for (auto itr = to_remove_itr; itr != existing_changelogs.end();)
@ -533,20 +541,9 @@ void Changelog::writeAt(uint64_t index, const LogEntryPtr & log_entry)
}
}
auto entries_written = current_writer->getEntriesWritten();
/// Remove redundant logs from memory
/// Everything >= index must be removed
for (uint64_t i = index; ; ++i)
{
auto log_itr = logs.find(i);
if (log_itr == logs.end())
break;
logs.erase(log_itr);
index_to_start_pos.erase(i);
entries_written--;
}
current_writer->setEntriesWritten(entries_written);
std::erase_if(logs, [index] (const auto & item) { return item.first >= index; });
/// Now we can actually override entry at index
appendEntry(index, log_entry);
@ -579,7 +576,6 @@ void Changelog::compact(uint64_t up_to_log_index)
}
LOG_INFO(log, "Removing changelog {} because of compaction", itr->second.path);
std::erase_if(index_to_start_pos, [right_index = itr->second.to_log_index] (const auto & item) { return item.first <= right_index; });
std::filesystem::remove(itr->second.path);
itr = existing_changelogs.erase(itr);
}

16
src/Coordination/Changelog.h
View File

@ -5,7 +5,7 @@
#include <optional>
#include <IO/WriteBufferFromFile.h>
#include <IO/HashingWriteBuffer.h>
#include <Compression/CompressedWriteBuffer.h>
#include <IO/CompressionMethod.h>
#include <Disks/IDisk.h>
namespace DB
@ -25,9 +25,10 @@ enum class ChangelogVersion : uint8_t
{
V0 = 0,
V1 = 1, /// with 64 bit buffer header
V2 = 2, /// with compression and duplicate records
};
static constexpr auto CURRENT_CHANGELOG_VERSION = ChangelogVersion::V1;
static constexpr auto CURRENT_CHANGELOG_VERSION = ChangelogVersion::V2;
struct ChangelogRecordHeader
{
@ -52,6 +53,7 @@ struct ChangelogFileDescription
std::string prefix;
uint64_t from_log_index;
uint64_t to_log_index;
std::string extension;
std::string path;
@ -71,7 +73,8 @@ class Changelog
{
public:
Changelog(const std::string & changelogs_dir_, uint64_t rotate_interval_, bool force_sync_, Poco::Logger * log_);
Changelog(const std::string & changelogs_dir_, uint64_t rotate_interval_,
bool force_sync_, Poco::Logger * log_, bool compress_logs_ = true);
/// Read changelog from files on changelogs_dir_ skipping all entries before from_log_index
/// Truncate broken entries, remove files after broken entries.
@ -130,25 +133,24 @@ private:
void rotate(uint64_t new_start_log_index);
/// Remove all changelogs from disk with start_index bigger than start_to_remove_from_id
void removeAllLogsAfter(uint64_t start_to_remove_from_id);
void removeAllLogsAfter(uint64_t remove_after_log_start_index);
/// Remove all logs from disk
void removeAllLogs();
/// Init writer for existing log with some entries already written
void initWriter(const ChangelogFileDescription & description, uint64_t entries_already_written, std::optional<uint64_t> truncate_to_offset = {});
void initWriter(const ChangelogFileDescription & description);
private:
const std::string changelogs_dir;
const uint64_t rotate_interval;
const bool force_sync;
Poco::Logger * log;
bool compress_logs;
/// Currently existing changelogs
std::map<uint64_t, ChangelogFileDescription> existing_changelogs;
/// Current writer for changelog file
std::unique_ptr<ChangelogWriter> current_writer;
/// Mapping log_id -> binary offset in log file
IndexToOffset index_to_start_pos;
/// Mapping log_id -> log_entry
IndexToLogEntry logs;
/// Start log_id which exists in all "active" logs

4
src/Coordination/CoordinationSettings.h
View File

@ -11,6 +11,7 @@ namespace DB
struct Settings;
/** These settings represent fine tunes for internal details of Coordination storages
* and should not be changed by the user without a reason.
*/
@ -34,7 +35,8 @@ struct Settings;
M(UInt64, fresh_log_gap, 200, "When node became fresh", 0) \
M(UInt64, max_requests_batch_size, 100, "Max size of batch in requests count before it will be sent to RAFT", 0) \
M(Bool, quorum_reads, false, "Execute read requests as writes through whole RAFT consesus with similar speed", 0) \
M(Bool, force_sync, true, "Call fsync on each change in RAFT changelog", 0)
M(Bool, force_sync, true, "Call fsync on each change in RAFT changelog", 0) \
M(Bool, compress_logs, true, "Write compressed coordination logs in ZSTD format", 0)
DECLARE_SETTINGS_TRAITS(CoordinationSettingsTraits, LIST_OF_COORDINATION_SETTINGS)

5
src/Coordination/KeeperLogStore.cpp
View File

@ -1,11 +1,12 @@
#include <Coordination/KeeperLogStore.h>
#include <IO/CompressionMethod.h>
namespace DB
{
KeeperLogStore::KeeperLogStore(const std::string & changelogs_path, uint64_t rotate_interval_, bool force_sync_)
KeeperLogStore::KeeperLogStore(const std::string & changelogs_path, uint64_t rotate_interval_, bool force_sync_, bool compress_logs_)
: log(&Poco::Logger::get("KeeperLogStore"))
, changelog(changelogs_path, rotate_interval_, force_sync_, log)
, changelog(changelogs_path, rotate_interval_, force_sync_, log, compress_logs_)
{
if (force_sync_)
LOG_INFO(log, "force_sync enabled");

2
src/Coordination/KeeperLogStore.h
View File

@ -13,7 +13,7 @@ namespace DB
class KeeperLogStore : public nuraft::log_store
{
public:
KeeperLogStore(const std::string & changelogs_path, uint64_t rotate_interval_, bool force_sync_);
KeeperLogStore(const std::string & changelogs_path, uint64_t rotate_interval_, bool force_sync_, bool compress_logs_);
/// Read log storage from filesystem starting from last_commited_log_index
void init(uint64_t last_commited_log_index, uint64_t logs_to_keep);

4
src/Coordination/KeeperStateManager.cpp
View File

@ -34,7 +34,7 @@ KeeperStateManager::KeeperStateManager(int server_id_, const std::string & host,
: my_server_id(server_id_)
, my_port(port)
, secure(false)
, log_store(nuraft::cs_new<KeeperLogStore>(logs_path, 5000, false))
, log_store(nuraft::cs_new<KeeperLogStore>(logs_path, 5000, false, false))
, cluster_config(nuraft::cs_new<nuraft::cluster_config>())
{
auto peer_config = nuraft::cs_new<nuraft::srv_config>(my_server_id, host + ":" + std::to_string(port));
@ -51,7 +51,7 @@ KeeperStateManager::KeeperStateManager(
, secure(config.getBool(config_prefix + ".raft_configuration.secure", false))
, log_store(nuraft::cs_new<KeeperLogStore>(
getLogsPathFromConfig(config_prefix, config, standalone_keeper),
coordination_settings->rotate_log_storage_interval, coordination_settings->force_sync))
coordination_settings->rotate_log_storage_interval, coordination_settings->force_sync, coordination_settings->compress_logs))
, cluster_config(nuraft::cs_new<nuraft::cluster_config>())
{

477
src/Coordination/tests/gtest_coordination.cpp
View File

@ -53,14 +53,23 @@ struct ChangelogDirTest
}
};
TEST(CoordinationTest, BuildTest)
struct CompressionParam
{
bool enable_compression;
std::string extension;
};
class CoordinationTest : public ::testing::TestWithParam<CompressionParam>
{};
TEST_P(CoordinationTest, BuildTest)
{
DB::InMemoryLogStore store;
DB::SummingStateMachine machine;
EXPECT_EQ(1, 1);
}
TEST(CoordinationTest, BufferSerde)
TEST_P(CoordinationTest, BufferSerde)
{
Coordination::ZooKeeperRequestPtr request = Coordination::ZooKeeperRequestFactory::instance().get(Coordination::OpNum::Get);
request->xid = 3;
@ -173,7 +182,7 @@ nuraft::ptr<nuraft::buffer> getBuffer(int64_t number)
}
TEST(CoordinationTest, TestSummingRaft1)
TEST_P(CoordinationTest, TestSummingRaft1)
{
ChangelogDirTest test("./logs");
SummingRaftServer s1(1, "localhost", 44444, "./logs");
@ -204,10 +213,11 @@ DB::LogEntryPtr getLogEntry(const std::string & s, size_t term)
return nuraft::cs_new<nuraft::log_entry>(term, bufwriter.getBuffer());
}
TEST(CoordinationTest, ChangelogTestSimple)
TEST_P(CoordinationTest, ChangelogTestSimple)
{
auto params = GetParam();
ChangelogDirTest test("./logs");
DB::KeeperLogStore changelog("./logs", 5, true);
DB::KeeperLogStore changelog("./logs", 5, true, params.enable_compression);
changelog.init(1, 0);
auto entry = getLogEntry("hello world", 77);
changelog.append(entry);
@ -220,17 +230,19 @@ TEST(CoordinationTest, ChangelogTestSimple)
EXPECT_EQ(changelog.log_entries(1, 2)->size(), 1);
}
TEST(CoordinationTest, ChangelogTestFile)
TEST_P(CoordinationTest, ChangelogTestFile)
{
auto params = GetParam();
ChangelogDirTest test("./logs");
DB::KeeperLogStore changelog("./logs", 5, true);
DB::KeeperLogStore changelog("./logs", 5, true, params.enable_compression);
changelog.init(1, 0);
auto entry = getLogEntry("hello world", 77);
changelog.append(entry);
changelog.end_of_append_batch(0, 0);
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
for (const auto & p : fs::directory_iterator("./logs"))
EXPECT_EQ(p.path(), "./logs/changelog_1_5.bin");
EXPECT_EQ(p.path(), "./logs/changelog_1_5.bin" + params.extension);
changelog.append(entry);
changelog.append(entry);
@ -239,14 +251,15 @@ TEST(CoordinationTest, ChangelogTestFile)
changelog.append(entry);
changelog.end_of_append_batch(0, 0);
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
}
TEST(CoordinationTest, ChangelogReadWrite)
TEST_P(CoordinationTest, ChangelogReadWrite)
{
auto params = GetParam();
ChangelogDirTest test("./logs");
DB::KeeperLogStore changelog("./logs", 1000, true);
DB::KeeperLogStore changelog("./logs", 1000, true, params.enable_compression);
changelog.init(1, 0);
for (size_t i = 0; i < 10; ++i)
{
@ -256,7 +269,8 @@ TEST(CoordinationTest, ChangelogReadWrite)
changelog.end_of_append_batch(0, 0);
EXPECT_EQ(changelog.size(), 10);
DB::KeeperLogStore changelog_reader("./logs", 1000, true);
DB::KeeperLogStore changelog_reader("./logs", 1000, true, params.enable_compression);
changelog_reader.init(1, 0);
EXPECT_EQ(changelog_reader.size(), 10);
EXPECT_EQ(changelog_reader.last_entry()->get_term(), changelog.last_entry()->get_term());
@ -272,10 +286,11 @@ TEST(CoordinationTest, ChangelogReadWrite)
EXPECT_EQ(10, entries_from_range->size());
}
TEST(CoordinationTest, ChangelogWriteAt)
TEST_P(CoordinationTest, ChangelogWriteAt)
{
auto params = GetParam();
ChangelogDirTest test("./logs");
DB::KeeperLogStore changelog("./logs", 1000, true);
DB::KeeperLogStore changelog("./logs", 1000, true, params.enable_compression);
changelog.init(1, 0);
for (size_t i = 0; i < 10; ++i)
{
@ -295,7 +310,7 @@ TEST(CoordinationTest, ChangelogWriteAt)
EXPECT_EQ(changelog.entry_at(7)->get_term(), 77);
EXPECT_EQ(changelog.next_slot(), 8);
DB::KeeperLogStore changelog_reader("./logs", 1000, true);
DB::KeeperLogStore changelog_reader("./logs", 1000, true, params.enable_compression);
changelog_reader.init(1, 0);
EXPECT_EQ(changelog_reader.size(), changelog.size());
@ -305,10 +320,11 @@ TEST(CoordinationTest, ChangelogWriteAt)
}
TEST(CoordinationTest, ChangelogTestAppendAfterRead)
TEST_P(CoordinationTest, ChangelogTestAppendAfterRead)
{
auto params = GetParam();
ChangelogDirTest test("./logs");
DB::KeeperLogStore changelog("./logs", 5, true);
DB::KeeperLogStore changelog("./logs", 5, true, params.enable_compression);
changelog.init(1, 0);
for (size_t i = 0; i < 7; ++i)
{
@ -318,10 +334,10 @@ TEST(CoordinationTest, ChangelogTestAppendAfterRead)
changelog.end_of_append_batch(0, 0);
EXPECT_EQ(changelog.size(), 7);
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
DB::KeeperLogStore changelog_reader("./logs", 5, true);
DB::KeeperLogStore changelog_reader("./logs", 5, true, params.enable_compression);
changelog_reader.init(1, 0);
EXPECT_EQ(changelog_reader.size(), 7);
@ -332,8 +348,8 @@ TEST(CoordinationTest, ChangelogTestAppendAfterRead)
}
changelog_reader.end_of_append_batch(0, 0);
EXPECT_EQ(changelog_reader.size(), 10);
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
size_t logs_count = 0;
for (const auto & _ [[maybe_unused]]: fs::directory_iterator("./logs"))
@ -345,9 +361,9 @@ TEST(CoordinationTest, ChangelogTestAppendAfterRead)
changelog_reader.append(entry);
changelog_reader.end_of_append_batch(0, 0);
EXPECT_EQ(changelog_reader.size(), 11);
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
logs_count = 0;
for (const auto & _ [[maybe_unused]]: fs::directory_iterator("./logs"))
@ -356,10 +372,11 @@ TEST(CoordinationTest, ChangelogTestAppendAfterRead)
EXPECT_EQ(logs_count, 3);
}
TEST(CoordinationTest, ChangelogTestCompaction)
TEST_P(CoordinationTest, ChangelogTestCompaction)
{
auto params = GetParam();
ChangelogDirTest test("./logs");
DB::KeeperLogStore changelog("./logs", 5, true);
DB::KeeperLogStore changelog("./logs", 5, true, params.enable_compression);
changelog.init(1, 0);
for (size_t i = 0; i < 3; ++i)
@ -377,7 +394,7 @@ TEST(CoordinationTest, ChangelogTestCompaction)
EXPECT_EQ(changelog.start_index(), 3);
EXPECT_EQ(changelog.next_slot(), 4);
EXPECT_EQ(changelog.last_entry()->get_term(), 20);
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
auto e1 = getLogEntry("hello world", 30);
changelog.append(e1);
@ -389,20 +406,20 @@ TEST(CoordinationTest, ChangelogTestCompaction)
changelog.append(e4);
changelog.end_of_append_batch(0, 0);
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
changelog.compact(6);
EXPECT_FALSE(fs::exists("./logs/changelog_1_5.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
EXPECT_EQ(changelog.size(), 1);
EXPECT_EQ(changelog.start_index(), 7);
EXPECT_EQ(changelog.next_slot(), 8);
EXPECT_EQ(changelog.last_entry()->get_term(), 60);
/// And we able to read it
DB::KeeperLogStore changelog_reader("./logs", 5, true);
DB::KeeperLogStore changelog_reader("./logs", 5, true, params.enable_compression);
changelog_reader.init(7, 0);
EXPECT_EQ(changelog_reader.size(), 1);
@ -411,10 +428,11 @@ TEST(CoordinationTest, ChangelogTestCompaction)
EXPECT_EQ(changelog_reader.last_entry()->get_term(), 60);
}
TEST(CoordinationTest, ChangelogTestBatchOperations)
TEST_P(CoordinationTest, ChangelogTestBatchOperations)
{
auto params = GetParam();
ChangelogDirTest test("./logs");
DB::KeeperLogStore changelog("./logs", 100, true);
DB::KeeperLogStore changelog("./logs", 100, true, params.enable_compression);
changelog.init(1, 0);
for (size_t i = 0; i < 10; ++i)
{
@ -427,7 +445,7 @@ TEST(CoordinationTest, ChangelogTestBatchOperations)
auto entries = changelog.pack(1, 5);
DB::KeeperLogStore apply_changelog("./logs", 100, true);
DB::KeeperLogStore apply_changelog("./logs", 100, true, params.enable_compression);
apply_changelog.init(1, 0);
for (size_t i = 0; i < 10; ++i)
@ -455,10 +473,11 @@ TEST(CoordinationTest, ChangelogTestBatchOperations)
EXPECT_EQ(apply_changelog.entry_at(12)->get_term(), 40);
}
TEST(CoordinationTest, ChangelogTestBatchOperationsEmpty)
TEST_P(CoordinationTest, ChangelogTestBatchOperationsEmpty)
{
auto params = GetParam();
ChangelogDirTest test("./logs");
DB::KeeperLogStore changelog("./logs", 100, true);
DB::KeeperLogStore changelog("./logs", 100, true, params.enable_compression);
changelog.init(1, 0);
for (size_t i = 0; i < 10; ++i)
{
@ -472,7 +491,7 @@ TEST(CoordinationTest, ChangelogTestBatchOperationsEmpty)
auto entries = changelog.pack(5, 5);
ChangelogDirTest test1("./logs1");
DB::KeeperLogStore changelog_new("./logs1", 100, true);
DB::KeeperLogStore changelog_new("./logs1", 100, true, params.enable_compression);
changelog_new.init(1, 0);
EXPECT_EQ(changelog_new.size(), 0);
@ -494,15 +513,16 @@ TEST(CoordinationTest, ChangelogTestBatchOperationsEmpty)
EXPECT_EQ(changelog_new.start_index(), 5);
EXPECT_EQ(changelog_new.next_slot(), 11);
DB::KeeperLogStore changelog_reader("./logs1", 100, true);
DB::KeeperLogStore changelog_reader("./logs1", 100, true, params.enable_compression);
changelog_reader.init(5, 0);
}
TEST(CoordinationTest, ChangelogTestWriteAtPreviousFile)
TEST_P(CoordinationTest, ChangelogTestWriteAtPreviousFile)
{
auto params = GetParam();
ChangelogDirTest test("./logs");
DB::KeeperLogStore changelog("./logs", 5, true);
DB::KeeperLogStore changelog("./logs", 5, true, params.enable_compression);
changelog.init(1, 0);
for (size_t i = 0; i < 33; ++i)
@ -512,13 +532,13 @@ TEST(CoordinationTest, ChangelogTestWriteAtPreviousFile)
}
changelog.end_of_append_batch(0, 0);
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
EXPECT_EQ(changelog.size(), 33);
@ -530,16 +550,16 @@ TEST(CoordinationTest, ChangelogTestWriteAtPreviousFile)
EXPECT_EQ(changelog.next_slot(), 8);
EXPECT_EQ(changelog.last_entry()->get_term(), 5555);
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_11_15.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_16_20.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_21_25.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_26_30.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_31_35.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
DB::KeeperLogStore changelog_read("./logs", 5, true);
DB::KeeperLogStore changelog_read("./logs", 5, true, params.enable_compression);
changelog_read.init(1, 0);
EXPECT_EQ(changelog_read.size(), 7);
EXPECT_EQ(changelog_read.start_index(), 1);
@ -547,10 +567,11 @@ TEST(CoordinationTest, ChangelogTestWriteAtPreviousFile)
EXPECT_EQ(changelog_read.last_entry()->get_term(), 5555);
}
TEST(CoordinationTest, ChangelogTestWriteAtFileBorder)
TEST_P(CoordinationTest, ChangelogTestWriteAtFileBorder)
{
auto params = GetParam();
ChangelogDirTest test("./logs");
DB::KeeperLogStore changelog("./logs", 5, true);
DB::KeeperLogStore changelog("./logs", 5, true, params.enable_compression);
changelog.init(1, 0);
for (size_t i = 0; i < 33; ++i)
@ -560,13 +581,13 @@ TEST(CoordinationTest, ChangelogTestWriteAtFileBorder)
}
changelog.end_of_append_batch(0, 0);
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
EXPECT_EQ(changelog.size(), 33);
@ -578,16 +599,16 @@ TEST(CoordinationTest, ChangelogTestWriteAtFileBorder)
EXPECT_EQ(changelog.next_slot(), 12);
EXPECT_EQ(changelog.last_entry()->get_term(), 5555);
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_16_20.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_21_25.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_26_30.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_31_35.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
DB::KeeperLogStore changelog_read("./logs", 5, true);
DB::KeeperLogStore changelog_read("./logs", 5, true, params.enable_compression);
changelog_read.init(1, 0);
EXPECT_EQ(changelog_read.size(), 11);
EXPECT_EQ(changelog_read.start_index(), 1);
@ -595,10 +616,11 @@ TEST(CoordinationTest, ChangelogTestWriteAtFileBorder)
EXPECT_EQ(changelog_read.last_entry()->get_term(), 5555);
}
TEST(CoordinationTest, ChangelogTestWriteAtAllFiles)
TEST_P(CoordinationTest, ChangelogTestWriteAtAllFiles)
{
auto params = GetParam();
ChangelogDirTest test("./logs");
DB::KeeperLogStore changelog("./logs", 5, true);
DB::KeeperLogStore changelog("./logs", 5, true, params.enable_compression);
changelog.init(1, 0);
for (size_t i = 0; i < 33; ++i)
@ -608,13 +630,13 @@ TEST(CoordinationTest, ChangelogTestWriteAtAllFiles)
}
changelog.end_of_append_batch(0, 0);
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
EXPECT_EQ(changelog.size(), 33);
@ -626,20 +648,21 @@ TEST(CoordinationTest, ChangelogTestWriteAtAllFiles)
EXPECT_EQ(changelog.next_slot(), 2);
EXPECT_EQ(changelog.last_entry()->get_term(), 5555);
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_6_10.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_11_15.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_16_20.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_21_25.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_26_30.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_31_35.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
}
TEST(CoordinationTest, ChangelogTestStartNewLogAfterRead)
TEST_P(CoordinationTest, ChangelogTestStartNewLogAfterRead)
{
auto params = GetParam();
ChangelogDirTest test("./logs");
DB::KeeperLogStore changelog("./logs", 5, true);
DB::KeeperLogStore changelog("./logs", 5, true, params.enable_compression);
changelog.init(1, 0);
for (size_t i = 0; i < 35; ++i)
@ -649,17 +672,17 @@ TEST(CoordinationTest, ChangelogTestStartNewLogAfterRead)
}
changelog.end_of_append_batch(0, 0);
EXPECT_EQ(changelog.size(), 35);
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_36_40.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_36_40.bin" + params.extension));
DB::KeeperLogStore changelog_reader("./logs", 5, true);
DB::KeeperLogStore changelog_reader("./logs", 5, true, params.enable_compression);
changelog_reader.init(1, 0);
auto entry = getLogEntry("36_hello_world", 360);
@ -667,22 +690,23 @@ TEST(CoordinationTest, ChangelogTestStartNewLogAfterRead)
changelog_reader.end_of_append_batch(0, 0);
EXPECT_EQ(changelog_reader.size(), 36);
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_36_40.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_36_40.bin" + params.extension));
}
TEST(CoordinationTest, ChangelogTestReadAfterBrokenTruncate)
TEST_P(CoordinationTest, ChangelogTestReadAfterBrokenTruncate)
{
auto params = GetParam();
ChangelogDirTest test("./logs");
DB::KeeperLogStore changelog("./logs", 5, true);
DB::KeeperLogStore changelog("./logs", 5, true, params.enable_compression);
changelog.init(1, 0);
for (size_t i = 0; i < 35; ++i)
@ -692,32 +716,32 @@ TEST(CoordinationTest, ChangelogTestReadAfterBrokenTruncate)
}
changelog.end_of_append_batch(0, 0);
EXPECT_EQ(changelog.size(), 35);
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
DB::WriteBufferFromFile plain_buf("./logs/changelog_11_15.bin", DBMS_DEFAULT_BUFFER_SIZE, O_APPEND | O_CREAT | O_WRONLY);
DB::WriteBufferFromFile plain_buf("./logs/changelog_11_15.bin" + params.extension, DBMS_DEFAULT_BUFFER_SIZE, O_APPEND | O_CREAT | O_WRONLY);
plain_buf.truncate(0);
DB::KeeperLogStore changelog_reader("./logs", 5, true);
DB::KeeperLogStore changelog_reader("./logs", 5, true, params.enable_compression);
changelog_reader.init(1, 0);
changelog_reader.end_of_append_batch(0, 0);
EXPECT_EQ(changelog_reader.size(), 10);
EXPECT_EQ(changelog_reader.last_entry()->get_term(), 90);
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_16_20.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_21_25.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_26_30.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_31_35.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
auto entry = getLogEntry("h", 7777);
changelog_reader.append(entry);
@ -725,26 +749,27 @@ TEST(CoordinationTest, ChangelogTestReadAfterBrokenTruncate)
EXPECT_EQ(changelog_reader.size(), 11);
EXPECT_EQ(changelog_reader.last_entry()->get_term(), 7777);
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_16_20.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_21_25.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_26_30.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_31_35.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
DB::KeeperLogStore changelog_reader2("./logs", 5, true);
DB::KeeperLogStore changelog_reader2("./logs", 5, true, params.enable_compression);
changelog_reader2.init(1, 0);
EXPECT_EQ(changelog_reader2.size(), 11);
EXPECT_EQ(changelog_reader2.last_entry()->get_term(), 7777);
}
TEST(CoordinationTest, ChangelogTestReadAfterBrokenTruncate2)
TEST_P(CoordinationTest, ChangelogTestReadAfterBrokenTruncate2)
{
auto params = GetParam();
ChangelogDirTest test("./logs");
DB::KeeperLogStore changelog("./logs", 20, true);
DB::KeeperLogStore changelog("./logs", 20, true, params.enable_compression);
changelog.init(1, 0);
for (size_t i = 0; i < 35; ++i)
@ -754,37 +779,37 @@ TEST(CoordinationTest, ChangelogTestReadAfterBrokenTruncate2)
}
changelog.end_of_append_batch(0, 0);
EXPECT_TRUE(fs::exists("./logs/changelog_1_20.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_21_40.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_20.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_21_40.bin" + params.extension));
DB::WriteBufferFromFile plain_buf("./logs/changelog_1_20.bin", DBMS_DEFAULT_BUFFER_SIZE, O_APPEND | O_CREAT | O_WRONLY);
DB::WriteBufferFromFile plain_buf("./logs/changelog_1_20.bin" + params.extension, DBMS_DEFAULT_BUFFER_SIZE, O_APPEND | O_CREAT | O_WRONLY);
plain_buf.truncate(140);
DB::KeeperLogStore changelog_reader("./logs", 20, true);
DB::KeeperLogStore changelog_reader("./logs", 20, true, params.enable_compression);
changelog_reader.init(1, 0);
EXPECT_EQ(changelog_reader.size(), 2);
EXPECT_EQ(changelog_reader.last_entry()->get_term(), 450);
EXPECT_TRUE(fs::exists("./logs/changelog_1_20.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_21_40.bin"));
EXPECT_EQ(changelog_reader.size(), 0);
EXPECT_TRUE(fs::exists("./logs/changelog_1_20.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_21_40.bin" + params.extension));
auto entry = getLogEntry("hello_world", 7777);
changelog_reader.append(entry);
changelog_reader.end_of_append_batch(0, 0);
EXPECT_EQ(changelog_reader.size(), 3);
EXPECT_EQ(changelog_reader.size(), 1);
EXPECT_EQ(changelog_reader.last_entry()->get_term(), 7777);
DB::KeeperLogStore changelog_reader2("./logs", 20, true);
DB::KeeperLogStore changelog_reader2("./logs", 1, true, params.enable_compression);
changelog_reader2.init(1, 0);
EXPECT_EQ(changelog_reader2.size(), 3);
EXPECT_EQ(changelog_reader2.size(), 1);
EXPECT_EQ(changelog_reader2.last_entry()->get_term(), 7777);
}
TEST(CoordinationTest, ChangelogTestLostFiles)
TEST_P(CoordinationTest, ChangelogTestLostFiles)
{
auto params = GetParam();
ChangelogDirTest test("./logs");
DB::KeeperLogStore changelog("./logs", 20, true);
DB::KeeperLogStore changelog("./logs", 20, true, params.enable_compression);
changelog.init(1, 0);
for (size_t i = 0; i < 35; ++i)
@ -794,19 +819,19 @@ TEST(CoordinationTest, ChangelogTestLostFiles)
}
changelog.end_of_append_batch(0, 0);
EXPECT_TRUE(fs::exists("./logs/changelog_1_20.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_21_40.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_20.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_21_40.bin" + params.extension));
fs::remove("./logs/changelog_1_20.bin");
fs::remove("./logs/changelog_1_20.bin" + params.extension);
DB::KeeperLogStore changelog_reader("./logs", 20, true);
DB::KeeperLogStore changelog_reader("./logs", 20, true, params.enable_compression);
/// It should print error message, but still able to start
changelog_reader.init(5, 0);
EXPECT_FALSE(fs::exists("./logs/changelog_1_20.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_21_40.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_1_20.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_21_40.bin" + params.extension));
}
TEST(CoordinationTest, SnapshotableHashMapSimple)
TEST_P(CoordinationTest, SnapshotableHashMapSimple)
{
DB::SnapshotableHashTable<int> hello;
EXPECT_TRUE(hello.insert("hello", 5));
@ -821,7 +846,7 @@ TEST(CoordinationTest, SnapshotableHashMapSimple)
EXPECT_EQ(hello.size(), 0);
}
TEST(CoordinationTest, SnapshotableHashMapTrySnapshot)
TEST_P(CoordinationTest, SnapshotableHashMapTrySnapshot)
{
DB::SnapshotableHashTable<int> map_snp;
EXPECT_TRUE(map_snp.insert("/hello", 7));
@ -907,7 +932,7 @@ void addNode(DB::KeeperStorage & storage, const std::string & path, const std::s
storage.container.insertOrReplace(path, node);
}
TEST(CoordinationTest, TestStorageSnapshotSimple)
TEST_P(CoordinationTest, TestStorageSnapshotSimple)
{
ChangelogDirTest test("./snapshots");
DB::KeeperSnapshotManager manager("./snapshots", 3);
@ -954,7 +979,7 @@ TEST(CoordinationTest, TestStorageSnapshotSimple)
EXPECT_EQ(restored_storage->session_and_timeout.size(), 2);
}
TEST(CoordinationTest, TestStorageSnapshotMoreWrites)
TEST_P(CoordinationTest, TestStorageSnapshotMoreWrites)
{
ChangelogDirTest test("./snapshots");
DB::KeeperSnapshotManager manager("./snapshots", 3);
@ -994,7 +1019,7 @@ TEST(CoordinationTest, TestStorageSnapshotMoreWrites)
}
TEST(CoordinationTest, TestStorageSnapshotManySnapshots)
TEST_P(CoordinationTest, TestStorageSnapshotManySnapshots)
{
ChangelogDirTest test("./snapshots");
DB::KeeperSnapshotManager manager("./snapshots", 3);
@ -1032,7 +1057,7 @@ TEST(CoordinationTest, TestStorageSnapshotManySnapshots)
}
}
TEST(CoordinationTest, TestStorageSnapshotMode)
TEST_P(CoordinationTest, TestStorageSnapshotMode)
{
ChangelogDirTest test("./snapshots");
DB::KeeperSnapshotManager manager("./snapshots", 3);
@ -1083,7 +1108,7 @@ TEST(CoordinationTest, TestStorageSnapshotMode)
}
TEST(CoordinationTest, TestStorageSnapshotBroken)
TEST_P(CoordinationTest, TestStorageSnapshotBroken)
{
ChangelogDirTest test("./snapshots");
DB::KeeperSnapshotManager manager("./snapshots", 3);
@ -1121,7 +1146,7 @@ nuraft::ptr<nuraft::log_entry> getLogEntryFromZKRequest(size_t term, int64_t ses
return nuraft::cs_new<nuraft::log_entry>(term, buffer);
}
void testLogAndStateMachine(Coordination::CoordinationSettingsPtr settings, uint64_t total_logs)
void testLogAndStateMachine(Coordination::CoordinationSettingsPtr settings, uint64_t total_logs, bool enable_compression)
{
using namespace Coordination;
using namespace DB;
@ -1133,7 +1158,7 @@ void testLogAndStateMachine(Coordination::CoordinationSettingsPtr settings, uint
SnapshotsQueue snapshots_queue{1};
auto state_machine = std::make_shared<KeeperStateMachine>(queue, snapshots_queue, "./snapshots", settings);
state_machine->init();
DB::KeeperLogStore changelog("./logs", settings->rotate_log_storage_interval, true);
DB::KeeperLogStore changelog("./logs", settings->rotate_log_storage_interval, true, enable_compression);
changelog.init(state_machine->last_commit_index() + 1, settings->reserved_log_items);
for (size_t i = 1; i < total_logs + 1; ++i)
{
@ -1173,7 +1198,7 @@ void testLogAndStateMachine(Coordination::CoordinationSettingsPtr settings, uint
restore_machine->init();
EXPECT_EQ(restore_machine->last_commit_index(), total_logs - total_logs % settings->snapshot_distance);
DB::KeeperLogStore restore_changelog("./logs", settings->rotate_log_storage_interval, true);
DB::KeeperLogStore restore_changelog("./logs", settings->rotate_log_storage_interval, true, enable_compression);
restore_changelog.init(restore_machine->last_commit_index() + 1, settings->reserved_log_items);
EXPECT_EQ(restore_changelog.size(), std::min(settings->reserved_log_items + total_logs % settings->snapshot_distance, total_logs));
@ -1199,77 +1224,78 @@ void testLogAndStateMachine(Coordination::CoordinationSettingsPtr settings, uint
}
}
TEST(CoordinationTest, TestStateMachineAndLogStore)
TEST_P(CoordinationTest, TestStateMachineAndLogStore)
{
using namespace Coordination;
using namespace DB;
auto params = GetParam();
{
CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
settings->snapshot_distance = 10;
settings->reserved_log_items = 10;
settings->rotate_log_storage_interval = 10;
testLogAndStateMachine(settings, 37);
testLogAndStateMachine(settings, 37, params.enable_compression);
}
{
CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
settings->snapshot_distance = 10;
settings->reserved_log_items = 10;
settings->rotate_log_storage_interval = 10;
testLogAndStateMachine(settings, 11);
testLogAndStateMachine(settings, 11, params.enable_compression);
}
{
CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
settings->snapshot_distance = 10;
settings->reserved_log_items = 10;
settings->rotate_log_storage_interval = 10;
testLogAndStateMachine(settings, 40);
testLogAndStateMachine(settings, 40, params.enable_compression);
}
{
CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
settings->snapshot_distance = 10;
settings->reserved_log_items = 20;
settings->rotate_log_storage_interval = 30;
testLogAndStateMachine(settings, 40);
testLogAndStateMachine(settings, 40, params.enable_compression);
}
{
CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
settings->snapshot_distance = 10;
settings->reserved_log_items = 0;
settings->rotate_log_storage_interval = 10;
testLogAndStateMachine(settings, 40);
testLogAndStateMachine(settings, 40, params.enable_compression);
}
{
CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
settings->snapshot_distance = 1;
settings->reserved_log_items = 1;
settings->rotate_log_storage_interval = 32;
testLogAndStateMachine(settings, 32);
testLogAndStateMachine(settings, 32, params.enable_compression);
}
{
CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
settings->snapshot_distance = 10;
settings->reserved_log_items = 7;
settings->rotate_log_storage_interval = 1;
testLogAndStateMachine(settings, 33);
testLogAndStateMachine(settings, 33, params.enable_compression);
}
{
CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
settings->snapshot_distance = 37;
settings->reserved_log_items = 1000;
settings->rotate_log_storage_interval = 5000;
testLogAndStateMachine(settings, 33);
testLogAndStateMachine(settings, 33, params.enable_compression);
}
{
CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
settings->snapshot_distance = 37;
settings->reserved_log_items = 1000;
settings->rotate_log_storage_interval = 5000;
testLogAndStateMachine(settings, 45);
testLogAndStateMachine(settings, 45, params.enable_compression);
}
}
TEST(CoordinationTest, TestEphemeralNodeRemove)
TEST_P(CoordinationTest, TestEphemeralNodeRemove)
{
using namespace Coordination;
using namespace DB;
@ -1300,12 +1326,13 @@ TEST(CoordinationTest, TestEphemeralNodeRemove)
}
TEST(CoordinationTest, TestRotateIntervalChanges)
TEST_P(CoordinationTest, TestRotateIntervalChanges)
{
using namespace Coordination;
auto params = GetParam();
ChangelogDirTest snapshots("./logs");
{
DB::KeeperLogStore changelog("./logs", 100, true);
DB::KeeperLogStore changelog("./logs", 100, true, params.enable_compression);
changelog.init(0, 3);
for (size_t i = 1; i < 55; ++i)
@ -1319,9 +1346,9 @@ TEST(CoordinationTest, TestRotateIntervalChanges)
}
EXPECT_TRUE(fs::exists("./logs/changelog_1_100.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_100.bin" + params.extension));
DB::KeeperLogStore changelog_1("./logs", 10, true);
DB::KeeperLogStore changelog_1("./logs", 10, true, params.enable_compression);
changelog_1.init(0, 50);
for (size_t i = 0; i < 55; ++i)
{
@ -1332,10 +1359,10 @@ TEST(CoordinationTest, TestRotateIntervalChanges)
changelog_1.end_of_append_batch(0, 0);
}
EXPECT_TRUE(fs::exists("./logs/changelog_1_100.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_101_110.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_1_100.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_101_110.bin" + params.extension));
DB::KeeperLogStore changelog_2("./logs", 7, true);
DB::KeeperLogStore changelog_2("./logs", 7, true, params.enable_compression);
changelog_2.init(98, 55);
for (size_t i = 0; i < 17; ++i)
@ -1349,13 +1376,13 @@ TEST(CoordinationTest, TestRotateIntervalChanges)
changelog_2.compact(105);
EXPECT_FALSE(fs::exists("./logs/changelog_1_100.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_101_110.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_111_117.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_118_124.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_125_131.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_1_100.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_101_110.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_111_117.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_118_124.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_125_131.bin" + params.extension));
DB::KeeperLogStore changelog_3("./logs", 5, true);
DB::KeeperLogStore changelog_3("./logs", 5, true, params.enable_compression);
changelog_3.init(116, 3);
for (size_t i = 0; i < 17; ++i)
{
@ -1367,17 +1394,17 @@ TEST(CoordinationTest, TestRotateIntervalChanges)
}
changelog_3.compact(125);
EXPECT_FALSE(fs::exists("./logs/changelog_101_110.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_111_117.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_118_124.bin"));
EXPECT_FALSE(fs::exists("./logs/changelog_101_110.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_111_117.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_118_124.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_125_131.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_132_136.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_137_141.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_142_146.bin"));
EXPECT_TRUE(fs::exists("./logs/changelog_125_131.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_132_136.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_137_141.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_142_146.bin" + params.extension));
}
TEST(CoordinationTest, TestSessionExpiryQueue)
TEST_P(CoordinationTest, TestSessionExpiryQueue)
{
using namespace Coordination;
SessionExpiryQueue queue(500);
@ -1395,6 +1422,56 @@ TEST(CoordinationTest, TestSessionExpiryQueue)
}
TEST_P(CoordinationTest, TestCompressedLogsMultipleRewrite)
{
using namespace Coordination;
auto test_params = GetParam();
ChangelogDirTest snapshots("./logs");
DB::KeeperLogStore changelog("./logs", 100, true, test_params.enable_compression);
changelog.init(0, 3);
for (size_t i = 1; i < 55; ++i)
{
std::shared_ptr<ZooKeeperCreateRequest> request = std::make_shared<ZooKeeperCreateRequest>();
request->path = "/hello_" + std::to_string(i);
auto entry = getLogEntryFromZKRequest(0, 1, request);
changelog.append(entry);
changelog.end_of_append_batch(0, 0);
}
DB::KeeperLogStore changelog1("./logs", 100, true, test_params.enable_compression);
changelog1.init(0, 3);
for (size_t i = 55; i < 70; ++i)
{
std::shared_ptr<ZooKeeperCreateRequest> request = std::make_shared<ZooKeeperCreateRequest>();
request->path = "/hello_" + std::to_string(i);
auto entry = getLogEntryFromZKRequest(0, 1, request);
changelog1.append(entry);
changelog1.end_of_append_batch(0, 0);
}
DB::KeeperLogStore changelog2("./logs", 100, true, test_params.enable_compression);
changelog2.init(0, 3);
for (size_t i = 70; i < 80; ++i)
{
std::shared_ptr<ZooKeeperCreateRequest> request = std::make_shared<ZooKeeperCreateRequest>();
request->path = "/hello_" + std::to_string(i);
auto entry = getLogEntryFromZKRequest(0, 1, request);
changelog2.append(entry);
changelog2.end_of_append_batch(0, 0);
}
}
INSTANTIATE_TEST_SUITE_P(CoordinationTestSuite,
CoordinationTest,
::testing::ValuesIn(std::initializer_list<CompressionParam>{
CompressionParam{true, ".zstd"},
CompressionParam{false, ""}
})
);
int main(int argc, char ** argv)
{
Poco::AutoPtr<Poco::ConsoleChannel> channel(new Poco::ConsoleChannel(std::cerr));

13
src/Coordination/ya.make
View File

@ -1,13 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
SRCS(
)
END()

12
src/Coordination/ya.make.in
View File

@ -1,12 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
SRCS(
)
END()

51
src/Core/ya.make
View File

@ -1,51 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
contrib/libs/sparsehash
contrib/restricted/boost/libs
)
SRCS(
BackgroundSchedulePool.cpp
BaseSettings.cpp
Block.cpp
BlockInfo.cpp
ColumnWithTypeAndName.cpp
ExternalResultDescription.cpp
ExternalTable.cpp
Field.cpp
MySQL/Authentication.cpp
MySQL/IMySQLReadPacket.cpp
MySQL/IMySQLWritePacket.cpp
MySQL/MySQLClient.cpp
MySQL/MySQLGtid.cpp
MySQL/MySQLReplication.cpp
MySQL/PacketEndpoint.cpp
MySQL/PacketsConnection.cpp
MySQL/PacketsGeneric.cpp
MySQL/PacketsProtocolText.cpp
MySQL/PacketsReplication.cpp
NamesAndTypes.cpp
PostgreSQL/Connection.cpp
PostgreSQL/PoolWithFailover.cpp
PostgreSQL/Utils.cpp
PostgreSQL/insertPostgreSQLValue.cpp
PostgreSQLProtocol.cpp
QueryProcessingStage.cpp
ServerUUID.cpp
Settings.cpp
SettingsEnums.cpp
SettingsFields.cpp
SettingsQuirks.cpp
SortDescription.cpp
UUID.cpp
iostream_debug_helpers.cpp
)
END()

16
src/Core/ya.make.in
View File

@ -1,16 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
contrib/libs/sparsehash
contrib/restricted/boost/libs
)
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | sed 's/^\.\// /' | sort ?>
)
END()

6
src/DataStreams/RemoteQueryExecutor.cpp
View File

@ -208,6 +208,12 @@ void RemoteQueryExecutor::sendQuery()
auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(settings);
ClientInfo modified_client_info = context->getClientInfo();
modified_client_info.query_kind = ClientInfo::QueryKind::SECONDARY_QUERY;
/// Set initial_query_id to query_id for the clickhouse-benchmark.
///
/// (since first query of clickhouse-benchmark will be issued as SECONDARY_QUERY,
/// due to it executes queries via RemoteBlockInputStream)
if (modified_client_info.initial_query_id.empty())
modified_client_info.initial_query_id = query_id;
if (CurrentThread::isInitialized())
{
modified_client_info.client_trace_context = CurrentThread::get().thread_trace_context;

59
src/DataStreams/ya.make
View File

@ -1,59 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
contrib/libs/poco/MongoDB
contrib/restricted/boost/libs
)
NO_COMPILER_WARNINGS()
SRCS(
AddingDefaultBlockOutputStream.cpp
AsynchronousBlockInputStream.cpp
BlockIO.cpp
BlockStreamProfileInfo.cpp
CheckConstraintsBlockOutputStream.cpp
ColumnGathererStream.cpp
ConnectionCollector.cpp
ConvertingBlockInputStream.cpp
CountingBlockOutputStream.cpp
DistinctSortedBlockInputStream.cpp
ExecutionSpeedLimits.cpp
ExpressionBlockInputStream.cpp
IBlockInputStream.cpp
ITTLAlgorithm.cpp
InternalTextLogsRowOutputStream.cpp
MaterializingBlockInputStream.cpp
MongoDBSource.cpp
NativeBlockInputStream.cpp
NativeBlockOutputStream.cpp
PushingToViewsBlockOutputStream.cpp
RemoteBlockInputStream.cpp
RemoteBlockOutputStream.cpp
RemoteQueryExecutor.cpp
RemoteQueryExecutorReadContext.cpp
SQLiteSource.cpp
SizeLimits.cpp
SquashingBlockInputStream.cpp
SquashingBlockOutputStream.cpp
SquashingTransform.cpp
TTLAggregationAlgorithm.cpp
TTLBlockInputStream.cpp
TTLCalcInputStream.cpp
TTLColumnAlgorithm.cpp
TTLDeleteAlgorithm.cpp
TTLUpdateInfoAlgorithm.cpp
copyData.cpp
finalizeBlock.cpp
formatBlock.cpp
materializeBlock.cpp
narrowBlockInputStreams.cpp
)
END()

18
src/DataStreams/ya.make.in
View File

@ -1,18 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
contrib/libs/poco/MongoDB
contrib/restricted/boost/libs
)
NO_COMPILER_WARNINGS()
SRCS(
<? find . -name '*.cpp' | grep -v -P 'tests|PostgreSQL' | grep -v -F examples | sed 's/^\.\// /' | sort ?>
)
END()

74
src/DataTypes/ya.make
View File

@ -1,74 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
clickhouse/src/Formats
)
SRCS(
DataTypeAggregateFunction.cpp
DataTypeArray.cpp
DataTypeCustomGeo.cpp
DataTypeCustomIPv4AndIPv6.cpp
DataTypeCustomSimpleAggregateFunction.cpp
DataTypeDate.cpp
DataTypeDate32.cpp
DataTypeDateTime.cpp
DataTypeDateTime64.cpp
DataTypeDecimalBase.cpp
DataTypeEnum.cpp
DataTypeFactory.cpp
DataTypeFixedString.cpp
DataTypeFunction.cpp
DataTypeInterval.cpp
DataTypeLowCardinality.cpp
DataTypeLowCardinalityHelpers.cpp
DataTypeMap.cpp
DataTypeNested.cpp
DataTypeNothing.cpp
DataTypeNullable.cpp
DataTypeNumberBase.cpp
DataTypeString.cpp
DataTypeTuple.cpp
DataTypeUUID.cpp
DataTypesDecimal.cpp
DataTypesNumber.cpp
EnumValues.cpp
FieldToDataType.cpp
IDataType.cpp
NestedUtils.cpp
Serializations/ISerialization.cpp
Serializations/SerializationAggregateFunction.cpp
Serializations/SerializationArray.cpp
Serializations/SerializationCustomSimpleText.cpp
Serializations/SerializationDate.cpp
Serializations/SerializationDate32.cpp
Serializations/SerializationDateTime.cpp
Serializations/SerializationDateTime64.cpp
Serializations/SerializationDecimal.cpp
Serializations/SerializationDecimalBase.cpp
Serializations/SerializationEnum.cpp
Serializations/SerializationFixedString.cpp
Serializations/SerializationIP.cpp
Serializations/SerializationLowCardinality.cpp
Serializations/SerializationMap.cpp
Serializations/SerializationNothing.cpp
Serializations/SerializationNullable.cpp
Serializations/SerializationNumber.cpp
Serializations/SerializationString.cpp
Serializations/SerializationTuple.cpp
Serializations/SerializationTupleElement.cpp
Serializations/SerializationUUID.cpp
Serializations/SerializationWrapper.cpp
convertMySQLDataType.cpp
getLeastSupertype.cpp
getMostSubtype.cpp
registerDataTypeDateTime.cpp
)
END()

15
src/DataTypes/ya.make.in
View File

@ -1,15 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
clickhouse/src/Formats
)
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | sed 's/^\.\// /' | sort ?>
)
END()

38
src/Databases/ya.make
View File

@ -1,38 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
SRCS(
DDLDependencyVisitor.cpp
DatabaseAtomic.cpp
DatabaseDictionary.cpp
DatabaseFactory.cpp
DatabaseLazy.cpp
DatabaseMemory.cpp
DatabaseOnDisk.cpp
DatabaseOrdinary.cpp
DatabaseReplicated.cpp
DatabaseReplicatedSettings.cpp
DatabaseReplicatedWorker.cpp
DatabasesCommon.cpp
MySQL/ConnectionMySQLSettings.cpp
MySQL/DatabaseMaterializedMySQL.cpp
MySQL/DatabaseMySQL.cpp
MySQL/FetchTablesColumnsList.cpp
MySQL/MaterializeMetadata.cpp
MySQL/MaterializedMySQLSettings.cpp
MySQL/MaterializedMySQLSyncThread.cpp
SQLite/DatabaseSQLite.cpp
SQLite/SQLiteUtils.cpp
SQLite/fetchSQLiteTableStructure.cpp
TablesLoader.cpp
)
END()

14
src/Databases/ya.make.in
View File

@ -1,14 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
SRCS(
<? find . -name '*.cpp' | grep -v -F examples | grep -v -F 'PostgreSQL' | sed 's/^\.\// /' | sort ?>
)
END()

11
src/Dictionaries/IDictionary.h
View File

@ -1,22 +1,17 @@
#pragma once
#include <Core/Names.h>
#include <DataStreams/IBlockStream_fwd.h>
#include <Interpreters/IExternalLoadable.h>
#include <Interpreters/StorageID.h>
#include <Poco/Util/XMLConfiguration.h>
#include <Common/PODArray.h>
#include <common/StringRef.h>
#include "IDictionarySource.h"
#include <Columns/ColumnsNumber.h>
#include <Dictionaries/IDictionarySource.h>
#include <Dictionaries/DictionaryStructure.h>
#include <DataTypes/IDataType.h>
#include <Columns/ColumnsNumber.h>
#include <chrono>
#include <memory>
#include <mutex>
namespace DB
{
namespace ErrorCodes

16
src/Dictionaries/MongoDBDictionarySource.cpp
View File

@ -20,7 +20,21 @@ void registerDictionarySourceMongoDB(DictionarySourceFactory & factory)
bool /* created_from_ddl */)
{
const auto config_prefix = root_config_prefix + ".mongodb";
auto configuration = getExternalDataSourceConfiguration(config, config_prefix, context);
ExternalDataSourceConfiguration configuration;
auto named_collection = getExternalDataSourceConfiguration(config, config_prefix, context);
if (named_collection)
{
configuration = *named_collection;
}
else
{
configuration.host = config.getString(config_prefix + ".host", "");
configuration.port = config.getUInt(config_prefix + ".port", 0);
configuration.username = config.getString(config_prefix + ".user", "");
configuration.password = config.getString(config_prefix + ".password", "");
configuration.database = config.getString(config_prefix + ".db", "");
}
return std::make_unique<MongoDBDictionarySource>(dict_struct,
config.getString(config_prefix + ".uri", ""),
configuration.host,

26
src/Dictionaries/MySQLDictionarySource.cpp
View File

@ -44,7 +44,22 @@ void registerDictionarySourceMysql(DictionarySourceFactory & factory)
config.getBool(config_prefix + ".mysql.fail_on_connection_loss", false) ? 1 : default_num_tries_on_connection_loss);
auto settings_config_prefix = config_prefix + ".mysql";
auto configuration = getExternalDataSourceConfiguration(config, settings_config_prefix, global_context);
std::shared_ptr<mysqlxx::PoolWithFailover> pool;
ExternalDataSourceConfiguration configuration;
auto named_collection = created_from_ddl ? getExternalDataSourceConfiguration(config, settings_config_prefix, global_context) : std::nullopt;
if (named_collection)
{
configuration = *named_collection;
std::vector<std::pair<String, UInt16>> addresses{std::make_pair(configuration.host, configuration.port)};
pool = std::make_shared<mysqlxx::PoolWithFailover>(configuration.database, addresses, configuration.username, configuration.password);
}
else
{
configuration.database = config.getString(settings_config_prefix + ".db", "");
configuration.table = config.getString(settings_config_prefix + ".table", "");
pool = std::make_shared<mysqlxx::PoolWithFailover>(mysqlxx::PoolFactory::instance().get(config, settings_config_prefix));
}
auto query = config.getString(settings_config_prefix + ".query", "");
if (query.empty() && configuration.table.empty())
throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "MySQL dictionary source configuration must contain table or query field");
@ -61,15 +76,6 @@ void registerDictionarySourceMysql(DictionarySourceFactory & factory)
.dont_check_update_time = config.getBool(settings_config_prefix + ".dont_check_update_time", false)
};
std::shared_ptr<mysqlxx::PoolWithFailover> pool;
if (created_from_ddl)
{
std::vector<std::pair<String, UInt16>> addresses{std::make_pair(configuration.host, configuration.port)};
pool = std::make_shared<mysqlxx::PoolWithFailover>(configuration.database, addresses, configuration.username, configuration.password);
}
else
pool = std::make_shared<mysqlxx::PoolWithFailover>(mysqlxx::PoolFactory::instance().get(config, settings_config_prefix));
return std::make_unique<MySQLDictionarySource>(dict_struct, dictionary_configuration, std::move(pool), sample_block, mysql_input_stream_settings);
#else
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,

26
src/Dictionaries/getDictionaryConfigurationFromAST.cpp
View File

@ -28,12 +28,14 @@ namespace ErrorCodes
extern const int INCORRECT_DICTIONARY_DEFINITION;
}
/// There are a lot of code, but it's very simple and straightforward
/// We just convert
/// We just perform conversion.
namespace
{
using NamesToTypeNames = std::unordered_map<std::string, std::string>;
/// Get value from field and convert it to string.
/// Also remove quotes from strings.
String getFieldAsString(const Field & field)
@ -46,8 +48,8 @@ String getFieldAsString(const Field & field)
using namespace Poco;
using namespace Poco::XML;
/*
* Transforms next definition
/* Transforms next definition
* LIFETIME(MIN 10, MAX 100)
* to the next configuration
* <lifetime>
@ -76,8 +78,7 @@ void buildLifetimeConfiguration(
}
}
/*
* Transforms next definition
/* Transforms next definition
* LAYOUT(FLAT())
* to the next configuration
* <layout>
@ -104,6 +105,7 @@ void buildLayoutConfiguration(
layout_element->appendChild(layout_type_element);
if (layout->parameters)
{
for (const auto & param : layout->parameters->children)
{
const ASTPair * pair = param->as<ASTPair>();
@ -136,10 +138,11 @@ void buildLayoutConfiguration(
layout_type_parameter_element->appendChild(value_to_append);
layout_type_element->appendChild(layout_type_parameter_element);
}
}
}
/*
* Transforms next definition
/* Transforms next definition
* RANGE(MIN StartDate, MAX EndDate)
* to the next configuration
* <range_min><name>StartDate</name></range_min>
@ -216,8 +219,7 @@ void buildAttributeExpressionIfNeeded(
}
}
/**
* Transofrms single dictionary attribute to configuration
/** Transofrms single dictionary attribute to configuration
* third_column UInt8 DEFAULT 2 EXPRESSION rand() % 100 * 77
* to
* <attribute>
@ -281,8 +283,7 @@ void buildSingleAttribute(
}
/**
* Transforms
/** Transforms
* PRIMARY KEY Attr1 ,..., AttrN
* to the next configuration
* <id><name>Attr1</name></id>
@ -369,8 +370,7 @@ void buildPrimaryKeyConfiguration(
}
/**
* Transforms list of ASTDictionaryAttributeDeclarations to list of dictionary attributes
/** Transforms list of ASTDictionaryAttributeDeclarations to list of dictionary attributes
*/
NamesToTypeNames buildDictionaryAttributesConfiguration(
AutoPtr<Document> doc,

72
src/Dictionaries/ya.make
View File

@ -1,72 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
contrib/libs/poco/Data
contrib/libs/poco/MongoDB
contrib/libs/poco/Redis
contrib/libs/sparsehash
contrib/restricted/abseil-cpp
)
IF (USE_ODBC)
PEERDIR(contrib/libs/poco/Data/ODBC)
ENDIF ()
NO_COMPILER_WARNINGS()
SRCS(
CacheDictionary.cpp
CacheDictionaryUpdateQueue.cpp
CassandraDictionarySource.cpp
CassandraHelpers.cpp
CassandraSource.cpp
ClickHouseDictionarySource.cpp
DictionaryFactory.cpp
DictionaryHelpers.cpp
DictionarySource.cpp
DictionarySourceBase.cpp
DictionarySourceFactory.cpp
DictionarySourceHelpers.cpp
DictionaryStructure.cpp
DirectDictionary.cpp
Embedded/GeoDictionariesLoader.cpp
Embedded/GeodataProviders/HierarchiesProvider.cpp
Embedded/GeodataProviders/HierarchyFormatReader.cpp
Embedded/GeodataProviders/NamesFormatReader.cpp
Embedded/GeodataProviders/NamesProvider.cpp
Embedded/RegionsHierarchies.cpp
Embedded/RegionsHierarchy.cpp
Embedded/RegionsNames.cpp
ExecutableDictionarySource.cpp
ExecutablePoolDictionarySource.cpp
ExternalQueryBuilder.cpp
FileDictionarySource.cpp
FlatDictionary.cpp
HTTPDictionarySource.cpp
HashedDictionary.cpp
HierarchyDictionariesUtils.cpp
IPAddressDictionary.cpp
LibraryDictionarySource.cpp
MongoDBDictionarySource.cpp
MySQLDictionarySource.cpp
PolygonDictionary.cpp
PolygonDictionaryImplementations.cpp
PolygonDictionaryUtils.cpp
RangeHashedDictionary.cpp
RedisDictionarySource.cpp
RedisSource.cpp
XDBCDictionarySource.cpp
getDictionaryConfigurationFromAST.cpp
readInvalidateQuery.cpp
registerCacheDictionaries.cpp
registerDictionaries.cpp
writeParenthesisedString.cpp
)
END()

25
src/Dictionaries/ya.make.in
View File

@ -1,25 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
contrib/libs/poco/Data
contrib/libs/poco/MongoDB
contrib/libs/poco/Redis
contrib/libs/sparsehash
contrib/restricted/abseil-cpp
)
IF (USE_ODBC)
PEERDIR(contrib/libs/poco/Data/ODBC)
ENDIF ()
NO_COMPILER_WARNINGS()
SRCS(
<? find . -name '*.cpp' | grep -v -P 'tests|PostgreSQL' | grep -v -F examples | sed 's/^\.\// /' | sort ?>
)
END()

2
src/Disks/DiskWebServer.cpp
View File

@ -40,7 +40,7 @@ void DiskWebServer::initialize(const String & uri_path) const
ReadWriteBufferFromHTTP::OutStreamCallback(),
ConnectionTimeouts::getHTTPTimeouts(getContext()));
String file_name;
FileData file_data;
FileData file_data{};
String dir_name = fs::path(uri_path.substr(url.size())) / "";
LOG_TRACE(&Poco::Logger::get("DiskWeb"), "Adding directory: {}", dir_name);

4
src/Disks/DiskWebServer.h
View File

@ -185,8 +185,8 @@ private:
struct FileData
{
FileType type;
size_t size;
FileType type{};
size_t size = 0;
};
using Files = std::unordered_map<String, FileData>; /// file path -> file data

17
src/Disks/S3/ya.make
View File

@ -1,17 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
SRCS(
DiskS3.cpp
registerDiskS3.cpp
ProxyListConfiguration.cpp
ProxyResolverConfiguration.cpp
)
END()

37
src/Disks/ya.make
View File

@ -1,37 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
SRCS(
DiskCacheWrapper.cpp
DiskDecorator.cpp
DiskEncrypted.cpp
DiskFactory.cpp
DiskLocal.cpp
DiskMemory.cpp
DiskRestartProxy.cpp
DiskSelector.cpp
DiskWebServer.cpp
IDisk.cpp
IDiskRemote.cpp
IVolume.cpp
LocalDirectorySyncGuard.cpp
ReadIndirectBufferFromRemoteFS.cpp
ReadIndirectBufferFromWebServer.cpp
SingleDiskVolume.cpp
StoragePolicy.cpp
TemporaryFileOnDisk.cpp
VolumeJBOD.cpp
VolumeRAID1.cpp
WriteIndirectBufferFromRemoteFS.cpp
createVolume.cpp
registerDisks.cpp
)
END()

13
src/Disks/ya.make.in
View File

@ -1,13 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | grep -v -P 'S3|HDFS' | sed 's/^\.\// /' | sort ?>
)
END()

30
src/Formats/ya.make
View File

@ -1,30 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
contrib/libs/protobuf
contrib/libs/protoc
)
SRCS(
FormatFactory.cpp
FormatSchemaInfo.cpp
JSONEachRowUtils.cpp
MySQLSource.cpp
NativeFormat.cpp
NullFormat.cpp
ParsedTemplateFormatString.cpp
ProtobufReader.cpp
ProtobufSchemas.cpp
ProtobufSerializer.cpp
ProtobufWriter.cpp
registerFormats.cpp
verbosePrintString.cpp
)
END()

16
src/Formats/ya.make.in
View File

@ -1,16 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
contrib/libs/protobuf
contrib/libs/protoc
)
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | sed 's/^\.\// /' | sort ?>
)
END()

19
src/Functions/h3GetFaces.cpp
View File

@ -56,10 +56,12 @@ public:
const auto * column = checkAndGetColumn<ColumnUInt64>(arguments[0].column.get());
const auto & data = column->getData();
auto dst = ColumnArray::create(ColumnUInt8::create());
auto & dst_data = reinterpret_cast<ColumnUInt8 &>(dst->getData());
auto & dst_offsets = dst->getOffsets();
dst_offsets.resize(input_rows_count);
auto result_column_data = ColumnUInt8::create();
auto & result_data = result_column_data->getData();
auto result_column_offsets = ColumnArray::ColumnOffsets::create();
auto & result_offsets = result_column_offsets->getData();
result_offsets.resize(input_rows_count);
auto current_offset = 0;
std::vector<int> faces;
@ -69,6 +71,7 @@ public:
int max_faces = maxFaceCount(data[row]);
faces.resize(max_faces);
// function name h3GetFaces (v3.x) changed to getIcosahedronFaces (v4.0.0).
getIcosahedronFaces(data[row], faces.data());
@ -78,13 +81,15 @@ public:
if (faces[i] >= 0 && faces[i] <= 19)
{
++current_offset;
dst_data.insert(faces[i]);
result_data.emplace_back(faces[i]);
}
}
dst_offsets[row] = current_offset;
result_offsets[row] = current_offset;
faces.clear();
}
return dst;
return ColumnArray::create(std::move(result_column_data), std::move(result_column_offsets));
}
};

10
src/Functions/h3ResIsClassIII.cpp → src/Functions/h3IsResClassIII.cpp
View File

@ -24,12 +24,12 @@ extern const int ILLEGAL_TYPE_OF_ARGUMENT;
namespace
{
class FunctionH3ResIsClassIII : public IFunction
class FunctionH3IsResClassIII : public IFunction
{
public:
static constexpr auto name = "h3ResIsClassIII";
static constexpr auto name = "h3IsResClassIII";
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionH3ResIsClassIII>(); }
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionH3IsResClassIII>(); }
std::string getName() const override { return name; }
@ -69,9 +69,9 @@ public:
}
void registerFunctionH3ResIsClassIII(FunctionFactory & factory)
void registerFunctionH3IsResClassIII(FunctionFactory & factory)
{
factory.registerFunction<FunctionH3ResIsClassIII>();
factory.registerFunction<FunctionH3IsResClassIII>();
}
}

4
src/Functions/registerFunctionsGeo.cpp
View File

@ -42,7 +42,7 @@ void registerFunctionH3IndexesAreNeighbors(FunctionFactory &);
void registerFunctionStringToH3(FunctionFactory &);
void registerFunctionH3ToString(FunctionFactory &);
void registerFunctionH3HexAreaM2(FunctionFactory &);
void registerFunctionH3ResIsClassIII(FunctionFactory &);
void registerFunctionH3IsResClassIII(FunctionFactory &);
void registerFunctionH3IsPentagon(FunctionFactory &);
void registerFunctionH3GetFaces(FunctionFactory &);
#endif
@ -98,7 +98,7 @@ void registerFunctionsGeo(FunctionFactory & factory)
registerFunctionStringToH3(factory);
registerFunctionH3ToString(factory);
registerFunctionH3HexAreaM2(factory);
registerFunctionH3ResIsClassIII(factory);
registerFunctionH3IsResClassIII(factory);
registerFunctionH3IsPentagon(factory);
registerFunctionH3GetFaces(factory);
#endif

591
src/Functions/ya.make
View File

@ -1,591 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
CFLAGS(
-DUSE_SSL -DUSE_XXHASH
)
ADDINCL(
library/cpp/consistent_hashing
contrib/libs/farmhash
contrib/libs/h3/h3lib/include
contrib/libs/hyperscan/src
contrib/libs/libdivide
contrib/libs/rapidjson/include
contrib/libs/xxhash
GLOBAL contrib/restricted/murmurhash
)
PEERDIR(
clickhouse/src/Common
clickhouse/src/Parsers
clickhouse/src/Dictionaries
contrib/libs/farmhash
contrib/libs/fastops/fastops
contrib/libs/h3
contrib/libs/hyperscan
contrib/libs/icu
contrib/libs/libdivide
contrib/libs/metrohash
contrib/libs/rapidjson
contrib/libs/xxhash
contrib/restricted/murmurhash
library/cpp/consistent_hashing
)
# "Arcadia" build is slightly deficient. It lacks many libraries that we need.
SRCS(
CRC.cpp
CastOverloadResolver.cpp
DateOrDateTimeFunctionsConvertion.cpp
FunctionChar.cpp
FunctionFQDN.cpp
FunctionFactory.cpp
FunctionFile.cpp
FunctionHelpers.cpp
FunctionJoinGet.cpp
FunctionSQLJSON.cpp
FunctionsAES.cpp
FunctionsBinaryRepr.cpp
FunctionsBitToArray.cpp
FunctionsCodingIP.cpp
FunctionsCodingUUID.cpp
FunctionsConversion.cpp
FunctionsEmbeddedDictionaries.cpp
FunctionsExternalDictionaries.cpp
FunctionsExternalModels.cpp
FunctionsHashing.cpp
FunctionsJSON.cpp
FunctionsLogical.cpp
FunctionsRandom.cpp
FunctionsRound.cpp
FunctionsStringArray.cpp
FunctionsStringHash.cpp
FunctionsStringSimilarity.cpp
GatherUtils/concat.cpp
GatherUtils/createArraySink.cpp
GatherUtils/createArraySource.cpp
GatherUtils/createValueSource.cpp
GatherUtils/has_all.cpp
GatherUtils/has_any.cpp
GatherUtils/has_substr.cpp
GatherUtils/push.cpp
GatherUtils/resizeConstantSize.cpp
GatherUtils/resizeDynamicSize.cpp
GatherUtils/sliceDynamicOffsetBounded.cpp
GatherUtils/sliceDynamicOffsetUnbounded.cpp
GatherUtils/sliceFromLeftConstantOffsetBounded.cpp
GatherUtils/sliceFromLeftConstantOffsetUnbounded.cpp
GatherUtils/sliceFromRightConstantOffsetBounded.cpp
GatherUtils/sliceFromRightConstantOffsetUnbounded.cpp
GeoHash.cpp
IFunction.cpp
JSONPath/Parsers/ParserJSONPath.cpp
JSONPath/Parsers/ParserJSONPathMemberAccess.cpp
JSONPath/Parsers/ParserJSONPathQuery.cpp
JSONPath/Parsers/ParserJSONPathRange.cpp
JSONPath/Parsers/ParserJSONPathRoot.cpp
JSONPath/Parsers/ParserJSONPathStar.cpp
TargetSpecific.cpp
URL/URLHierarchy.cpp
URL/URLPathHierarchy.cpp
URL/basename.cpp
URL/cutFragment.cpp
URL/cutQueryString.cpp
URL/cutQueryStringAndFragment.cpp
URL/cutToFirstSignificantSubdomain.cpp
URL/cutToFirstSignificantSubdomainCustom.cpp
URL/cutURLParameter.cpp
URL/cutWWW.cpp
URL/decodeURLComponent.cpp
URL/domain.cpp
URL/domainWithoutWWW.cpp
URL/extractURLParameter.cpp
URL/extractURLParameterNames.cpp
URL/extractURLParameters.cpp
URL/firstSignificantSubdomain.cpp
URL/firstSignificantSubdomainCustom.cpp
URL/fragment.cpp
URL/netloc.cpp
URL/path.cpp
URL/pathFull.cpp
URL/port.cpp
URL/protocol.cpp
URL/queryString.cpp
URL/queryStringAndFragment.cpp
URL/registerFunctionsURL.cpp
URL/tldLookup.generated.cpp
URL/topLevelDomain.cpp
abs.cpp
acos.cpp
acosh.cpp
addDays.cpp
addHours.cpp
addMinutes.cpp
addMonths.cpp
addQuarters.cpp
addSeconds.cpp
addWeeks.cpp
addYears.cpp
addressToLine.cpp
addressToSymbol.cpp
aes_decrypt_mysql.cpp
aes_encrypt_mysql.cpp
appendTrailingCharIfAbsent.cpp
array/array.cpp
array/arrayAUC.cpp
array/arrayAggregation.cpp
array/arrayAll.cpp
array/arrayCompact.cpp
array/arrayConcat.cpp
array/arrayCount.cpp
array/arrayCumSum.cpp
array/arrayCumSumNonNegative.cpp
array/arrayDifference.cpp
array/arrayDistinct.cpp
array/arrayElement.cpp
array/arrayEnumerate.cpp
array/arrayEnumerateDense.cpp
array/arrayEnumerateDenseRanked.cpp
array/arrayEnumerateRanked.cpp
array/arrayEnumerateUniq.cpp
array/arrayEnumerateUniqRanked.cpp
array/arrayExists.cpp
array/arrayFill.cpp
array/arrayFilter.cpp
array/arrayFirst.cpp
array/arrayFirstIndex.cpp
array/arrayFlatten.cpp
array/arrayIntersect.cpp
array/arrayJoin.cpp
array/arrayMap.cpp
array/arrayPopBack.cpp
array/arrayPopFront.cpp
array/arrayPushBack.cpp
array/arrayPushFront.cpp
array/arrayReduce.cpp
array/arrayReduceInRanges.cpp
array/arrayResize.cpp
array/arrayReverse.cpp
array/arraySlice.cpp
array/arraySort.cpp
array/arraySplit.cpp
array/arrayUniq.cpp
array/arrayWithConstant.cpp
array/arrayZip.cpp
array/countEqual.cpp
array/emptyArray.cpp
array/emptyArrayToSingle.cpp
array/has.cpp
array/hasAll.cpp
array/hasAny.cpp
array/hasSubstr.cpp
array/indexOf.cpp
array/length.cpp
array/mapOp.cpp
array/mapPopulateSeries.cpp
array/range.cpp
array/registerFunctionsArray.cpp
asin.cpp
asinh.cpp
assumeNotNull.cpp
atan.cpp
atan2.cpp
atanh.cpp
bar.cpp
base64Decode.cpp
base64Encode.cpp
bitAnd.cpp
bitBoolMaskAnd.cpp
bitBoolMaskOr.cpp
bitCount.cpp
bitHammingDistance.cpp
bitNot.cpp
bitOr.cpp
bitRotateLeft.cpp
bitRotateRight.cpp
bitShiftLeft.cpp
bitShiftRight.cpp
bitSwapLastTwo.cpp
bitTest.cpp
bitTestAll.cpp
bitTestAny.cpp
bitWrapperFunc.cpp
bitXor.cpp
blockNumber.cpp
blockSerializedSize.cpp
blockSize.cpp
byteSize.cpp
caseWithExpression.cpp
cbrt.cpp
coalesce.cpp
concat.cpp
connectionId.cpp
convertCharset.cpp
cos.cpp
cosh.cpp
countDigits.cpp
countMatches.cpp
countSubstrings.cpp
countSubstringsCaseInsensitive.cpp
countSubstringsCaseInsensitiveUTF8.cpp
currentDatabase.cpp
currentProfiles.cpp
currentRoles.cpp
currentUser.cpp
dateDiff.cpp
dateName.cpp
date_trunc.cpp
decodeXMLComponent.cpp
decrypt.cpp
defaultValueOfArgumentType.cpp
defaultValueOfTypeName.cpp
demange.cpp
divide.cpp
divide/divide.cpp
divide/divideImpl.cpp
dumpColumnStructure.cpp
empty.cpp
encodeXMLComponent.cpp
encrypt.cpp
endsWith.cpp
equals.cpp
erf.cpp
erfc.cpp
errorCodeToName.cpp
evalMLMethod.cpp
exp.cpp
exp10.cpp
exp2.cpp
extract.cpp
extractAllGroupsHorizontal.cpp
extractAllGroupsVertical.cpp
extractGroups.cpp
extractTextFromHTML.cpp
extractTimeZoneFromFunctionArguments.cpp
filesystem.cpp
finalizeAggregation.cpp
formatDateTime.cpp
formatReadableQuantity.cpp
formatReadableSize.cpp
formatReadableTimeDelta.cpp
formatRow.cpp
formatString.cpp
fromModifiedJulianDay.cpp
fromUnixTimestamp64Micro.cpp
fromUnixTimestamp64Milli.cpp
fromUnixTimestamp64Nano.cpp
fuzzBits.cpp
gcd.cpp
generateUUIDv4.cpp
geoToH3.cpp
geoToS2.cpp
geohashDecode.cpp
geohashEncode.cpp
geohashesInBox.cpp
getMacro.cpp
getScalar.cpp
getServerPort.cpp
getSetting.cpp
getSizeOfEnumType.cpp
globalVariable.cpp
greatCircleDistance.cpp
greater.cpp
greaterOrEquals.cpp
greatest.cpp
h3EdgeAngle.cpp
h3EdgeLengthM.cpp
h3GetBaseCell.cpp
h3GetFaces.cpp
h3GetResolution.cpp
h3HexAreaM2.cpp
h3IndexesAreNeighbors.cpp
h3IsPentagon.cpp
h3IsValid.cpp
h3ResIsClassIII.cpp
h3ToChildren.cpp
h3ToGeoBoundary.cpp
h3ToParent.cpp
h3ToString.cpp
h3kRing.cpp
h3toGeo.cpp
hasColumnInTable.cpp
hasThreadFuzzer.cpp
hasToken.cpp
hasTokenCaseInsensitive.cpp
hyperscanRegexpChecker.cpp
hypot.cpp
identity.cpp
if.cpp
ifNotFinite.cpp
ifNull.cpp
ignore.cpp
ilike.cpp
in.cpp
indexHint.cpp
initialQueryID.cpp
initializeAggregation.cpp
intDiv.cpp
intDivOrZero.cpp
intExp10.cpp
intExp2.cpp
isConstant.cpp
isDecimalOverflow.cpp
isFinite.cpp
isIPAddressContainedIn.cpp
isInfinite.cpp
isNaN.cpp
isNotNull.cpp
isNull.cpp
isValidUTF8.cpp
isZeroOrNull.cpp
jumpConsistentHash.cpp
lcm.cpp
least.cpp
lemmatize.cpp
lengthUTF8.cpp
less.cpp
lessOrEquals.cpp
lgamma.cpp
like.cpp
log.cpp
log10.cpp
log1p.cpp
log2.cpp
logTrace.cpp
lowCardinalityIndices.cpp
lowCardinalityKeys.cpp
lower.cpp
lowerUTF8.cpp
map.cpp
match.cpp
materialize.cpp
mathConstants.cpp
minus.cpp
modulo.cpp
moduloOrZero.cpp
multiFuzzyMatchAllIndices.cpp
multiFuzzyMatchAny.cpp
multiFuzzyMatchAnyIndex.cpp
multiIf.cpp
multiMatchAllIndices.cpp
multiMatchAny.cpp
multiMatchAnyIndex.cpp
multiSearchAllPositions.cpp
multiSearchAllPositionsCaseInsensitive.cpp
multiSearchAllPositionsCaseInsensitiveUTF8.cpp
multiSearchAllPositionsUTF8.cpp
multiSearchAny.cpp
multiSearchAnyCaseInsensitive.cpp
multiSearchAnyCaseInsensitiveUTF8.cpp
multiSearchAnyUTF8.cpp
multiSearchFirstIndex.cpp
multiSearchFirstIndexCaseInsensitive.cpp
multiSearchFirstIndexCaseInsensitiveUTF8.cpp
multiSearchFirstIndexUTF8.cpp
multiSearchFirstPosition.cpp
multiSearchFirstPositionCaseInsensitive.cpp
multiSearchFirstPositionCaseInsensitiveUTF8.cpp
multiSearchFirstPositionUTF8.cpp
multiply.cpp
negate.cpp
neighbor.cpp
normalizeQuery.cpp
normalizedQueryHash.cpp
notEmpty.cpp
notEquals.cpp
notILike.cpp
notLike.cpp
now.cpp
now64.cpp
nullIf.cpp
padString.cpp
partitionId.cpp
plus.cpp
pointInEllipses.cpp
pointInPolygon.cpp
polygonArea.cpp
polygonConvexHull.cpp
polygonPerimeter.cpp
polygonsDistance.cpp
polygonsEquals.cpp
polygonsIntersection.cpp
polygonsSymDifference.cpp
polygonsUnion.cpp
polygonsWithin.cpp
position.cpp
positionCaseInsensitive.cpp
positionCaseInsensitiveUTF8.cpp
positionUTF8.cpp
pow.cpp
queryID.cpp
rand.cpp
rand64.cpp
randConstant.cpp
randomFixedString.cpp
randomPrintableASCII.cpp
randomString.cpp
randomStringUTF8.cpp
readWkt.cpp
regexpQuoteMeta.cpp
registerFunctions.cpp
registerFunctionsArithmetic.cpp
registerFunctionsComparison.cpp
registerFunctionsConditional.cpp
registerFunctionsConsistentHashing.cpp
registerFunctionsDateTime.cpp
registerFunctionsFormatting.cpp
registerFunctionsGeo.cpp
registerFunctionsHigherOrder.cpp
registerFunctionsIntrospection.cpp
registerFunctionsMath.cpp
registerFunctionsMiscellaneous.cpp
registerFunctionsNull.cpp
registerFunctionsRandom.cpp
registerFunctionsReinterpret.cpp
registerFunctionsSnowflake.cpp
registerFunctionsString.cpp
registerFunctionsStringRegexp.cpp
registerFunctionsStringSearch.cpp
registerFunctionsTuple.cpp
registerFunctionsUnixTimestamp64.cpp
registerFunctionsVisitParam.cpp
reinterpretAs.cpp
repeat.cpp
replaceAll.cpp
replaceOne.cpp
replaceRegexpAll.cpp
replaceRegexpOne.cpp
replicate.cpp
reverse.cpp
reverseUTF8.cpp
roundAge.cpp
roundDuration.cpp
roundToExp2.cpp
rowNumberInAllBlocks.cpp
rowNumberInBlock.cpp
runningAccumulate.cpp
runningConcurrency.cpp
runningDifference.cpp
runningDifferenceStartingWithFirstValue.cpp
s2CapContains.cpp
s2CapUnion.cpp
s2CellsIntersect.cpp
s2GetNeighbors.cpp
s2RectAdd.cpp
s2RectContains.cpp
s2RectIntersection.cpp
s2RectUnion.cpp
s2ToGeo.cpp
serverConstants.cpp
sigmoid.cpp
sign.cpp
sin.cpp
sinh.cpp
sleep.cpp
sleepEachRow.cpp
snowflake.cpp
sqrt.cpp
startsWith.cpp
stem.cpp
stringCutToZero.cpp
stringToH3.cpp
substring.cpp
subtractDays.cpp
subtractHours.cpp
subtractMinutes.cpp
subtractMonths.cpp
subtractQuarters.cpp
subtractSeconds.cpp
subtractWeeks.cpp
subtractYears.cpp
svg.cpp
synonyms.cpp
tan.cpp
tanh.cpp
tgamma.cpp
throwIf.cpp
tid.cpp
timeSlot.cpp
timeSlots.cpp
timezoneOf.cpp
timezoneOffset.cpp
toColumnTypeName.cpp
toCustomWeek.cpp
toDayOfMonth.cpp
toDayOfWeek.cpp
toDayOfYear.cpp
toFixedString.cpp
toHour.cpp
toISOWeek.cpp
toISOYear.cpp
toJSONString.cpp
toLowCardinality.cpp
toMinute.cpp
toModifiedJulianDay.cpp
toMonday.cpp
toMonth.cpp
toNullable.cpp
toQuarter.cpp
toRelativeDayNum.cpp
toRelativeHourNum.cpp
toRelativeMinuteNum.cpp
toRelativeMonthNum.cpp
toRelativeQuarterNum.cpp
toRelativeSecondNum.cpp
toRelativeWeekNum.cpp
toRelativeYearNum.cpp
toSecond.cpp
toStartOfDay.cpp
toStartOfFifteenMinutes.cpp
toStartOfFiveMinute.cpp
toStartOfHour.cpp
toStartOfISOYear.cpp
toStartOfInterval.cpp
toStartOfMinute.cpp
toStartOfMonth.cpp
toStartOfQuarter.cpp
toStartOfSecond.cpp
toStartOfTenMinutes.cpp
toStartOfYear.cpp
toTime.cpp
toTimezone.cpp
toTypeName.cpp
toUnixTimestamp64Micro.cpp
toUnixTimestamp64Milli.cpp
toUnixTimestamp64Nano.cpp
toValidUTF8.cpp
toYYYYMM.cpp
toYYYYMMDD.cpp
toYYYYMMDDhhmmss.cpp
toYear.cpp
today.cpp
transform.cpp
trap.cpp
trim.cpp
tryBase64Decode.cpp
tuple.cpp
tupleElement.cpp
tupleHammingDistance.cpp
tupleToNameValuePairs.cpp
upper.cpp
upperUTF8.cpp
validateNestedArraySizes.cpp
visibleWidth.cpp
visitParamExtractBool.cpp
visitParamExtractFloat.cpp
visitParamExtractInt.cpp
visitParamExtractRaw.cpp
visitParamExtractString.cpp
visitParamExtractUInt.cpp
visitParamHas.cpp
wkt.cpp
yandexConsistentHash.cpp
yesterday.cpp
)
END()

43
src/Functions/ya.make.in
View File

@ -1,43 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
CFLAGS(
-DUSE_SSL -DUSE_XXHASH
)
ADDINCL(
library/cpp/consistent_hashing
contrib/libs/farmhash
contrib/libs/h3/h3lib/include
contrib/libs/hyperscan/src
contrib/libs/libdivide
contrib/libs/rapidjson/include
contrib/libs/xxhash
GLOBAL contrib/restricted/murmurhash
)
PEERDIR(
clickhouse/src/Common
clickhouse/src/Parsers
clickhouse/src/Dictionaries
contrib/libs/farmhash
contrib/libs/fastops/fastops
contrib/libs/h3
contrib/libs/hyperscan
contrib/libs/icu
contrib/libs/libdivide
contrib/libs/metrohash
contrib/libs/rapidjson
contrib/libs/xxhash
contrib/restricted/murmurhash
library/cpp/consistent_hashing
)
# "Arcadia" build is slightly deficient. It lacks many libraries that we need.
SRCS(
<? find . -name '*.cpp' | grep -i -v -P 'tests|Bitmap|abtesting' | grep -v -F examples | sed 's/^\.\// /' | sort ?>
)
END()

164
src/IO/ZstdDeflatingAppendableWriteBuffer.cpp Normal file
View File

@ -0,0 +1,164 @@
#include <IO/ZstdDeflatingAppendableWriteBuffer.h>
#include <Common/MemoryTracker.h>
#include <Common/Exception.h>
namespace DB
{
namespace ErrorCodes
{
extern const int ZSTD_ENCODER_FAILED;
}
ZstdDeflatingAppendableWriteBuffer::ZstdDeflatingAppendableWriteBuffer(
WriteBuffer & out_, int compression_level, bool append_to_existing_stream_,
size_t buf_size, char * existing_memory, size_t alignment)
: BufferWithOwnMemory<WriteBuffer>(buf_size, existing_memory, alignment)
, out(out_)
, append_to_existing_stream(append_to_existing_stream_)
{
cctx = ZSTD_createCCtx();
if (cctx == nullptr)
throw Exception(ErrorCodes::ZSTD_ENCODER_FAILED, "zstd stream encoder init failed: zstd version: {}", ZSTD_VERSION_STRING);
size_t ret = ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compression_level);
if (ZSTD_isError(ret))
throw Exception(ErrorCodes::ZSTD_ENCODER_FAILED, "zstd stream encoder option setting failed: error code: {}; zstd version: {}", ret, ZSTD_VERSION_STRING);
input = {nullptr, 0, 0};
output = {nullptr, 0, 0};
}
void ZstdDeflatingAppendableWriteBuffer::nextImpl()
{
if (!offset())
return;
ZSTD_EndDirective mode = ZSTD_e_flush;
input.src = reinterpret_cast<unsigned char *>(working_buffer.begin());
input.size = offset();
input.pos = 0;
if (first_write && append_to_existing_stream)
{
addEmptyBlock();
first_write = false;
}
try
{
bool ended = false;
do
{
out.nextIfAtEnd();
output.dst = reinterpret_cast<unsigned char *>(out.buffer().begin());
output.size = out.buffer().size();
output.pos = out.offset();
size_t compression_result = ZSTD_compressStream2(cctx, &output, &input, mode);
if (ZSTD_isError(compression_result))
throw Exception(
ErrorCodes::ZSTD_ENCODER_FAILED, "Zstd stream encoding failed: error code: {}; zstd version: {}", ZSTD_getErrorName(compression_result), ZSTD_VERSION_STRING);
out.position() = out.buffer().begin() + output.pos;
bool everything_was_compressed = (input.pos == input.size);
bool everything_was_flushed = compression_result == 0;
ended = everything_was_compressed && everything_was_flushed;
} while (!ended);
}
catch (...)
{
/// Do not try to write next time after exception.
out.position() = out.buffer().begin();
throw;
}
}
void ZstdDeflatingAppendableWriteBuffer::finish()
{
if (finished || first_write)
{
/// Nothing was written or we have already finished
return;
}
try
{
finishImpl();
out.finalize();
finished = true;
}
catch (...)
{
/// Do not try to flush next time after exception.
out.position() = out.buffer().begin();
finished = true;
throw;
}
}
ZstdDeflatingAppendableWriteBuffer::~ZstdDeflatingAppendableWriteBuffer()
{
/// FIXME move final flush into the caller
MemoryTracker::LockExceptionInThread lock(VariableContext::Global);
finish();
try
{
int err = ZSTD_freeCCtx(cctx);
/// This is just in case, since it is impossible to get an error by using this wrapper.
if (unlikely(err))
throw Exception(ErrorCodes::ZSTD_ENCODER_FAILED, "ZSTD_freeCCtx failed: error: '{}'; zstd version: {}", ZSTD_getErrorName(err), ZSTD_VERSION_STRING);
}
catch (...)
{
/// It is OK not to terminate under an error from ZSTD_freeCCtx()
/// since all data already written to the stream.
tryLogCurrentException(__PRETTY_FUNCTION__);
}
}
void ZstdDeflatingAppendableWriteBuffer::finishImpl()
{
next();
out.nextIfAtEnd();
input.src = reinterpret_cast<unsigned char *>(working_buffer.begin());
input.size = offset();
input.pos = 0;
output.dst = reinterpret_cast<unsigned char *>(out.buffer().begin());
output.size = out.buffer().size();
output.pos = out.offset();
size_t remaining = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);
while (remaining != 0)
{
if (ZSTD_isError(remaining))
throw Exception(ErrorCodes::ZSTD_ENCODER_FAILED, "zstd stream encoder end failed: error: '{}' zstd version: {}", ZSTD_getErrorName(remaining), ZSTD_VERSION_STRING);
remaining = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);
}
out.position() = out.buffer().begin() + output.pos;
}
void ZstdDeflatingAppendableWriteBuffer::addEmptyBlock()
{
/// HACK: https://github.com/facebook/zstd/issues/2090#issuecomment-620158967
static const char empty_block[3] = {0x01, 0x00, 0x00};
if (out.buffer().size() - out.offset() < sizeof(empty_block))
out.next();
std::memcpy(out.buffer().begin() + out.offset(), empty_block, sizeof(empty_block));
out.position() = out.buffer().begin() + out.offset() + sizeof(empty_block);
}
}

71
src/IO/ZstdDeflatingAppendableWriteBuffer.h Normal file
View File

@ -0,0 +1,71 @@
#pragma once
#include <IO/BufferWithOwnMemory.h>
#include <IO/CompressionMethod.h>
#include <IO/WriteBuffer.h>
#include <zstd.h>
namespace DB
{
/// Performs stream compression using zstd library and writes compressed data to out_ WriteBuffer.
/// Main differences from ZstdDeflatingWriteBuffer:
/// 1) Allows to continue to write to the same output even if finalize() (or destructor) was not called, for example
/// when server was killed with 9 signal. Natively zstd doesn't support such feature because
/// ZSTD_decompressStream expect to see empty block at the end of each frame. There is not API function for it
/// so we just use HACK and add empty block manually on the first write (see addEmptyBlock). Maintainers of zstd
/// said that there is no risks of compatibility issues https://github.com/facebook/zstd/issues/2090#issuecomment-620158967.
/// 2) Doesn't support internal ZSTD check-summing, because ZSTD checksums written at the end of frame (frame epilogue).
///
class ZstdDeflatingAppendableWriteBuffer : public BufferWithOwnMemory<WriteBuffer>
{
public:
ZstdDeflatingAppendableWriteBuffer(
WriteBuffer & out_,
int compression_level,
bool append_to_existing_stream_, /// if true then out mustn't be empty
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
char * existing_memory = nullptr,
size_t alignment = 0);
/// Write terminating ZSTD_e_end: empty block + frame epilogue. BTW it
/// should be almost noop, because frame epilogue contains only checksums,
/// and they are disabled for this buffer.
void finalize() override { finish(); }
~ZstdDeflatingAppendableWriteBuffer() override;
void sync() override
{
next();
out.sync();
}
private:
/// NOTE: will fill compressed data to the out.working_buffer, but will not call out.next method until the buffer is full
void nextImpl() override;
/// Flush all pending data and write zstd footer to the underlying buffer.
/// After the first call to this function, subsequent calls will have no effect and
/// an attempt to write to this buffer will result in exception.
void finish();
void finishImpl();
/// Adding zstd empty block to out.working_buffer
void addEmptyBlock();
WriteBuffer & out;
/// We appending data to existing stream so on the first nextImpl call we
/// will append empty block.
bool append_to_existing_stream;
ZSTD_CCtx * cctx;
ZSTD_inBuffer input;
ZSTD_outBuffer output;
/// Flipped in finish call
bool finished = false;
/// Flipped on the first nextImpl call
bool first_write = true;
};
}

14
src/IO/ZstdDeflatingWriteBuffer.cpp
View File

@ -40,7 +40,7 @@ ZstdDeflatingWriteBuffer::~ZstdDeflatingWriteBuffer()
int err = ZSTD_freeCCtx(cctx);
/// This is just in case, since it is impossible to get an error by using this wrapper.
if (unlikely(err))
throw Exception(ErrorCodes::ZSTD_ENCODER_FAILED, "ZSTD_freeCCtx failed: error code: {}; zstd version: {}", err, ZSTD_VERSION_STRING);
throw Exception(ErrorCodes::ZSTD_ENCODER_FAILED, "ZSTD_freeCCtx failed: error: '{}'; zstd version: {}", ZSTD_getErrorName(err), ZSTD_VERSION_STRING);
}
catch (...)
{
@ -73,9 +73,17 @@ void ZstdDeflatingWriteBuffer::nextImpl()
output.pos = out->offset();
ZSTD_compressStream2(cctx, &output, &input, mode);
size_t compression_result = ZSTD_compressStream2(cctx, &output, &input, mode);
if (ZSTD_isError(compression_result))
throw Exception(
ErrorCodes::ZSTD_ENCODER_FAILED, "Zstd stream encoding failed: error: '{}'; zstd version: {}", ZSTD_getErrorName(compression_result), ZSTD_VERSION_STRING);
out->position() = out->buffer().begin() + output.pos;
ended = (input.pos == input.size);
bool everything_was_compressed = (input.pos == input.size);
bool everything_was_flushed = compression_result == 0;
ended = everything_was_compressed && everything_was_flushed;
} while (!ended);
}
catch (...)

5
src/IO/ZstdDeflatingWriteBuffer.h
View File

@ -24,6 +24,11 @@ public:
~ZstdDeflatingWriteBuffer() override;
void sync() override
{
out->sync();
}
private:
void nextImpl() override;

4
src/IO/ZstdInflatingReadBuffer.cpp
View File

@ -52,10 +52,10 @@ bool ZstdInflatingReadBuffer::nextImpl()
size_t ret = ZSTD_decompressStream(dctx, &output, &input);
if (ZSTD_isError(ret))
throw Exception(
ErrorCodes::ZSTD_DECODER_FAILED, "Zstd stream decoding failed: error code: {}; zstd version: {}", ret, ZSTD_VERSION_STRING);
ErrorCodes::ZSTD_DECODER_FAILED, "Zstd stream encoding failed: error '{}'; zstd version: {}", ZSTD_getErrorName(ret), ZSTD_VERSION_STRING);
/// Check that something has changed after decompress (input or output position)
assert(output.pos > 0 || in->position() < in->buffer().begin() + input.pos);
assert(in->eof() || output.pos > 0 || in->position() < in->buffer().begin() + input.pos);
/// move position to the end of read data
in->position() = in->buffer().begin() + input.pos;

92
src/IO/ya.make
View File

@ -1,92 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
ADDINCL(
contrib/libs/zstd/include
contrib/libs/lz4
contrib/restricted/fast_float/include
)
PEERDIR(
clickhouse/src/Common
contrib/libs/brotli/dec
contrib/libs/brotli/enc
contrib/libs/poco/NetSSL_OpenSSL
contrib/libs/zstd
contrib/libs/lz4
contrib/restricted/fast_float
)
SRCS(
AIO.cpp
AsynchronousReadBufferFromFile.cpp
AsynchronousReadBufferFromFileDescriptor.cpp
BrotliReadBuffer.cpp
BrotliWriteBuffer.cpp
Bzip2ReadBuffer.cpp
Bzip2WriteBuffer.cpp
CascadeWriteBuffer.cpp
CompressionMethod.cpp
DoubleConverter.cpp
FileEncryptionCommon.cpp
HTTPChunkedReadBuffer.cpp
HTTPCommon.cpp
HashingWriteBuffer.cpp
LZMADeflatingWriteBuffer.cpp
LZMAInflatingReadBuffer.cpp
LimitReadBuffer.cpp
Lz4DeflatingWriteBuffer.cpp
Lz4InflatingReadBuffer.cpp
MMapReadBufferFromFile.cpp
MMapReadBufferFromFileDescriptor.cpp
MMapReadBufferFromFileWithCache.cpp
MMappedFile.cpp
MMappedFileDescriptor.cpp
MemoryReadWriteBuffer.cpp
MySQLBinlogEventReadBuffer.cpp
MySQLPacketPayloadReadBuffer.cpp
MySQLPacketPayloadWriteBuffer.cpp
NullWriteBuffer.cpp
OpenedFile.cpp
PeekableReadBuffer.cpp
Progress.cpp
ReadBufferFromEncryptedFile.cpp
ReadBufferFromFile.cpp
ReadBufferFromFileBase.cpp
ReadBufferFromFileDecorator.cpp
ReadBufferFromFileDescriptor.cpp
ReadBufferFromIStream.cpp
ReadBufferFromMemory.cpp
ReadBufferFromPocoSocket.cpp
ReadHelpers.cpp
SeekAvoidingReadBuffer.cpp
SynchronousReader.cpp
ThreadPoolReader.cpp
TimeoutSetter.cpp
UseSSL.cpp
WriteBufferFromEncryptedFile.cpp
WriteBufferFromFile.cpp
WriteBufferFromFileBase.cpp
WriteBufferFromFileDecorator.cpp
WriteBufferFromFileDescriptor.cpp
WriteBufferFromFileDescriptorDiscardOnFailure.cpp
WriteBufferFromHTTP.cpp
WriteBufferFromOStream.cpp
WriteBufferFromPocoSocket.cpp
WriteBufferFromTemporaryFile.cpp
WriteBufferValidUTF8.cpp
WriteHelpers.cpp
ZlibDeflatingWriteBuffer.cpp
ZlibInflatingReadBuffer.cpp
ZstdDeflatingWriteBuffer.cpp
ZstdInflatingReadBuffer.cpp
copyData.cpp
createReadBufferFromFileBase.cpp
parseDateTimeBestEffort.cpp
readFloatText.cpp
)
END()

25
src/IO/ya.make.in
View File

@ -1,25 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
ADDINCL(
contrib/libs/zstd/include
contrib/libs/lz4
contrib/restricted/fast_float/include
)
PEERDIR(
clickhouse/src/Common
contrib/libs/brotli/dec
contrib/libs/brotli/enc
contrib/libs/poco/NetSSL_OpenSSL
contrib/libs/zstd
contrib/libs/lz4
contrib/restricted/fast_float
)
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | grep -v -P 'S3|HDFS' | sed 's/^\.\// /' | sort ?>
)
END()

2
src/Interpreters/Context.h
View File

@ -283,7 +283,7 @@ private:
/// XXX: move this stuff to shared part instead.
ContextMutablePtr buffer_context; /// Buffer context. Could be equal to this.
/// A flag, used to distinguish between user query and internal query to a database engine (MaterializePostgreSQL).
/// A flag, used to distinguish between user query and internal query to a database engine (MaterializedPostgreSQL).
bool is_internal_query = false;

4
src/Interpreters/IdentifierSemantic.cpp
View File

@ -162,7 +162,7 @@ IdentifierSemantic::ColumnMatch IdentifierSemantic::canReferColumnToTable(const
{
/// database.table.column
if (doesIdentifierBelongTo(identifier, db_and_table.database, db_and_table.table))
return ColumnMatch::DbAndTable;
return ColumnMatch::DBAndTable;
/// alias.column
if (doesIdentifierBelongTo(identifier, db_and_table.alias))
@ -199,7 +199,7 @@ void IdentifierSemantic::setColumnShortName(ASTIdentifier & identifier, const Da
case ColumnMatch::TableAlias:
to_strip = 1;
break;
case ColumnMatch::DbAndTable:
case ColumnMatch::DBAndTable:
to_strip = 2;
break;
default:

2
src/Interpreters/IdentifierSemantic.h
View File

@ -31,7 +31,7 @@ struct IdentifierSemantic
ColumnName, /// column qualified with column names list
AliasedTableName, /// column qualified with table name (but table has an alias so its priority is lower than TableName)
TableName, /// column qualified with table name
DbAndTable, /// column qualified with database and table name
DBAndTable, /// column qualified with database and table name
TableAlias, /// column qualified with table alias
Ambiguous,
};

16
src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -154,7 +154,9 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
throw Exception(ErrorCodes::UNKNOWN_DATABASE_ENGINE, "Unknown database engine: {}", serializeAST(*create.storage));
}
if (create.storage->engine->name == "Atomic" || create.storage->engine->name == "Replicated" || create.storage->engine->name == "MaterializedPostgreSQL")
if (create.storage->engine->name == "Atomic"
|| create.storage->engine->name == "Replicated"
|| create.storage->engine->name == "MaterializedPostgreSQL")
{
if (create.attach && create.uuid == UUIDHelpers::Nil)
throw Exception(ErrorCodes::INCORRECT_QUERY, "UUID must be specified for ATTACH. "
@ -167,7 +169,8 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
if (!create.attach && fs::exists(metadata_path))
throw Exception(ErrorCodes::DATABASE_ALREADY_EXISTS, "Metadata directory {} already exists", metadata_path.string());
}
else if (create.storage->engine->name == "MaterializeMySQL" || create.storage->engine->name == "MaterializedMySQL")
else if (create.storage->engine->name == "MaterializeMySQL"
|| create.storage->engine->name == "MaterializedMySQL")
{
/// It creates nested database with Ordinary or Atomic engine depending on UUID in query and default engine setting.
/// Do nothing if it's an internal ATTACH on server startup or short-syntax ATTACH query from user,
@ -215,13 +218,17 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
"Enable allow_experimental_database_materialized_mysql to use it.", ErrorCodes::UNKNOWN_DATABASE_ENGINE);
}
if (create.storage->engine->name == "Replicated" && !getContext()->getSettingsRef().allow_experimental_database_replicated && !internal)
if (create.storage->engine->name == "Replicated"
&& !getContext()->getSettingsRef().allow_experimental_database_replicated
&& !internal)
{
throw Exception("Replicated is an experimental database engine. "
"Enable allow_experimental_database_replicated to use it.", ErrorCodes::UNKNOWN_DATABASE_ENGINE);
}
if (create.storage->engine->name == "MaterializedPostgreSQL" && !getContext()->getSettingsRef().allow_experimental_database_materialized_postgresql && !internal)
if (create.storage->engine->name == "MaterializedPostgreSQL"
&& !getContext()->getSettingsRef().allow_experimental_database_materialized_postgresql
&& !internal)
{
throw Exception("MaterializedPostgreSQL is an experimental database engine. "
"Enable allow_experimental_database_materialized_postgresql to use it.", ErrorCodes::UNKNOWN_DATABASE_ENGINE);
@ -624,7 +631,6 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti
else
throw Exception("Incorrect CREATE query: required list of column descriptions or AS section or SELECT.", ErrorCodes::INCORRECT_QUERY);
/// Even if query has list of columns, canonicalize it (unfold Nested columns).
if (!create.columns_list)
create.set(create.columns_list, std::make_shared<ASTColumns>());

2
src/Interpreters/JoinedTables.cpp
View File

@ -128,7 +128,7 @@ private:
/// Table has an alias. We do not need to rewrite qualified names with table alias (match == ColumnMatch::TableName).
auto match = IdentifierSemantic::canReferColumnToTable(identifier, table);
if (match == IdentifierSemantic::ColumnMatch::AliasedTableName ||
match == IdentifierSemantic::ColumnMatch::DbAndTable)
match == IdentifierSemantic::ColumnMatch::DBAndTable)
{
if (rewritten)
throw Exception("Failed to rewrite distributed table names. Ambiguous column '" + identifier.name() + "'",

2
src/Interpreters/TranslateQualifiedNamesVisitor.cpp
View File

@ -334,7 +334,7 @@ void RestoreQualifiedNamesMatcher::Data::changeTable(ASTIdentifier & identifier)
{
case IdentifierSemantic::ColumnMatch::AliasedTableName:
case IdentifierSemantic::ColumnMatch::TableName:
case IdentifierSemantic::ColumnMatch::DbAndTable:
case IdentifierSemantic::ColumnMatch::DBAndTable:
IdentifierSemantic::setColumnLongName(identifier, remote_table);
break;
default:

18
src/Interpreters/evaluateConstantExpression.cpp
View File

@ -104,24 +104,6 @@ ASTPtr evaluateConstantExpressionForDatabaseName(const ASTPtr & node, ContextPtr
return res;
}
std::tuple<bool, ASTPtr> evaluateDatabaseNameForMergeEngine(const ASTPtr & node, ContextPtr context)
{
if (const auto * func = node->as<ASTFunction>(); func && func->name == "REGEXP")
{
if (func->arguments->children.size() != 1)
throw Exception("Arguments for REGEXP in Merge ENGINE should be 1", ErrorCodes::BAD_ARGUMENTS);
auto * literal = func->arguments->children[0]->as<ASTLiteral>();
if (!literal || literal->value.safeGet<String>().empty())
throw Exception("Argument for REGEXP in Merge ENGINE should be a non empty String Literal", ErrorCodes::BAD_ARGUMENTS);
return std::tuple{true, func->arguments->children[0]};
}
auto ast = evaluateConstantExpressionForDatabaseName(node, context);
return std::tuple{false, ast};
}
namespace
{

2
src/Interpreters/evaluateConstantExpression.h
View File

@ -53,6 +53,4 @@ ASTPtr evaluateConstantExpressionForDatabaseName(const ASTPtr & node, ContextPtr
*/
std::optional<Blocks> evaluateExpressionOverConstantCondition(const ASTPtr & node, const ExpressionActionsPtr & target_expr, size_t & limit);
// Evaluate database name or regexp for StorageMerge and TableFunction merge
std::tuple<bool, ASTPtr> evaluateDatabaseNameForMergeEngine(const ASTPtr & node, ContextPtr context);
}

199
src/Interpreters/ya.make
View File

@ -1,199 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
ADDINCL(
contrib/libs/libdivide
contrib/libs/pdqsort
)
PEERDIR(
clickhouse/src/Core
contrib/libs/libdivide
contrib/libs/pdqsort
)
NO_COMPILER_WARNINGS()
SRCS(
ActionLocksManager.cpp
ActionsDAG.cpp
ActionsVisitor.cpp
AggregateDescription.cpp
Aggregator.cpp
ApplyWithAliasVisitor.cpp
ApplyWithGlobalVisitor.cpp
ApplyWithSubqueryVisitor.cpp
ArithmeticOperationsInAgrFuncOptimize.cpp
ArrayJoinAction.cpp
AsynchronousInsertQueue.cpp
AsynchronousMetricLog.cpp
AsynchronousMetrics.cpp
BloomFilter.cpp
CatBoostModel.cpp
ClientInfo.cpp
Cluster.cpp
ClusterProxy/SelectStreamFactory.cpp
ClusterProxy/executeQuery.cpp
CollectJoinOnKeysVisitor.cpp
ColumnAliasesVisitor.cpp
Context.cpp
ConvertStringsToEnumVisitor.cpp
CrashLog.cpp
CrossToInnerJoinVisitor.cpp
DDLTask.cpp
DDLWorker.cpp
DNSCacheUpdater.cpp
DatabaseAndTableWithAlias.cpp
DatabaseCatalog.cpp
DictionaryReader.cpp
DuplicateOrderByVisitor.cpp
EmbeddedDictionaries.cpp
ExecuteScalarSubqueriesVisitor.cpp
ExpressionActions.cpp
ExpressionActionsSettings.cpp
ExpressionAnalyzer.cpp
ExternalDictionariesLoader.cpp
ExternalLoader.cpp
ExternalLoaderDictionaryStorageConfigRepository.cpp
ExternalLoaderTempConfigRepository.cpp
ExternalLoaderXMLConfigRepository.cpp
ExternalModelsLoader.cpp
ExternalUserDefinedExecutableFunctionsLoader.cpp
ExtractExpressionInfoVisitor.cpp
FillingRow.cpp
FunctionNameNormalizer.cpp
GatherFunctionQuantileVisitor.cpp
HashJoin.cpp
IExternalLoadable.cpp
IInterpreter.cpp
IInterpreterUnionOrSelectQuery.cpp
IdentifierSemantic.cpp
InJoinSubqueriesPreprocessor.cpp
InternalTextLogsQueue.cpp
InterpreterAlterQuery.cpp
InterpreterBackupQuery.cpp
InterpreterCheckQuery.cpp
InterpreterCreateFunctionQuery.cpp
InterpreterCreateQuery.cpp
InterpreterCreateQuotaQuery.cpp
InterpreterCreateRoleQuery.cpp
InterpreterCreateRowPolicyQuery.cpp
InterpreterCreateSettingsProfileQuery.cpp
InterpreterCreateUserQuery.cpp
InterpreterDescribeQuery.cpp
InterpreterDropAccessEntityQuery.cpp
InterpreterDropFunctionQuery.cpp
InterpreterDropQuery.cpp
InterpreterExistsQuery.cpp
InterpreterExplainQuery.cpp
InterpreterExternalDDLQuery.cpp
InterpreterFactory.cpp
InterpreterGrantQuery.cpp
InterpreterInsertQuery.cpp
InterpreterKillQueryQuery.cpp
InterpreterOptimizeQuery.cpp
InterpreterRenameQuery.cpp
InterpreterSelectIntersectExceptQuery.cpp
InterpreterSelectQuery.cpp
InterpreterSelectWithUnionQuery.cpp
InterpreterSetQuery.cpp
InterpreterSetRoleQuery.cpp
InterpreterShowAccessEntitiesQuery.cpp
InterpreterShowAccessQuery.cpp
InterpreterShowCreateAccessEntityQuery.cpp
InterpreterShowCreateQuery.cpp
InterpreterShowGrantsQuery.cpp
InterpreterShowPrivilegesQuery.cpp
InterpreterShowProcesslistQuery.cpp
InterpreterShowTablesQuery.cpp
InterpreterSystemQuery.cpp
InterpreterUseQuery.cpp
InterpreterWatchQuery.cpp
InterserverCredentials.cpp
JoinSwitcher.cpp
JoinToSubqueryTransformVisitor.cpp
JoinedTables.cpp
Lemmatizers.cpp
LogicalExpressionsOptimizer.cpp
MarkTableIdentifiersVisitor.cpp
MergeJoin.cpp
MetricLog.cpp
MutationsInterpreter.cpp
MySQL/InterpretersMySQLDDLQuery.cpp
NormalizeSelectWithUnionQueryVisitor.cpp
NullableUtils.cpp
OpenTelemetrySpanLog.cpp
OptimizeIfChains.cpp
OptimizeIfWithConstantConditionVisitor.cpp
OptimizeShardingKeyRewriteInVisitor.cpp
PartLog.cpp
PredicateExpressionsOptimizer.cpp
PredicateRewriteVisitor.cpp
ProcessList.cpp
ProfileEventsExt.cpp
QueryAliasesVisitor.cpp
QueryLog.cpp
QueryNormalizer.cpp
QueryParameterVisitor.cpp
QueryThreadLog.cpp
QueryViewsLog.cpp
RemoveInjectiveFunctionsVisitor.cpp
RenameColumnVisitor.cpp
ReplaceQueryParameterVisitor.cpp
RequiredSourceColumnsData.cpp
RequiredSourceColumnsVisitor.cpp
RewriteAnyFunctionVisitor.cpp
RewriteCountVariantsVisitor.cpp
RewriteFunctionToSubcolumnVisitor.cpp
RewriteSumIfFunctionVisitor.cpp
RowRefs.cpp
SelectIntersectExceptQueryVisitor.cpp
Session.cpp
SessionLog.cpp
Set.cpp
SetVariants.cpp
SortedBlocksWriter.cpp
StorageID.cpp
SubqueryForSet.cpp
SynonymsExtensions.cpp
SystemLog.cpp
TableJoin.cpp
TablesStatus.cpp
TextLog.cpp
ThreadStatusExt.cpp
TraceLog.cpp
TranslateQualifiedNamesVisitor.cpp
TreeOptimizer.cpp
TreeRewriter.cpp
UserDefinedExecutableFunction.cpp
UserDefinedExecutableFunctionFactory.cpp
UserDefinedSQLFunctionFactory.cpp
UserDefinedSQLFunctionVisitor.cpp
UserDefinedSQLObjectsLoader.cpp
WindowDescription.cpp
ZooKeeperLog.cpp
addMissingDefaults.cpp
addTypeConversionToAST.cpp
castColumn.cpp
convertFieldToType.cpp
createBlockSelector.cpp
evaluateConstantExpression.cpp
executeDDLQueryOnCluster.cpp
executeQuery.cpp
getClusterName.cpp
getHeaderForProcessingStage.cpp
getTableExpressions.cpp
inplaceBlockConversions.cpp
interpretSubquery.cpp
join_common.cpp
loadMetadata.cpp
processColumnTransformers.cpp
replaceAliasColumnsInQuery.cpp
sortBlock.cpp
)
END()

23
src/Interpreters/ya.make.in
View File

@ -1,23 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
ADDINCL(
contrib/libs/libdivide
contrib/libs/pdqsort
)
PEERDIR(
clickhouse/src/Core
contrib/libs/libdivide
contrib/libs/pdqsort
)
NO_COMPILER_WARNINGS()
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F JIT | grep -v -F examples | sed 's/^\.\// /' | sort ?>
)
END()

48
src/Parsers/ExpressionElementParsers.cpp
View File

@ -1524,6 +1524,25 @@ bool ParserNull::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
return false;
}
static bool parseNumber(char * buffer, size_t size, bool negative, int base, Field & res)
{
errno = 0; /// Functions strto* don't clear errno.
char * pos_integer = buffer;
UInt64 uint_value = std::strtoull(buffer, &pos_integer, base);
if (pos_integer == buffer + size && errno != ERANGE && (!negative || uint_value <= (1ULL << 63)))
{
if (negative)
res = static_cast<Int64>(-uint_value);
else
res = uint_value;
return true;
}
return false;
}
bool ParserNumber::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
{
@ -1564,6 +1583,22 @@ bool ParserNumber::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
Float64 float_value = std::strtod(buf, &pos_double);
if (pos_double != buf + pos->size() || errno == ERANGE)
{
/// Try to parse number as binary literal representation. Example: 0b0001.
if (pos->size() > 2 && buf[0] == '0' && buf[1] == 'b')
{
char * buf_skip_prefix = buf + 2;
if (parseNumber(buf_skip_prefix, pos->size() - 2, negative, 2, res))
{
auto literal = std::make_shared<ASTLiteral>(res);
literal->begin = literal_begin;
literal->end = ++pos;
node = literal;
return true;
}
}
expected.add(pos, "number");
return false;
}
@ -1578,22 +1613,13 @@ bool ParserNumber::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
/// try to use more exact type: UInt64
char * pos_integer = buf;
errno = 0;
UInt64 uint_value = std::strtoull(buf, &pos_integer, 0);
if (pos_integer == pos_double && errno != ERANGE && (!negative || uint_value <= (1ULL << 63)))
{
if (negative)
res = static_cast<Int64>(-uint_value);
else
res = uint_value;
}
parseNumber(buf, pos->size(), negative, 0, res);
auto literal = std::make_shared<ASTLiteral>(res);
literal->begin = literal_begin;
literal->end = ++pos;
node = literal;
return true;
}

161
src/Parsers/ya.make
View File

@ -1,161 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
SRCS(
ASTAlterQuery.cpp
ASTAsterisk.cpp
ASTBackupQuery.cpp
ASTColumnDeclaration.cpp
ASTColumnsMatcher.cpp
ASTColumnsTransformers.cpp
ASTConstraintDeclaration.cpp
ASTCreateFunctionQuery.cpp
ASTCreateQuery.cpp
ASTCreateQuotaQuery.cpp
ASTCreateRoleQuery.cpp
ASTCreateRowPolicyQuery.cpp
ASTCreateSettingsProfileQuery.cpp
ASTCreateUserQuery.cpp
ASTDatabaseOrNone.cpp
ASTDictionary.cpp
ASTDictionaryAttributeDeclaration.cpp
ASTDropAccessEntityQuery.cpp
ASTDropFunctionQuery.cpp
ASTDropQuery.cpp
ASTExpressionList.cpp
ASTFunction.cpp
ASTFunctionWithKeyValueArguments.cpp
ASTGrantQuery.cpp
ASTIdentifier.cpp
ASTIndexDeclaration.cpp
ASTInsertQuery.cpp
ASTKillQueryQuery.cpp
ASTLiteral.cpp
ASTNameTypePair.cpp
ASTOptimizeQuery.cpp
ASTOrderByElement.cpp
ASTPartition.cpp
ASTProjectionDeclaration.cpp
ASTProjectionSelectQuery.cpp
ASTQualifiedAsterisk.cpp
ASTQueryParameter.cpp
ASTQueryWithOnCluster.cpp
ASTQueryWithOutput.cpp
ASTQueryWithTableAndOutput.cpp
ASTRolesOrUsersSet.cpp
ASTRowPolicyName.cpp
ASTSampleRatio.cpp
ASTSelectIntersectExceptQuery.cpp
ASTSelectQuery.cpp
ASTSelectWithUnionQuery.cpp
ASTSetQuery.cpp
ASTSetRoleQuery.cpp
ASTSettingsProfileElement.cpp
ASTShowAccessEntitiesQuery.cpp
ASTShowCreateAccessEntityQuery.cpp
ASTShowGrantsQuery.cpp
ASTShowTablesQuery.cpp
ASTSubquery.cpp
ASTSystemQuery.cpp
ASTTTLElement.cpp
ASTTablesInSelectQuery.cpp
ASTUserNameWithHost.cpp
ASTWindowDefinition.cpp
ASTWithAlias.cpp
ASTWithElement.cpp
CommonParsers.cpp
ExpressionElementParsers.cpp
ExpressionListParsers.cpp
IAST.cpp
IParserBase.cpp
InsertQuerySettingsPushDownVisitor.cpp
Lexer.cpp
MySQL/ASTAlterCommand.cpp
MySQL/ASTAlterQuery.cpp
MySQL/ASTCreateDefines.cpp
MySQL/ASTCreateQuery.cpp
MySQL/ASTDeclareColumn.cpp
MySQL/ASTDeclareConstraint.cpp
MySQL/ASTDeclareIndex.cpp
MySQL/ASTDeclareOption.cpp
MySQL/ASTDeclarePartition.cpp
MySQL/ASTDeclarePartitionOptions.cpp
MySQL/ASTDeclareReference.cpp
MySQL/ASTDeclareSubPartition.cpp
MySQL/ASTDeclareTableOptions.cpp
ParserAlterQuery.cpp
ParserBackupQuery.cpp
ParserCase.cpp
ParserCheckQuery.cpp
ParserCreateFunctionQuery.cpp
ParserCreateQuery.cpp
ParserCreateQuotaQuery.cpp
ParserCreateRoleQuery.cpp
ParserCreateRowPolicyQuery.cpp
ParserCreateSettingsProfileQuery.cpp
ParserCreateUserQuery.cpp
ParserDataType.cpp
ParserDatabaseOrNone.cpp
ParserDescribeTableQuery.cpp
ParserDictionary.cpp
ParserDictionaryAttributeDeclaration.cpp
ParserDropAccessEntityQuery.cpp
ParserDropFunctionQuery.cpp
ParserDropQuery.cpp
ParserExplainQuery.cpp
ParserExternalDDLQuery.cpp
ParserGrantQuery.cpp
ParserInsertQuery.cpp
ParserKillQueryQuery.cpp
ParserOptimizeQuery.cpp
ParserPartition.cpp
ParserProjectionSelectQuery.cpp
ParserQuery.cpp
ParserQueryWithOutput.cpp
ParserRenameQuery.cpp
ParserRolesOrUsersSet.cpp
ParserRowPolicyName.cpp
ParserSampleRatio.cpp
ParserSelectQuery.cpp
ParserSelectWithUnionQuery.cpp
ParserSetQuery.cpp
ParserSetRoleQuery.cpp
ParserSettingsProfileElement.cpp
ParserShowAccessEntitiesQuery.cpp
ParserShowCreateAccessEntityQuery.cpp
ParserShowGrantsQuery.cpp
ParserShowPrivilegesQuery.cpp
ParserShowTablesQuery.cpp
ParserSystemQuery.cpp
ParserTablePropertiesQuery.cpp
ParserTablesInSelectQuery.cpp
ParserUnionQueryElement.cpp
ParserUseQuery.cpp
ParserUserNameWithHost.cpp
ParserWatchQuery.cpp
ParserWithElement.cpp
QueryWithOutputSettingsPushDownVisitor.cpp
TokenIterator.cpp
formatAST.cpp
formatSettingName.cpp
getInsertQuery.cpp
iostream_debug_helpers.cpp
makeASTForLogicalFunction.cpp
obfuscateQueries.cpp
parseDatabaseAndTableName.cpp
parseIdentifierOrStringLiteral.cpp
parseIntervalKind.cpp
parseQuery.cpp
parseUserName.cpp
queryToString.cpp
)
END()

14
src/Parsers/ya.make.in
View File

@ -1,14 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
SRCS(
<? find . -name '*.cpp' | grep -v -F New | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | sed 's/^\.\// /' | sort ?>
)
END()

69
src/Processors/Transforms/WindowTransform.cpp
View File

@ -1658,68 +1658,6 @@ struct WindowFunctionLagLeadInFrame final : public WindowFunction
}
};
struct WindowFunctionNthValue final : public WindowFunction
{
WindowFunctionNthValue(const std::string & name_,
const DataTypes & argument_types_, const Array & parameters_)
: WindowFunction(name_, argument_types_, parameters_)
{
if (!parameters.empty())
{
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Function {} cannot be parameterized", name_);
}
if (argument_types.size() != 2)
{
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Function '{}' accepts 2 arguments, {} given",
name_, argument_types.size());
}
}
DataTypePtr getReturnType() const override
{ return argument_types[0]; }
bool allocatesMemoryInArena() const override { return false; }
void windowInsertResultInto(const WindowTransform * transform,
size_t function_index) override
{
const auto & current_block = transform->blockAt(transform->current_row);
IColumn & to = *(current_block.output_columns[function_index]);
const auto & workspace = transform->workspaces[function_index];
int64_t offset = (*current_block.input_columns[
workspace.argument_column_indices[1]])[
transform->current_row.row].get<Int64>();
/// Either overflow or really negative value, both is not acceptable.
if (offset <= 0)
{
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"The offset for function {} must be in (0, {}], {} given",
getName(), INT64_MAX, offset);
}
--offset;
const auto [target_row, offset_left] = transform->moveRowNumber(transform->frame_start, offset);
if (offset_left != 0
|| target_row < transform->frame_start
|| transform->frame_end <= target_row)
{
// Offset is outside the frame.
to.insertDefault();
}
else
{
// Offset is inside the frame.
to.insertFrom(*transform->blockAt(target_row).input_columns[
workspace.argument_column_indices[0]],
target_row.row);
}
}
};
void registerWindowFunctions(AggregateFunctionFactory & factory)
{
@ -1784,13 +1722,6 @@ void registerWindowFunctions(AggregateFunctionFactory & factory)
return std::make_shared<WindowFunctionLagLeadInFrame<true>>(
name, argument_types, parameters);
}, properties});
factory.registerFunction("nth_value", {[](const std::string & name,
const DataTypes & argument_types, const Array & parameters, const Settings *)
{
return std::make_shared<WindowFunctionNthValue>(
name, argument_types, parameters);
}, properties}, AggregateFunctionFactory::CaseInsensitive);
}
}

192
src/Processors/ya.make
View File

@ -1,192 +0,0 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
contrib/libs/msgpack
contrib/libs/protobuf
contrib/libs/apache/arrow
contrib/libs/apache/orc
)
ADDINCL(
contrib/libs/apache/arrow/src
contrib/libs/apache/orc/c++/include
)
CFLAGS(-DUSE_ARROW=1 -DUSE_PARQUET=1 -DUSE_ORC=1)
SRCS(
Chunk.cpp
ConcatProcessor.cpp
DelayedPortsProcessor.cpp
Executors/ExecutingGraph.cpp
Executors/PipelineExecutingBlockInputStream.cpp
Executors/PipelineExecutor.cpp
Executors/PollingQueue.cpp
Executors/PullingAsyncPipelineExecutor.cpp
Executors/PullingPipelineExecutor.cpp
Executors/StreamingFormatExecutor.cpp
ForkProcessor.cpp
Formats/IInputFormat.cpp
Formats/IOutputFormat.cpp
Formats/IRowInputFormat.cpp
Formats/IRowOutputFormat.cpp
Formats/Impl/ArrowBlockInputFormat.cpp
Formats/Impl/ArrowBlockOutputFormat.cpp
Formats/Impl/ArrowBufferedStreams.cpp
Formats/Impl/ArrowColumnToCHColumn.cpp
Formats/Impl/BinaryRowInputFormat.cpp
Formats/Impl/BinaryRowOutputFormat.cpp
Formats/Impl/CHColumnToArrowColumn.cpp
Formats/Impl/CSVRowInputFormat.cpp
Formats/Impl/CSVRowOutputFormat.cpp
Formats/Impl/ConstantExpressionTemplate.cpp
Formats/Impl/JSONAsStringRowInputFormat.cpp
Formats/Impl/JSONCompactEachRowRowInputFormat.cpp
Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp
Formats/Impl/JSONCompactRowOutputFormat.cpp
Formats/Impl/JSONEachRowRowInputFormat.cpp
Formats/Impl/JSONEachRowRowOutputFormat.cpp
Formats/Impl/JSONEachRowWithProgressRowOutputFormat.cpp
Formats/Impl/JSONRowOutputFormat.cpp
Formats/Impl/LineAsStringRowInputFormat.cpp
Formats/Impl/MarkdownRowOutputFormat.cpp
Formats/Impl/MsgPackRowInputFormat.cpp
Formats/Impl/MsgPackRowOutputFormat.cpp
Formats/Impl/MySQLOutputFormat.cpp
Formats/Impl/NullFormat.cpp
Formats/Impl/ODBCDriver2BlockOutputFormat.cpp
Formats/Impl/ParallelFormattingOutputFormat.cpp
Formats/Impl/ParallelParsingInputFormat.cpp
Formats/Impl/PostgreSQLOutputFormat.cpp
Formats/Impl/PrettyBlockOutputFormat.cpp
Formats/Impl/PrettyCompactBlockOutputFormat.cpp
Formats/Impl/PrettySpaceBlockOutputFormat.cpp
Formats/Impl/ProtobufRowInputFormat.cpp
Formats/Impl/ProtobufRowOutputFormat.cpp
Formats/Impl/RawBLOBRowInputFormat.cpp
Formats/Impl/RawBLOBRowOutputFormat.cpp
Formats/Impl/RegexpRowInputFormat.cpp
Formats/Impl/TSKVRowInputFormat.cpp
Formats/Impl/TSKVRowOutputFormat.cpp
Formats/Impl/TabSeparatedRowInputFormat.cpp
Formats/Impl/TabSeparatedRowOutputFormat.cpp
Formats/Impl/TemplateBlockOutputFormat.cpp
Formats/Impl/TemplateRowInputFormat.cpp
Formats/Impl/ValuesBlockInputFormat.cpp
Formats/Impl/ValuesRowOutputFormat.cpp
Formats/Impl/VerticalRowOutputFormat.cpp
Formats/Impl/XMLRowOutputFormat.cpp
Formats/LazyOutputFormat.cpp
Formats/OutputStreamToOutputFormat.cpp
Formats/PullingOutputFormat.cpp
Formats/RowInputFormatWithDiagnosticInfo.cpp
IAccumulatingTransform.cpp
IInflatingTransform.cpp
IProcessor.cpp
ISimpleTransform.cpp
ISink.cpp
ISource.cpp
LimitTransform.cpp
Merges/Algorithms/AggregatingSortedAlgorithm.cpp
Merges/Algorithms/CollapsingSortedAlgorithm.cpp
Merges/Algorithms/FinishAggregatingInOrderAlgorithm.cpp
Merges/Algorithms/GraphiteRollupSortedAlgorithm.cpp
Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.cpp
Merges/Algorithms/IMergingAlgorithmWithSharedChunks.cpp
Merges/Algorithms/MergingSortedAlgorithm.cpp
Merges/Algorithms/ReplacingSortedAlgorithm.cpp
Merges/Algorithms/SummingSortedAlgorithm.cpp
Merges/Algorithms/VersionedCollapsingAlgorithm.cpp
Merges/IMergingTransform.cpp
Merges/MergingSortedTransform.cpp
OffsetTransform.cpp
Pipe.cpp
Port.cpp
QueryPipeline.cpp
QueryPlan/AggregatingStep.cpp
QueryPlan/ArrayJoinStep.cpp
QueryPlan/BuildQueryPipelineSettings.cpp
QueryPlan/CreatingSetsStep.cpp
QueryPlan/CubeStep.cpp
QueryPlan/DistinctStep.cpp
QueryPlan/ExpressionStep.cpp
QueryPlan/ExtremesStep.cpp
QueryPlan/FillingStep.cpp
QueryPlan/FilterStep.cpp
QueryPlan/FinishSortingStep.cpp
QueryPlan/IQueryPlanStep.cpp
QueryPlan/ISourceStep.cpp
QueryPlan/ITransformingStep.cpp
QueryPlan/IntersectOrExceptStep.cpp
QueryPlan/JoinStep.cpp
QueryPlan/LimitByStep.cpp
QueryPlan/LimitStep.cpp
QueryPlan/MergeSortingStep.cpp
QueryPlan/MergingAggregatedStep.cpp
QueryPlan/MergingSortedStep.cpp
QueryPlan/OffsetStep.cpp
QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp
QueryPlan/Optimizations/filterPushDown.cpp
QueryPlan/Optimizations/liftUpArrayJoin.cpp
QueryPlan/Optimizations/limitPushDown.cpp
QueryPlan/Optimizations/mergeExpressions.cpp
QueryPlan/Optimizations/optimizeTree.cpp
QueryPlan/Optimizations/splitFilter.cpp
QueryPlan/PartialSortingStep.cpp
QueryPlan/QueryIdHolder.cpp
QueryPlan/QueryPlan.cpp
QueryPlan/ReadFromMergeTree.cpp
QueryPlan/ReadFromPreparedSource.cpp
QueryPlan/ReadFromRemote.cpp
QueryPlan/ReadNothingStep.cpp
QueryPlan/RollupStep.cpp
QueryPlan/SettingQuotaAndLimitsStep.cpp
QueryPlan/TotalsHavingStep.cpp
QueryPlan/UnionStep.cpp
QueryPlan/WindowStep.cpp
ResizeProcessor.cpp
Sources/DelayedSource.cpp
Sources/RemoteSource.cpp
Sources/SinkToOutputStream.cpp
Sources/SourceFromInputStream.cpp
Sources/SourceWithProgress.cpp
Transforms/AddingDefaultsTransform.cpp
Transforms/AddingSelectorTransform.cpp
Transforms/AggregatingInOrderTransform.cpp
Transforms/AggregatingTransform.cpp
Transforms/ArrayJoinTransform.cpp
Transforms/CheckSortedTransform.cpp
Transforms/CopyTransform.cpp
Transforms/CreatingSetsTransform.cpp
Transforms/CubeTransform.cpp
Transforms/DistinctTransform.cpp
Transforms/ExpressionTransform.cpp
Transforms/ExtremesTransform.cpp
Transforms/FillingTransform.cpp
Transforms/FilterTransform.cpp
Transforms/FinishSortingTransform.cpp
Transforms/IntersectOrExceptTransform.cpp
Transforms/JoiningTransform.cpp
Transforms/LimitByTransform.cpp
Transforms/LimitsCheckingTransform.cpp
Transforms/MaterializingTransform.cpp
Transforms/MergeSortingTransform.cpp
Transforms/MergingAggregatedMemoryEfficientTransform.cpp
Transforms/MergingAggregatedTransform.cpp
Transforms/PartialSortingTransform.cpp
Transforms/ReverseTransform.cpp
Transforms/RollupTransform.cpp
Transforms/SortingTransform.cpp
Transforms/SquashingChunksTransform.cpp
Transforms/TotalsHavingTransform.cpp
Transforms/WindowTransform.cpp
Transforms/getSourceFromASTInsertQuery.cpp
printPipeline.cpp
)
END()

24
src/Processors/ya.make.in
View File

@ -1,24 +0,0 @@
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
contrib/libs/msgpack
contrib/libs/protobuf
contrib/libs/apache/arrow
contrib/libs/apache/orc
)
ADDINCL(
contrib/libs/apache/arrow/src
contrib/libs/apache/orc/c++/include
)
CFLAGS(-DUSE_ARROW=1 -DUSE_PARQUET=1 -DUSE_ORC=1)
SRCS(
<? find . -name '*.cpp' | grep -v -F tests | grep -v -F examples | grep -v -F fuzzers | grep -v -P 'Avro|ORC|Parquet|CapnProto' | sed 's/^\.\// /' | sort ?>
)
END()

Some files were not shown because too many files have changed in this diff Show More