diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c93accf1009..257040c68b7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,4 @@ dbms/* @ClickHouse/core-assigner +utils/* @ClickHouse/core-assigner docs/* @ClickHouse/docs docs/zh/* @ClickHouse/docs-zh diff --git a/.github/labeler.keywords.yml b/.github/labeler.keywords.yml deleted file mode 100644 index c70ea45de53..00000000000 --- a/.github/labeler.keywords.yml +++ /dev/null @@ -1 +0,0 @@ -pr-feature: "New Feature" diff --git a/.github/labeler.yml b/.github/labeler.yml deleted file mode 100644 index 72cf714f039..00000000000 --- a/.github/labeler.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Documentation PRs -documentation: - - "**/*.md" - - "docs/**/*" -pr-documentation: - - "**/*.md" - - "docs/**/*" - -# Component labels -comp-mutations: - - "**/*Mutation*" -comp-matview: - - "**/*MaterializedView*" -comp-skipidx: - - "**/*Indices*" -comp-kafka: - - "dbms/src/Storages/Kafka/**/*" - - "dbms/tests/integration/test_storage_kafka/**/*" - - "utils/kafka/**/*" diff --git a/.github/stale.yml b/.github/stale.yml deleted file mode 100644 index 6b18b043853..00000000000 --- a/.github/stale.yml +++ /dev/null @@ -1,67 +0,0 @@ -# Configuration for probot-stale - https://github.com/probot/stale - -# Number of days of inactivity before an Issue or Pull Request becomes stale -daysUntilStale: 45 - -# Number of days of inactivity before an Issue or Pull Request with the stale label is closed. -# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale. -daysUntilClose: 30 - -# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled) -onlyLabels: [] - -# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable -exemptLabels: - - bug - - feature - - memory - - performance - - prio-crit - - prio-major - - st-accepted - - st-in-progress - - st-waiting-for-fix - -# Set to true to ignore issues in a project (defaults to false) -exemptProjects: false - -# Set to true to ignore issues in a milestone (defaults to false) -exemptMilestones: false - -# Set to true to ignore issues with an assignee (defaults to false) -exemptAssignees: false - -# Label to use when marking as stale -staleLabel: stale - -# Comment to post when marking as stale. Set to `false` to disable -markComment: > - This issue has been automatically marked as stale because it has not had - recent activity. It will be closed if no further activity occurs. Thank you - for your contributions. - -# Comment to post when removing the stale label. -# unmarkComment: > -# Your comment here. - -# Comment to post when closing a stale Issue or Pull Request. -# closeComment: > -# Your comment here. - -# Limit the number of actions per hour, from 1-30. Default is 30 -limitPerRun: 30 - -# Limit to only `issues` or `pulls` -# only: issues - -# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls': -pulls: - daysUntilStale: 365 - markComment: > - This pull request has been automatically marked as stale because it has not had - any activity for over a year. It will be closed if no further activity occurs. Thank you - for your contributions. - -# issues: -# exemptLabels: -# - confirmed diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml deleted file mode 100644 index 0110ef7b516..00000000000 --- a/.github/workflows/labeler.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: "Pull Request Labeler" -on: - pull_request - -jobs: - by-filename: - runs-on: ubuntu-latest - steps: - - uses: "actions/labeler@v2" - with: - repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.gitignore b/.gitignore index 817e333d833..4ec07227e66 100644 --- a/.gitignore +++ b/.gitignore @@ -245,3 +245,6 @@ website/package-lock.json /.ccls-cache /compile_commands.json + +# Toolchains +/cmake/toolchain/* diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a8c32b588a..272b0d298d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,253 @@ +## ClickHouse release v19.16.2.2, 30-10-2019 + +### Backward Incompatible Change +* Add missing arity validation for count/counIf. + [#7095](https://github.com/ClickHouse/ClickHouse/issues/7095) +[#7298](https://github.com/ClickHouse/ClickHouse/pull/7298) ([Vdimir](https://github.com/Vdimir)) +* Remove legacy `asterisk_left_columns_only` setting (it was disabled by default). + [#7335](https://github.com/ClickHouse/ClickHouse/pull/7335) ([Artem +Zuikov](https://github.com/4ertus2)) +* Format strings for Template data format are now specified in files. + [#7118](https://github.com/ClickHouse/ClickHouse/pull/7118) +([tavplubix](https://github.com/tavplubix)) + +### New Feature +* Introduce uniqCombined64() to calculate cardinality greater than UINT_MAX. + [#7213](https://github.com/ClickHouse/ClickHouse/pull/7213), +[#7222](https://github.com/ClickHouse/ClickHouse/pull/7222) ([Azat +Khuzhin](https://github.com/azat)) +* Support Bloom filter indexes on Array columns. + [#6984](https://github.com/ClickHouse/ClickHouse/pull/6984) +([achimbab](https://github.com/achimbab)) +* Add a function `getMacro(name)` that returns String with the value of corresponding `` + from server configuration. [#7240](https://github.com/ClickHouse/ClickHouse/pull/7240) +([alexey-milovidov](https://github.com/alexey-milovidov)) +* Set two configuration options for a dictionary based on an HTTP source: `credentials` and + `http-headers`. [#7092](https://github.com/ClickHouse/ClickHouse/pull/7092) ([Guillaume +Tassery](https://github.com/YiuRULE)) +* Add a new ProfileEvent `Merge` that counts the number of launched background merges. + [#7093](https://github.com/ClickHouse/ClickHouse/pull/7093) ([Mikhail +Korotov](https://github.com/millb)) +* Add fullHostName function that returns a fully qualified domain name. + [#7263](https://github.com/ClickHouse/ClickHouse/issues/7263) +[#7291](https://github.com/ClickHouse/ClickHouse/pull/7291) ([sundyli](https://github.com/sundy-li)) +* Add function `arraySplit` and `arrayReverseSplit` which split an array by "cut off" + conditions. They are useful in time sequence handling. +[#7294](https://github.com/ClickHouse/ClickHouse/pull/7294) ([hcz](https://github.com/hczhcz)) +* Add new functions that return the Array of all matched indices in multiMatch family of functions. + [#7299](https://github.com/ClickHouse/ClickHouse/pull/7299) ([Danila +Kutenin](https://github.com/danlark1)) +* Add a new database engine `Lazy` that is optimized for storing a large number of small -Log + tables. [#7171](https://github.com/ClickHouse/ClickHouse/pull/7171) ([Nikita +Vasilev](https://github.com/nikvas0)) +* Add aggregate functions groupBitmapAnd, -Or, -Xor for bitmap columns. [#7109](https://github.com/ClickHouse/ClickHouse/pull/7109) ([Zhichang +Yu](https://github.com/yuzhichang)) +* Add aggregate function combinators -OrNull and -OrDefault, which return null + or default values when there is nothing to aggregate. +[#7331](https://github.com/ClickHouse/ClickHouse/pull/7331) +([hcz](https://github.com/hczhcz)) +* Introduce CustomSeparated data format that supports custom escaping and + delimiter rules. [#7118](https://github.com/ClickHouse/ClickHouse/pull/7118) +([tavplubix](https://github.com/tavplubix)) + +### Bug Fix +* Fix wrong query result if it has `WHERE IN (SELECT ...)` section and `optimize_read_in_order` is + used. [#7371](https://github.com/ClickHouse/ClickHouse/pull/7371) ([Anton +Popov](https://github.com/CurtizJ)) +* Disabled MariaDB authentication plugin, which depends on files outside of project. + [#7140](https://github.com/ClickHouse/ClickHouse/pull/7140) ([Yuriy +Baranov](https://github.com/yurriy)) +* Fix exception `Cannot convert column ... because it is constant but values of constants are + different in source and result` which could rarely happen when functions `now()`, `today()`, +`yesterday()`, `randConstant()` are used. +[#7156](https://github.com/ClickHouse/ClickHouse/pull/7156) ([Nikolai +Kochetov](https://github.com/KochetovNicolai)) +* Fixed issue of using HTTP keep alive timeout instead of TCP keep alive timeout. + [#7351](https://github.com/ClickHouse/ClickHouse/pull/7351) ([Vasily +Nemkov](https://github.com/Enmk)) +* Fixed a segmentation fault in groupBitmapOr (issue [#7109](https://github.com/ClickHouse/ClickHouse/issues/7109)). + [#7289](https://github.com/ClickHouse/ClickHouse/pull/7289) ([Zhichang +Yu](https://github.com/yuzhichang)) +* For materialized views the commit for Kafka is called after all data were written. + [#7175](https://github.com/ClickHouse/ClickHouse/pull/7175) ([Ivan](https://github.com/abyss7)) +* Fixed wrong `duration_ms` value in `system.part_log` table. It was ten times off. + [#7172](https://github.com/ClickHouse/ClickHouse/pull/7172) ([Vladimir +Chebotarev](https://github.com/excitoon)) +* A quick fix to resolve crash in LIVE VIEW table and re-enabling all LIVE VIEW tests. + [#7201](https://github.com/ClickHouse/ClickHouse/pull/7201) +([vzakaznikov](https://github.com/vzakaznikov)) +* Serialize NULL values correctly in min/max indexes of MergeTree parts. + [#7234](https://github.com/ClickHouse/ClickHouse/pull/7234) ([Alexander +Kuzmenkov](https://github.com/akuzm)) +* Don't put virtual columns to .sql metadata when table is created as `CREATE TABLE AS`. + [#7183](https://github.com/ClickHouse/ClickHouse/pull/7183) ([Ivan](https://github.com/abyss7)) +* Fix segmentation fault in `ATTACH PART` query. + [#7185](https://github.com/ClickHouse/ClickHouse/pull/7185) +([alesapin](https://github.com/alesapin)) +* Fix wrong result for some queries given by the optimization of empty IN subqueries and empty + INNER/RIGHT JOIN. [#7284](https://github.com/ClickHouse/ClickHouse/pull/7284) ([Nikolai +Kochetov](https://github.com/KochetovNicolai)) +* Fixing AddressSanitizer error in the LIVE VIEW getHeader() method. + [#7271](https://github.com/ClickHouse/ClickHouse/pull/7271) +([vzakaznikov](https://github.com/vzakaznikov)) + +### Improvement +* Add a message in case of queue_wait_max_ms wait takes place. + [#7390](https://github.com/ClickHouse/ClickHouse/pull/7390) ([Azat +Khuzhin](https://github.com/azat)) +* Made setting `s3_min_upload_part_size` table-level. + [#7059](https://github.com/ClickHouse/ClickHouse/pull/7059) ([Vladimir +Chebotarev](https://github.com/excitoon)) +* Check TTL in StorageFactory. [#7304](https://github.com/ClickHouse/ClickHouse/pull/7304) + ([sundyli](https://github.com/sundy-li)) +* Squash left-hand blocks in partial merge join (optimization). + [#7122](https://github.com/ClickHouse/ClickHouse/pull/7122) ([Artem +Zuikov](https://github.com/4ertus2)) +* Do not allow non-deterministic functions in mutations of Replicated table engines, because this + can introduce inconsistencies between replicas. +[#7247](https://github.com/ClickHouse/ClickHouse/pull/7247) ([Alexander +Kazakov](https://github.com/Akazz)) +* Disable memory tracker while converting exception stack trace to string. It can prevent the loss + of error messages of type `Memory limit exceeded` on server, which caused the `Attempt to read +after eof` exception on client. [#7264](https://github.com/ClickHouse/ClickHouse/pull/7264) +([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Miscellaneous format improvements. Resolves + [#6033](https://github.com/ClickHouse/ClickHouse/issues/6033), +[#2633](https://github.com/ClickHouse/ClickHouse/issues/2633), +[#6611](https://github.com/ClickHouse/ClickHouse/issues/6611), +[#6742](https://github.com/ClickHouse/ClickHouse/issues/6742) +[#7215](https://github.com/ClickHouse/ClickHouse/pull/7215) +([tavplubix](https://github.com/tavplubix)) +* ClickHouse ignores values on the right side of IN operator that are not convertible to the left + side type. Make it work properly for compound types -- Array and Tuple. +[#7283](https://github.com/ClickHouse/ClickHouse/pull/7283) ([Alexander +Kuzmenkov](https://github.com/akuzm)) +* Support missing inequalities for ASOF JOIN. It's possible to join less-or-equal variant and strict + greater and less variants for ASOF column in ON syntax. +[#7282](https://github.com/ClickHouse/ClickHouse/pull/7282) ([Artem +Zuikov](https://github.com/4ertus2)) +* Optimize partial merge join. [#7070](https://github.com/ClickHouse/ClickHouse/pull/7070) + ([Artem Zuikov](https://github.com/4ertus2)) +* Do not use more then 98K of memory in uniqCombined functions. + [#7236](https://github.com/ClickHouse/ClickHouse/pull/7236), +[#7270](https://github.com/ClickHouse/ClickHouse/pull/7270) ([Azat +Khuzhin](https://github.com/azat)) +* Flush parts of right-hand joining table on disk in PartialMergeJoin (if there is not enough + memory). Load data back when needed. [#7186](https://github.com/ClickHouse/ClickHouse/pull/7186) +([Artem Zuikov](https://github.com/4ertus2)) + +### Performance Improvement +* Speed up joinGet with const arguments by avoiding data duplication. + [#7359](https://github.com/ClickHouse/ClickHouse/pull/7359) ([Amos +Bird](https://github.com/amosbird)) +* Return early if the subquery is empty. + [#7007](https://github.com/ClickHouse/ClickHouse/pull/7007) ([小路](https://github.com/nicelulu)) +* Optimize parsing of SQL expression in Values. + [#6781](https://github.com/ClickHouse/ClickHouse/pull/6781) +([tavplubix](https://github.com/tavplubix)) + +### Build/Testing/Packaging Improvement +* Disable some contribs for cross-compilation to Mac OS. + [#7101](https://github.com/ClickHouse/ClickHouse/pull/7101) ([Ivan](https://github.com/abyss7)) +* Add missing linking with PocoXML for clickhouse_common_io. + [#7200](https://github.com/ClickHouse/ClickHouse/pull/7200) ([Azat +Khuzhin](https://github.com/azat)) +* Accept multiple test filter arguments in clickhouse-test. + [#7226](https://github.com/ClickHouse/ClickHouse/pull/7226) ([Alexander +Kuzmenkov](https://github.com/akuzm)) +* Enable musl and jemalloc for ARM. [#7300](https://github.com/ClickHouse/ClickHouse/pull/7300) + ([Amos Bird](https://github.com/amosbird)) +* Added `--client-option` parameter to `clickhouse-test` to pass additional parameters to client. + [#7277](https://github.com/ClickHouse/ClickHouse/pull/7277) ([Nikolai +Kochetov](https://github.com/KochetovNicolai)) +* Preserve existing configs on rpm package upgrade. + [#7103](https://github.com/ClickHouse/ClickHouse/pull/7103) +([filimonov](https://github.com/filimonov)) +* Fix errors detected by PVS. [#7153](https://github.com/ClickHouse/ClickHouse/pull/7153) ([Artem + Zuikov](https://github.com/4ertus2)) +* Fix build for Darwin. [#7149](https://github.com/ClickHouse/ClickHouse/pull/7149) + ([Ivan](https://github.com/abyss7)) +* glibc 2.29 compatibility. [#7142](https://github.com/ClickHouse/ClickHouse/pull/7142) ([Amos + Bird](https://github.com/amosbird)) +* Make sure dh_clean does not touch potential source files. + [#7205](https://github.com/ClickHouse/ClickHouse/pull/7205) ([Amos +Bird](https://github.com/amosbird)) +* Attempt to avoid conflict when updating from altinity rpm - it has config file packaged separately + in clickhouse-server-common. [#7073](https://github.com/ClickHouse/ClickHouse/pull/7073) +([filimonov](https://github.com/filimonov)) +* Optimize some header files for faster rebuilds. + [#7212](https://github.com/ClickHouse/ClickHouse/pull/7212), +[#7231](https://github.com/ClickHouse/ClickHouse/pull/7231) ([Alexander +Kuzmenkov](https://github.com/akuzm)) +* Add performance tests for Date and DateTime. [#7332](https://github.com/ClickHouse/ClickHouse/pull/7332) ([Vasily + Nemkov](https://github.com/Enmk)) +* Fix some tests that contained non-deterministic mutations. + [#7132](https://github.com/ClickHouse/ClickHouse/pull/7132) ([Alexander +Kazakov](https://github.com/Akazz)) +* Add build with MemorySanitizer to CI. [#7066](https://github.com/ClickHouse/ClickHouse/pull/7066) + ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Avoid use of uninitialized values in MetricsTransmitter. + [#7158](https://github.com/ClickHouse/ClickHouse/pull/7158) ([Azat +Khuzhin](https://github.com/azat)) +* Fix some issues in Fields found by MemorySanitizer. + [#7135](https://github.com/ClickHouse/ClickHouse/pull/7135), +[#7179](https://github.com/ClickHouse/ClickHouse/pull/7179) ([Alexander +Kuzmenkov](https://github.com/akuzm)), [#7376](https://github.com/ClickHouse/ClickHouse/pull/7376) +([Amos Bird](https://github.com/amosbird)) +* Fix undefined behavior in murmurhash32. [#7388](https://github.com/ClickHouse/ClickHouse/pull/7388) ([Amos + Bird](https://github.com/amosbird)) +* Fix undefined behavior in StoragesInfoStream. [#7384](https://github.com/ClickHouse/ClickHouse/pull/7384) + ([tavplubix](https://github.com/tavplubix)) +* Fixed constant expressions folding for external database engines (MySQL, ODBC, JDBC). In previous + versions it wasn't working for multiple constant expressions and was not working at all for Date, +DateTime and UUID. This fixes [#7245](https://github.com/ClickHouse/ClickHouse/issues/7245) +[#7252](https://github.com/ClickHouse/ClickHouse/pull/7252) +([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixing ThreadSanitizer data race error in the LIVE VIEW when accessing no_users_thread variable. + [#7353](https://github.com/ClickHouse/ClickHouse/pull/7353) +([vzakaznikov](https://github.com/vzakaznikov)) +* Get rid of malloc symbols in libcommon + [#7134](https://github.com/ClickHouse/ClickHouse/pull/7134), +[#7065](https://github.com/ClickHouse/ClickHouse/pull/7065) ([Amos +Bird](https://github.com/amosbird)) +* Add global flag ENABLE_LIBRARIES for disabling all libraries. + [#7063](https://github.com/ClickHouse/ClickHouse/pull/7063) +([proller](https://github.com/proller)) + +### Code cleanup +* Generalize configuration repository to prepare for DDL for Dictionaries. [#7155](https://github.com/ClickHouse/ClickHouse/pull/7155) + ([alesapin](https://github.com/alesapin)) +* Parser for dictionaries DDL without any semantic. + [#7209](https://github.com/ClickHouse/ClickHouse/pull/7209) +([alesapin](https://github.com/alesapin)) +* Split ParserCreateQuery into different smaller parsers. + [#7253](https://github.com/ClickHouse/ClickHouse/pull/7253) +([alesapin](https://github.com/alesapin)) +* Small refactoring and renaming near external dictionaries. + [#7111](https://github.com/ClickHouse/ClickHouse/pull/7111) +([alesapin](https://github.com/alesapin)) +* Refactor some code to prepare for role-based access control. [#7235](https://github.com/ClickHouse/ClickHouse/pull/7235) ([Vitaly + Baranov](https://github.com/vitlibar)) +* Some improvements in DatabaseOrdinary code. + [#7086](https://github.com/ClickHouse/ClickHouse/pull/7086) ([Nikita +Vasilev](https://github.com/nikvas0)) +* Do not use iterators in find() and emplace() methods of hash tables. +[#7026](https://github.com/ClickHouse/ClickHouse/pull/7026) ([Alexander +Kuzmenkov](https://github.com/akuzm)) +* Fix getMultipleValuesFromConfig in case when parameter root is not empty. [#7374](https://github.com/ClickHouse/ClickHouse/pull/7374) +([Mikhail Korotov](https://github.com/millb)) +* Remove some copy-paste (TemporaryFile and TemporaryFileStream) + [#7166](https://github.com/ClickHouse/ClickHouse/pull/7166) ([Artem +Zuikov](https://github.com/4ertus2)) +* Improved code readability a little bit (`MergeTreeData::getActiveContainingPart`). + [#7361](https://github.com/ClickHouse/ClickHouse/pull/7361) ([Vladimir +Chebotarev](https://github.com/excitoon)) +* Wait for all scheduled jobs, which are using local objects, if `ThreadPool::schedule(...)` throws + an exception. Rename `ThreadPool::schedule(...)` to `ThreadPool::scheduleOrThrowOnError(...)` and +fix comments to make obvious that it may throw. +[#7350](https://github.com/ClickHouse/ClickHouse/pull/7350) +([tavplubix](https://github.com/tavplubix)) + ## ClickHouse release 19.15.2.2, 2019-10-01 ### New Feature @@ -345,6 +595,13 @@ ### Security Fix * Fix two vulnerabilities in codecs in decompression phase (malicious user can fabricate compressed data that will lead to buffer overflow in decompression). [#6670](https://github.com/ClickHouse/ClickHouse/pull/6670) ([Artem Zuikov](https://github.com/4ertus2)) +## ClickHouse release 19.11.13.74, 2019-11-01 + +### Bug Fix +* Fixed rare crash in `ALTER MODIFY COLUMN` and vertical merge when one of merged/altered parts is empty (0 rows). [#6780](https://github.com/ClickHouse/ClickHouse/pull/6780) ([alesapin](https://github.com/alesapin)) +* Manual update of `SIMDJSON`. This fixes possible flooding of stderr files with bogus json diagnostic messages. [#7548](https://github.com/ClickHouse/ClickHouse/pull/7548) ([Alexander Kazakov](https://github.com/Akazz)) +* Fixed bug with `mrk` file extension for mutations ([alesapin](https://github.com/alesapin)) + ## ClickHouse release 19.11.12.69, 2019-10-02 ### Bug Fix diff --git a/CMakeLists.txt b/CMakeLists.txt index 53021dbe666..986096ba9e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,10 @@ foreach(policy endforeach() project(ClickHouse) + +include (cmake/arch.cmake) include (cmake/target.cmake) +include (cmake/tools.cmake) # Ignore export() since we don't use it, # but it gets broken with a global targets via link_libraries() @@ -26,8 +29,6 @@ set(CMAKE_LINK_DEPENDS_NO_SHARED 1) # Do not relink all depended targets on .so set(CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Debug;Release;MinSizeRel" CACHE STRING "" FORCE) set(CMAKE_DEBUG_POSTFIX "d" CACHE STRING "Generate debug library name with a postfix.") # To be consistent with CMakeLists from contrib libs. -include (cmake/arch.cmake) - option(ENABLE_IPO "Enable inter-procedural optimization (aka LTO)" OFF) # need cmake 3.9+ if(ENABLE_IPO) cmake_policy(SET CMP0069 NEW) @@ -230,7 +231,6 @@ include(cmake/dbms_glob_sources.cmake) if (OS_LINUX) include(cmake/linux/default_libs.cmake) elseif (OS_DARWIN) - include(cmake/darwin/sdk.cmake) include(cmake/darwin/default_libs.cmake) endif () diff --git a/README.md b/README.md index 83cf3e9adbc..08be0b9ed07 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,9 @@ ClickHouse is an open-source column-oriented database management system that all * You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person. ## Upcoming Events -* [ClickHouse Meetup in Shanghai](https://www.huodongxing.com/event/4483760336000) on October 27. * [ClickHouse Meetup in Tokyo](https://clickhouse.connpass.com/event/147001/) on November 14. * [ClickHouse Meetup in Istanbul](https://www.eventbrite.com/e/clickhouse-meetup-istanbul-create-blazing-fast-experiences-w-clickhouse-tickets-73101120419) on November 19. * [ClickHouse Meetup in Ankara](https://www.eventbrite.com/e/clickhouse-meetup-ankara-create-blazing-fast-experiences-w-clickhouse-tickets-73100530655) on November 21. * [ClickHouse Meetup in Singapore](https://www.meetup.com/Singapore-Clickhouse-Meetup-Group/events/265085331/) on November 23. +* [ClickHouse Meetup in San Francisco](https://www.eventbrite.com/e/clickhouse-december-meetup-registration-78642047481) on December 3. + diff --git a/cmake/arch.cmake b/cmake/arch.cmake index f339236af64..79fe92c03e5 100644 --- a/cmake/arch.cmake +++ b/cmake/arch.cmake @@ -17,6 +17,7 @@ endif () if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64le.*|PPC64LE.*)") set (ARCH_PPC64LE 1) + # FIXME: move this check into tools.cmake if (COMPILER_CLANG OR (COMPILER_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8)) message(FATAL_ERROR "Only gcc-8 is supported for powerpc architecture") endif () diff --git a/cmake/darwin/default_libs.cmake b/cmake/darwin/default_libs.cmake index e684a10a08f..679fef8808a 100644 --- a/cmake/darwin/default_libs.cmake +++ b/cmake/darwin/default_libs.cmake @@ -11,6 +11,14 @@ message(STATUS "Default libraries: ${DEFAULT_LIBS}") set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS}) set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS}) +# Minimal supported SDK version + +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mmacosx-version-min=10.14") +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mmacosx-version-min=10.14") + +set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -mmacosx-version-min=10.14") +set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -mmacosx-version-min=10.14") + # Global libraries add_library(global-libs INTERFACE) diff --git a/cmake/darwin/sdk.cmake b/cmake/darwin/sdk.cmake deleted file mode 100644 index 92a9f8f66e5..00000000000 --- a/cmake/darwin/sdk.cmake +++ /dev/null @@ -1,11 +0,0 @@ -option (SDK_PATH "Path to the SDK to build with" "") - -if (NOT EXISTS "${SDK_PATH}/SDKSettings.plist") - message (FATAL_ERROR "Wrong SDK path provided: ${SDK_PATH}") -endif () - -set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -isysroot ${SDK_PATH} -mmacosx-version-min=10.14") -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isysroot ${SDK_PATH} -mmacosx-version-min=10.14") - -set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -isysroot ${SDK_PATH} -mmacosx-version-min=10.14") -set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -isysroot ${SDK_PATH} -mmacosx-version-min=10.14") diff --git a/cmake/darwin/toolchain-x86_64.cmake b/cmake/darwin/toolchain-x86_64.cmake new file mode 100644 index 00000000000..9128311e3bb --- /dev/null +++ b/cmake/darwin/toolchain-x86_64.cmake @@ -0,0 +1,13 @@ +set (CMAKE_SYSTEM_NAME "Darwin") +set (CMAKE_SYSTEM_PROCESSOR "x86_64") +set (CMAKE_C_COMPILER_TARGET "x86_64-apple-darwin") +set (CMAKE_CXX_COMPILER_TARGET "x86_64-apple-darwin") +set (CMAKE_OSX_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/darwin-x86_64") + +set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake + +set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) +set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) + +set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) +set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/find/capnp.cmake b/cmake/find/capnp.cmake index 1ac6241a5f1..0620a66808b 100644 --- a/cmake/find/capnp.cmake +++ b/cmake/find/capnp.cmake @@ -4,6 +4,14 @@ if (ENABLE_CAPNP) option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library instead of bundled" ${NOT_UNBUNDLED}) +if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/capnproto/CMakeLists.txt") + if(USE_INTERNAL_CAPNP_LIBRARY) + message(WARNING "submodule contrib/capnproto is missing. to fix try run: \n git submodule update --init --recursive") + endif() + set(MISSING_INTERNAL_CAPNP_LIBRARY 1) + set(USE_INTERNAL_CAPNP_LIBRARY 0) +endif() + # FIXME: refactor to use `add_library(… IMPORTED)` if possible. if (NOT USE_INTERNAL_CAPNP_LIBRARY) find_library (KJ kj) @@ -11,7 +19,7 @@ if (NOT USE_INTERNAL_CAPNP_LIBRARY) find_library (CAPNPC capnpc) set (CAPNP_LIBRARIES ${CAPNPC} ${CAPNP} ${KJ}) -else () +elseif(NOT MISSING_INTERNAL_CAPNP_LIBRARY) add_subdirectory(contrib/capnproto-cmake) set (CAPNP_LIBRARIES capnpc) @@ -23,4 +31,4 @@ endif () endif () -message (STATUS "Using capnp: ${CAPNP_LIBRARIES}") +message (STATUS "Using capnp=${USE_CAPNP}: ${CAPNP_LIBRARIES}") diff --git a/cmake/find/orc.cmake b/cmake/find/orc.cmake index 613016c6ba7..26253687c80 100644 --- a/cmake/find/orc.cmake +++ b/cmake/find/orc.cmake @@ -1,7 +1,8 @@ option (ENABLE_ORC "Enable ORC" ${ENABLE_LIBRARIES}) if(ENABLE_ORC) -option (USE_INTERNAL_ORC_LIBRARY "Set to FALSE to use system ORC instead of bundled" ${NOT_UNBUNDLED}) +include(cmake/find/snappy.cmake) +option(USE_INTERNAL_ORC_LIBRARY "Set to FALSE to use system ORC instead of bundled" ${NOT_UNBUNDLED}) if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include/orc/OrcFile.hh") if(USE_INTERNAL_ORC_LIBRARY) @@ -25,7 +26,7 @@ endif () if (ORC_LIBRARY AND ORC_INCLUDE_DIR) set(USE_ORC 1) -elseif(NOT MISSING_INTERNAL_ORC_LIBRARY AND ARROW_LIBRARY) # (LIBGSASL_LIBRARY AND LIBXML2_LIBRARY) +elseif(NOT MISSING_INTERNAL_ORC_LIBRARY AND ARROW_LIBRARY AND SNAPPY_LIBRARY) # (LIBGSASL_LIBRARY AND LIBXML2_LIBRARY) set(ORC_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include") set(ORC_LIBRARY orc) set(USE_ORC 1) diff --git a/cmake/find/parquet.cmake b/cmake/find/parquet.cmake index 14332fc9caf..d302fd64e1e 100644 --- a/cmake/find/parquet.cmake +++ b/cmake/find/parquet.cmake @@ -24,7 +24,10 @@ endif() if(ARROW_INCLUDE_DIR AND PARQUET_INCLUDE_DIR) elseif(NOT MISSING_INTERNAL_PARQUET_LIBRARY AND NOT OS_FREEBSD) include(cmake/find/snappy.cmake) - set(CAN_USE_INTERNAL_PARQUET_LIBRARY 1) + if(SNAPPY_LIBRARY) + set(CAN_USE_INTERNAL_PARQUET_LIBRARY 1) + endif() + include(CheckCXXSourceCompiles) if(NOT USE_INTERNAL_DOUBLE_CONVERSION_LIBRARY) set(CMAKE_REQUIRED_LIBRARIES ${DOUBLE_CONVERSION_LIBRARIES}) diff --git a/cmake/find/poco.cmake b/cmake/find/poco.cmake index ee2f5d9df1f..b44d2932276 100644 --- a/cmake/find/poco.cmake +++ b/cmake/find/poco.cmake @@ -8,6 +8,14 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/poco/CMakeLists.txt") set (MISSING_INTERNAL_POCO_LIBRARY 1) endif () +if (NOT ENABLE_LIBRARIES) + set (ENABLE_POCO_NETSSL ${ENABLE_LIBRARIES} CACHE BOOL "") + set (ENABLE_POCO_MONGODB ${ENABLE_LIBRARIES} CACHE BOOL "") + set (ENABLE_POCO_REDIS ${ENABLE_LIBRARIES} CACHE BOOL "") + set (ENABLE_POCO_ODBC ${ENABLE_LIBRARIES} CACHE BOOL "") + set (ENABLE_POCO_SQL ${ENABLE_LIBRARIES} CACHE BOOL "") +endif () + set (POCO_COMPONENTS Net XML SQL Data) if (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL) list (APPEND POCO_COMPONENTS Crypto NetSSL) diff --git a/cmake/find/unwind.cmake b/cmake/find/unwind.cmake index efa7618a53b..d3653973082 100644 --- a/cmake/find/unwind.cmake +++ b/cmake/find/unwind.cmake @@ -4,6 +4,11 @@ if (NOT CMAKE_SYSTEM MATCHES "Linux" OR ARCH_ARM OR ARCH_32) set (USE_UNWIND OFF) endif () +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libunwind/CMakeLists.txt") + message(WARNING "submodule contrib/libunwind is missing. to fix try run: \n git submodule update --init --recursive") + set (USE_UNWIND OFF) +endif () + if (USE_UNWIND) add_subdirectory(contrib/libunwind-cmake) set (UNWIND_LIBRARIES unwind) diff --git a/cmake/linux/default_libs.cmake b/cmake/linux/default_libs.cmake index d1f4730c97e..ef1354628fe 100644 --- a/cmake/linux/default_libs.cmake +++ b/cmake/linux/default_libs.cmake @@ -5,7 +5,7 @@ set (DEFAULT_LIBS "-nodefaultlibs") # We need builtins from Clang's RT even without libcxx - for ubsan+int128. # See https://bugs.llvm.org/show_bug.cgi?id=16404 -if (COMPILER_CLANG) +if (COMPILER_CLANG AND NOT (CMAKE_CROSSCOMPILING AND ARCH_AARCH64)) execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE) else () set (BUILTINS_LIBRARY "-lgcc") diff --git a/cmake/linux/toolchain-aarch64.cmake b/cmake/linux/toolchain-aarch64.cmake new file mode 100644 index 00000000000..e3924fdc537 --- /dev/null +++ b/cmake/linux/toolchain-aarch64.cmake @@ -0,0 +1,25 @@ +set (CMAKE_SYSTEM_NAME "Linux") +set (CMAKE_SYSTEM_PROCESSOR "aarch64") +set (CMAKE_C_COMPILER_TARGET "aarch64-linux-gnu") +set (CMAKE_CXX_COMPILER_TARGET "aarch64-linux-gnu") +set (CMAKE_ASM_COMPILER_TARGET "aarch64-linux-gnu") +set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64/aarch64-linux-gnu/libc") + +# We don't use compiler from toolchain because it's gcc-8, and we provide support only for gcc-9. +set (CMAKE_AR "${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64/bin/aarch64-linux-gnu-ar" CACHE FILEPATH "" FORCE) +set (CMAKE_RANLIB "${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64/bin/aarch64-linux-gnu-ranlib" CACHE FILEPATH "" FORCE) + +set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64") +set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64") +set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64") + +set (LINKER_NAME "lld" CACHE STRING "" FORCE) + +set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld") +set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld") + +set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) +set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) + +set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) +set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/target.cmake b/cmake/target.cmake index 5ce425b82cd..3c6aa225af9 100644 --- a/cmake/target.cmake +++ b/cmake/target.cmake @@ -9,62 +9,8 @@ elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin") add_definitions(-D OS_DARWIN) endif () -if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (COMPILER_GCC 1) -elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set (COMPILER_CLANG 1) -endif () - -if (COMPILER_GCC) - # Require minimum version of gcc - set (GCC_MINIMUM_VERSION 8) - if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${GCC_MINIMUM_VERSION} AND NOT CMAKE_VERSION VERSION_LESS 2.8.9) - message (FATAL_ERROR "GCC version must be at least ${GCC_MINIMUM_VERSION}. For example, if GCC ${GCC_MINIMUM_VERSION} is available under gcc-${GCC_MINIMUM_VERSION}, g++-${GCC_MINIMUM_VERSION} names, do the following: export CC=gcc-${GCC_MINIMUM_VERSION} CXX=g++-${GCC_MINIMUM_VERSION}; rm -rf CMakeCache.txt CMakeFiles; and re run cmake or ./release.") - endif () -elseif (COMPILER_CLANG) - # Require minimum version of clang - set (CLANG_MINIMUM_VERSION 7) - if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION}) - message (FATAL_ERROR "Clang version must be at least ${CLANG_MINIMUM_VERSION}.") - endif () -else () - message (WARNING "You are using an unsupported compiler. Compilation has only been tested with Clang 6+ and GCC 7+.") -endif () - -string(REGEX MATCH "-?[0-9]+(.[0-9]+)?$" COMPILER_POSTFIX ${CMAKE_CXX_COMPILER}) - -if (OS_LINUX) - find_program (LLD_PATH NAMES "lld${COMPILER_POSTFIX}" "lld") - find_program (GOLD_PATH NAMES "ld.gold" "gold") -endif() - -option (LINKER_NAME "Linker name or full path") -if (NOT LINKER_NAME) - if (COMPILER_CLANG AND LLD_PATH) - set (LINKER_NAME "lld") - elseif (GOLD_PATH) - set (LINKER_NAME "gold") - endif () -endif () - -if (LINKER_NAME) - message(STATUS "Using linker: ${LINKER_NAME} (selected from: LLD_PATH=${LLD_PATH}; GOLD_PATH=${GOLD_PATH}; COMPILER_POSTFIX=${COMPILER_POSTFIX})") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") -endif () - if (CMAKE_CROSSCOMPILING) if (OS_DARWIN) - set (CMAKE_SYSTEM_PROCESSOR x86_64) - set (CMAKE_C_COMPILER_TARGET x86_64-apple-darwin) - set (CMAKE_CXX_COMPILER_TARGET x86_64-apple-darwin) - - set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) - set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - - set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) - set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - # FIXME: broken dependencies set (USE_SNAPPY OFF CACHE INTERNAL "") set (ENABLE_SSL OFF CACHE INTERNAL "") @@ -73,12 +19,19 @@ if (CMAKE_CROSSCOMPILING) set (ENABLE_READLINE OFF CACHE INTERNAL "") set (ENABLE_ICU OFF CACHE INTERNAL "") set (ENABLE_FASTOPS OFF CACHE INTERNAL "") - - message (STATUS "Cross-compiling for Darwin") + elseif (OS_LINUX) + if (ARCH_AARCH64) + # FIXME: broken dependencies + set (ENABLE_PROTOBUF OFF CACHE INTERNAL "") + set (ENABLE_PARQUET OFF CACHE INTERNAL "") + set (ENABLE_MYSQL OFF CACHE INTERNAL "") + endif () else () - message (FATAL_ERROR "Trying to cross-compile to unsupported target: ${CMAKE_SYSTEM_NAME}!") + message (FATAL_ERROR "Trying to cross-compile to unsupported system: ${CMAKE_SYSTEM_NAME}!") endif () # Don't know why but CXX_STANDARD doesn't work for cross-compilation set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") + + message (STATUS "Cross-compiling for target: ${CMAKE_CXX_COMPILE_TARGET}") endif () diff --git a/cmake/toolchain/darwin-x86_64/README.txt b/cmake/toolchain/darwin-x86_64/README.txt new file mode 100644 index 00000000000..4ece240f029 --- /dev/null +++ b/cmake/toolchain/darwin-x86_64/README.txt @@ -0,0 +1,2 @@ +wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz +tar --strip-components=1 xJf MacOSX10.14.sdk.tar.xz diff --git a/cmake/toolchain/linux-aarch64/README.txt b/cmake/toolchain/linux-aarch64/README.txt new file mode 100644 index 00000000000..7146c1d2b9a --- /dev/null +++ b/cmake/toolchain/linux-aarch64/README.txt @@ -0,0 +1,2 @@ +wget https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en -O gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz +tar --strip-components=1 xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz diff --git a/cmake/tools.cmake b/cmake/tools.cmake new file mode 100644 index 00000000000..5a183f9eeba --- /dev/null +++ b/cmake/tools.cmake @@ -0,0 +1,41 @@ +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (COMPILER_GCC 1) +elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang|AppleClang") + set (COMPILER_CLANG 1) +endif () + +if (COMPILER_GCC) + # Require minimum version of gcc + set (GCC_MINIMUM_VERSION 8) + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${GCC_MINIMUM_VERSION} AND NOT CMAKE_VERSION VERSION_LESS 2.8.9) + message (FATAL_ERROR "GCC version must be at least ${GCC_MINIMUM_VERSION}. For example, if GCC ${GCC_MINIMUM_VERSION} is available under gcc-${GCC_MINIMUM_VERSION}, g++-${GCC_MINIMUM_VERSION} names, do the following: export CC=gcc-${GCC_MINIMUM_VERSION} CXX=g++-${GCC_MINIMUM_VERSION}; rm -rf CMakeCache.txt CMakeFiles; and re run cmake or ./release.") + endif () +elseif (COMPILER_CLANG) + # Require minimum version of clang + set (CLANG_MINIMUM_VERSION 7) + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION}) + message (FATAL_ERROR "Clang version must be at least ${CLANG_MINIMUM_VERSION}.") + endif () +else () + message (WARNING "You are using an unsupported compiler. Compilation has only been tested with Clang 6+ and GCC 7+.") +endif () + +option (LINKER_NAME "Linker name or full path") + +find_program (LLD_PATH NAMES "ld.lld" "lld") +find_program (GOLD_PATH NAMES "ld.gold" "gold") + +if (NOT LINKER_NAME) + if (LLD_PATH) + set (LINKER_NAME "lld") + elseif (GOLD_PATH) + set (LINKER_NAME "gold") + endif () +endif () + +if (LINKER_NAME) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") + + message(STATUS "Using custom linker by name: ${LINKER_NAME}") +endif () diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 06c33fb7e74..b0a271b21ac 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -124,7 +124,7 @@ if (USE_INTERNAL_SSL_LIBRARY) add_library(OpenSSL::SSL ALIAS ${OPENSSL_SSL_LIBRARY}) endif () -if (ENABLE_MYSQL AND USE_INTERNAL_MYSQL_LIBRARY) +function(mysql_support) set(CLIENT_PLUGIN_CACHING_SHA2_PASSWORD STATIC) set(CLIENT_PLUGIN_SHA256_PASSWORD STATIC) set(CLIENT_PLUGIN_REMOTE_IO OFF) @@ -136,7 +136,15 @@ if (ENABLE_MYSQL AND USE_INTERNAL_MYSQL_LIBRARY) if (GLIBC_COMPATIBILITY) set(LIBM glibc-compatibility) endif() + if (USE_INTERNAL_ZLIB_LIBRARY) + set(ZLIB_FOUND ON) + set(ZLIB_LIBRARY zlibstatic) + set(WITH_EXTERNAL_ZLIB ON) + endif() add_subdirectory (mariadb-connector-c) +endfunction() +if (ENABLE_MYSQL AND USE_INTERNAL_MYSQL_LIBRARY) + mysql_support() endif () if (USE_INTERNAL_RDKAFKA_LIBRARY) diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index a564f62384d..3ba24f49498 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -70,6 +70,14 @@ add_custom_command(OUTPUT orc_proto.pb.h orc_proto.pb.cc --cpp_out="${CMAKE_CURRENT_BINARY_DIR}" "${PROTO_DIR}/orc_proto.proto") +# arrow-cmake cmake file calling orc cmake subroutine which detects certain compiler features. +# Apple Clang compiler failed to compile this code without specifying c++11 standard. +# As result these compiler features detected as absent. In result it failed to compile orc itself. +# In orc makefile there is code that sets flags, but arrow-cmake ignores these flags. +if (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") + set (CXX11_FLAGS "-std=c++0x") +endif() + include(${ClickHouse_SOURCE_DIR}/contrib/orc/cmake_modules/CheckSourceCompiles.cmake) include(orc_check.cmake) configure_file("${ORC_INCLUDE_DIR}/orc/orc-config.hh.in" "${ORC_BUILD_INCLUDE_DIR}/orc/orc-config.hh") diff --git a/contrib/libcxx-cmake/CMakeLists.txt b/contrib/libcxx-cmake/CMakeLists.txt index 82d11e3e32d..a654ce59d6a 100644 --- a/contrib/libcxx-cmake/CMakeLists.txt +++ b/contrib/libcxx-cmake/CMakeLists.txt @@ -44,7 +44,7 @@ target_include_directories(cxx SYSTEM BEFORE PUBLIC $= 7 + +#define BRACK_PASTE_PREF "\033[200~" +#define BRACK_PASTE_SUFF "\033[201~" + +#define BRACK_PASTE_LAST '~' +#define BRACK_PASTE_SLEN 6 + +/// Make sure we don't get ^J for the enter character. +/// This handler also bypasses some unused macro/event checkings. +static int clickhouse_rl_bracketed_paste_begin(int /* count */, int /* key */) +{ + std::string buf; + buf.reserve(128); + + RL_SETSTATE(RL_STATE_MOREINPUT); + SCOPE_EXIT(RL_UNSETSTATE(RL_STATE_MOREINPUT)); + char c; + while ((c = rl_read_key()) >= 0) + { + if (c == '\r' || c == '\n') + c = '\n'; + buf.push_back(c); + if (buf.size() >= BRACK_PASTE_SLEN && c == BRACK_PASTE_LAST && buf.substr(buf.size() - BRACK_PASTE_SLEN) == BRACK_PASTE_SUFF) + { + buf.resize(buf.size() - BRACK_PASTE_SLEN); + break; + } + } + return static_cast(rl_insert_text(buf.c_str())) == buf.size() ? 0 : 1; +} + +#endif + namespace DB { @@ -462,6 +496,18 @@ private: if (rl_initialize()) throw Exception("Cannot initialize readline", ErrorCodes::CANNOT_READLINE); +#if RL_VERSION_MAJOR >= 7 + /// When bracketed paste mode is set, pasted text is bracketed with control sequences so + /// that the program can differentiate pasted text from typed-in text. This helps + /// clickhouse-client so that without -m flag, one can still paste multiline queries, and + /// possibly get better pasting performance. See https://cirw.in/blog/bracketed-paste for + /// more details. + rl_variable_bind("enable-bracketed-paste", "on"); + + /// Use our bracketed paste handler to get better user experience. See comments above. + rl_bind_keyseq(BRACK_PASTE_PREF, clickhouse_rl_bracketed_paste_begin); +#endif + auto clear_prompt_or_exit = [](int) { /// This is signal safe. @@ -632,7 +678,8 @@ private: /// If the user restarts the client then after pressing the "up" button /// every line of the query will be displayed separately. std::string logged_query = input; - std::replace(logged_query.begin(), logged_query.end(), '\n', ' '); + if (config().has("multiline")) + std::replace(logged_query.begin(), logged_query.end(), '\n', ' '); add_history(logged_query.c_str()); #if USE_READLINE && HAVE_READLINE_HISTORY diff --git a/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp b/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp index b188c0bea88..594cddfd3db 100644 --- a/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -38,12 +38,16 @@ namespace switch (type) { + case SQL_TINYINT: + return factory.get("Int8"); case SQL_INTEGER: return factory.get("Int32"); case SQL_SMALLINT: return factory.get("Int16"); + case SQL_BIGINT: + return factory.get("Int64"); case SQL_FLOAT: - return factory.get("Float32"); + return factory.get("Float64"); case SQL_REAL: return factory.get("Float32"); case SQL_DOUBLE: diff --git a/dbms/programs/performance-test/JSONString.h b/dbms/programs/performance-test/JSONString.h index 5695145442e..ebd850877d7 100644 --- a/dbms/programs/performance-test/JSONString.h +++ b/dbms/programs/performance-test/JSONString.h @@ -22,7 +22,7 @@ public: void set(const std::string & key, std::string value, bool wrap = true); template - std::enable_if_t> set(const std::string key, T value) + std::enable_if_t> set(const std::string key, T value) { set(key, std::to_string(value), /*wrap= */ false); } diff --git a/dbms/programs/server/CMakeLists.txt b/dbms/programs/server/CMakeLists.txt index 16aa7131291..1ebf227c3da 100644 --- a/dbms/programs/server/CMakeLists.txt +++ b/dbms/programs/server/CMakeLists.txt @@ -10,13 +10,11 @@ set(CLICKHOUSE_SERVER_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/TCPHandler.cpp ) -if (USE_SSL) - set(CLICKHOUSE_SERVER_SOURCES - ${CLICKHOUSE_SERVER_SOURCES} - ${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandler.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandlerFactory.cpp - ) -endif () +set(CLICKHOUSE_SERVER_SOURCES + ${CLICKHOUSE_SERVER_SOURCES} + ${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandler.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandlerFactory.cpp +) set(CLICKHOUSE_SERVER_LINK PRIVATE clickhouse_dictionaries clickhouse_common_io clickhouse_common_config clickhouse_common_zookeeper clickhouse_parsers string_utils PUBLIC daemon PRIVATE clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY}) if (USE_POCO_NETSSL) diff --git a/dbms/programs/server/MySQLHandler.cpp b/dbms/programs/server/MySQLHandler.cpp index f7429ebf2a7..68a5b1c6bc1 100644 --- a/dbms/programs/server/MySQLHandler.cpp +++ b/dbms/programs/server/MySQLHandler.cpp @@ -1,7 +1,6 @@ #include -#if USE_SSL -#include "MySQLHandler.h" +#include "MySQLHandler.h" #include #include #include @@ -15,37 +14,39 @@ #include #include #include -#include -#include -#include -#include #include +#if USE_POCO_NETSSL +#include +#include +#include +#include +#endif namespace DB { using namespace MySQLProtocol; - +#if USE_POCO_NETSSL using Poco::Net::SecureStreamSocket; using Poco::Net::SSLManager; - +#endif namespace ErrorCodes { extern const int MYSQL_CLIENT_INSUFFICIENT_CAPABILITIES; extern const int OPENSSL_ERROR; + extern const int SUPPORT_IS_DISABLED; } -MySQLHandler::MySQLHandler(IServer & server_, const Poco::Net::StreamSocket & socket_, RSA & public_key_, RSA & private_key_, bool ssl_enabled, size_t connection_id_) +MySQLHandler::MySQLHandler(IServer & server_, const Poco::Net::StreamSocket & socket_, + bool ssl_enabled, size_t connection_id_) : Poco::Net::TCPServerConnection(socket_) , server(server_) , log(&Poco::Logger::get("MySQLHandler")) , connection_context(server.context()) , connection_id(connection_id_) - , public_key(public_key_) - , private_key(private_key_) , auth_plugin(new MySQLProtocol::Authentication::Native41()) { server_capability_flags = CLIENT_PROTOCOL_41 | CLIENT_SECURE_CONNECTION | CLIENT_PLUGIN_AUTH | CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA | CLIENT_CONNECT_WITH_DB | CLIENT_DEPRECATE_EOF; @@ -197,21 +198,7 @@ void MySQLHandler::finishHandshake(MySQLProtocol::HandshakeResponse & packet) if (payload_size == SSL_REQUEST_PAYLOAD_SIZE) { - read_bytes(packet_size); /// Reading rest SSLRequest. - SSLRequest ssl_request; - ReadBufferFromMemory payload(buf, pos); - payload.ignore(PACKET_HEADER_SIZE); - ssl_request.readPayload(payload); - connection_context.mysql.client_capabilities = ssl_request.capability_flags; - connection_context.mysql.max_packet_size = ssl_request.max_packet_size ? ssl_request.max_packet_size : MAX_PACKET_LENGTH; - secure_connection = true; - ss = std::make_shared(SecureStreamSocket::attach(socket(), SSLManager::instance().defaultServerContext())); - in = std::make_shared(*ss); - out = std::make_shared(*ss); - connection_context.mysql.sequence_id = 2; - packet_sender = std::make_shared(*in, *out, connection_context.mysql.sequence_id); - packet_sender->max_packet_size = connection_context.mysql.max_packet_size; - packet_sender->receivePacket(packet); /// Reading HandshakeResponse from secure socket. + finishHandshakeSSL(packet_size, buf, pos, read_bytes, packet); } else { @@ -232,7 +219,9 @@ void MySQLHandler::authenticate(const String & user_name, const String & auth_pl // For compatibility with JavaScript MySQL client, Native41 authentication plugin is used when possible (if password is specified using double SHA1). Otherwise SHA256 plugin is used. auto user = connection_context.getUser(user_name); if (user->authentication.getType() != DB::Authentication::DOUBLE_SHA1_PASSWORD) - auth_plugin = std::make_unique(public_key, private_key, log); + { + authPluginSSL(); + } try { std::optional auth_response = auth_plugin_name == auth_plugin->getName() ? std::make_optional(initial_auth_response) : std::nullopt; @@ -302,5 +291,47 @@ void MySQLHandler::comQuery(ReadBuffer & payload) packet_sender->sendPacket(OK_Packet(0x00, client_capability_flags, 0, 0, 0), true); } +void MySQLHandler::authPluginSSL() +{ + throw Exception("Compiled without SSL", ErrorCodes::SUPPORT_IS_DISABLED); } + +void MySQLHandler::finishHandshakeSSL([[maybe_unused]] size_t packet_size, [[maybe_unused]] char * buf, [[maybe_unused]] size_t pos, [[maybe_unused]] std::function read_bytes, [[maybe_unused]] MySQLProtocol::HandshakeResponse & packet) +{ + throw Exception("Compiled without SSL", ErrorCodes::SUPPORT_IS_DISABLED); +} + +#if USE_SSL && USE_POCO_NETSSL +MySQLHandlerSSL::MySQLHandlerSSL(IServer & server_, const Poco::Net::StreamSocket & socket_, bool ssl_enabled, size_t connection_id_, RSA & public_key_, RSA & private_key_) + : MySQLHandler(server_, socket_, ssl_enabled, connection_id_) + , public_key(public_key_) + , private_key(private_key_) +{} + +void MySQLHandlerSSL::authPluginSSL() +{ + auth_plugin = std::make_unique(public_key, private_key, log); +} + +void MySQLHandlerSSL::finishHandshakeSSL(size_t packet_size, char * buf, size_t pos, std::function read_bytes, MySQLProtocol::HandshakeResponse & packet) +{ + read_bytes(packet_size); /// Reading rest SSLRequest. + SSLRequest ssl_request; + ReadBufferFromMemory payload(buf, pos); + payload.ignore(PACKET_HEADER_SIZE); + ssl_request.readPayload(payload); + connection_context.mysql.client_capabilities = ssl_request.capability_flags; + connection_context.mysql.max_packet_size = ssl_request.max_packet_size ? ssl_request.max_packet_size : MAX_PACKET_LENGTH; + secure_connection = true; + ss = std::make_shared(SecureStreamSocket::attach(socket(), SSLManager::instance().defaultServerContext())); + in = std::make_shared(*ss); + out = std::make_shared(*ss); + connection_context.mysql.sequence_id = 2; + packet_sender = std::make_shared(*in, *out, connection_context.mysql.sequence_id); + packet_sender->max_packet_size = connection_context.mysql.max_packet_size; + packet_sender->receivePacket(packet); /// Reading HandshakeResponse from secure socket. +} + #endif + +} diff --git a/dbms/programs/server/MySQLHandler.h b/dbms/programs/server/MySQLHandler.h index 9d51667925d..96cb353d897 100644 --- a/dbms/programs/server/MySQLHandler.h +++ b/dbms/programs/server/MySQLHandler.h @@ -1,13 +1,13 @@ #pragma once #include -#if USE_SSL - #include -#include #include #include #include "IServer.h" +#if USE_POCO_NETSSL +#include +#endif namespace DB { @@ -16,7 +16,7 @@ namespace DB class MySQLHandler : public Poco::Net::TCPServerConnection { public: - MySQLHandler(IServer & server_, const Poco::Net::StreamSocket & socket_, RSA & public_key_, RSA & private_key_, bool ssl_enabled, size_t connection_id_); + MySQLHandler(IServer & server_, const Poco::Net::StreamSocket & socket_, bool ssl_enabled, size_t connection_id_); void run() final; @@ -34,28 +34,47 @@ private: void authenticate(const String & user_name, const String & auth_plugin_name, const String & auth_response); + virtual void authPluginSSL(); + virtual void finishHandshakeSSL(size_t packet_size, char * buf, size_t pos, std::function read_bytes, MySQLProtocol::HandshakeResponse & packet); + IServer & server; + +protected: Poco::Logger * log; + Context connection_context; std::shared_ptr packet_sender; +private: size_t connection_id = 0; size_t server_capability_flags = 0; size_t client_capability_flags = 0; - RSA & public_key; - RSA & private_key; - +protected: std::unique_ptr auth_plugin; - std::shared_ptr ss; std::shared_ptr in; std::shared_ptr out; bool secure_connection = false; }; -} +#if USE_SSL && USE_POCO_NETSSL +class MySQLHandlerSSL : public MySQLHandler +{ +public: + MySQLHandlerSSL(IServer & server_, const Poco::Net::StreamSocket & socket_, bool ssl_enabled, size_t connection_id_, RSA & public_key_, RSA & private_key_); + +private: + void authPluginSSL() override; + void finishHandshakeSSL(size_t packet_size, char * buf, size_t pos, std::function read_bytes, MySQLProtocol::HandshakeResponse & packet) override; + + RSA & public_key; + RSA & private_key; + std::shared_ptr ss; +}; #endif + +} diff --git a/dbms/programs/server/MySQLHandlerFactory.cpp b/dbms/programs/server/MySQLHandlerFactory.cpp index 752d88e05d8..987efbfa347 100644 --- a/dbms/programs/server/MySQLHandlerFactory.cpp +++ b/dbms/programs/server/MySQLHandlerFactory.cpp @@ -1,7 +1,5 @@ #include "MySQLHandlerFactory.h" -#if USE_POCO_NETSSL && USE_SSL #include -#include #include #include #include @@ -9,6 +7,10 @@ #include "IServer.h" #include "MySQLHandler.h" +#if USE_POCO_NETSSL +#include +#endif + namespace DB { @@ -24,6 +26,8 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_) : server(server_) , log(&Logger::get("MySQLHandlerFactory")) { + +#if USE_POCO_NETSSL try { Poco::Net::SSLManager::instance().defaultServerContext(); @@ -33,7 +37,9 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_) LOG_INFO(log, "Failed to create SSL context. SSL will be disabled. Error: " << getCurrentExceptionMessage(false)); ssl_enabled = false; } +#endif +#if USE_SSL /// Reading rsa keys for SHA256 authentication plugin. try { @@ -44,8 +50,10 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_) LOG_WARNING(log, "Failed to read RSA keys. Error: " << getCurrentExceptionMessage(false)); generateRSAKeys(); } +#endif } +#if USE_SSL void MySQLHandlerFactory::readRSAKeys() { const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config(); @@ -113,13 +121,18 @@ void MySQLHandlerFactory::generateRSAKeys() if (!private_key) throw Exception("Failed to copy RSA key. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR); } +#endif Poco::Net::TCPServerConnection * MySQLHandlerFactory::createConnection(const Poco::Net::StreamSocket & socket) { size_t connection_id = last_connection_id++; LOG_TRACE(log, "MySQL connection. Id: " << connection_id << ". Address: " << socket.peerAddress().toString()); - return new MySQLHandler(server, socket, *public_key, *private_key, ssl_enabled, connection_id); +#if USE_POCO_NETSSL && USE_SSL + return new MySQLHandlerSSL(server, socket, ssl_enabled, connection_id, *public_key, *private_key); +#else + return new MySQLHandler(server, socket, ssl_enabled, connection_id); +#endif + } } -#endif diff --git a/dbms/programs/server/MySQLHandlerFactory.h b/dbms/programs/server/MySQLHandlerFactory.h index 78022ddce37..b7df9fa60e0 100644 --- a/dbms/programs/server/MySQLHandlerFactory.h +++ b/dbms/programs/server/MySQLHandlerFactory.h @@ -1,12 +1,12 @@ #pragma once #include -#if USE_POCO_NETSSL && USE_SSL - #include #include -#include #include "IServer.h" +#if USE_SSL +#include +#endif namespace DB { @@ -17,6 +17,7 @@ private: IServer & server; Poco::Logger * log; +#if USE_SSL struct RSADeleter { void operator()(RSA * ptr) { RSA_free(ptr); } @@ -27,6 +28,9 @@ private: RSAPtr private_key; bool ssl_enabled = true; +#else + bool ssl_enabled = false; +#endif std::atomic last_connection_id = 0; public: @@ -40,4 +44,3 @@ public: }; } -#endif diff --git a/dbms/programs/server/ReplicasStatusHandler.cpp b/dbms/programs/server/ReplicasStatusHandler.cpp index cbbdacccc56..22e059d9ccc 100644 --- a/dbms/programs/server/ReplicasStatusHandler.cpp +++ b/dbms/programs/server/ReplicasStatusHandler.cpp @@ -44,7 +44,7 @@ void ReplicasStatusHandler::handleRequest(Poco::Net::HTTPServerRequest & request if (db.second->getEngineName() == "Lazy") continue; - for (auto iterator = db.second->getIterator(context); iterator->isValid(); iterator->next()) + for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { auto & table = iterator->table(); StorageReplicatedMergeTree * table_replicated = dynamic_cast(table.get()); diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index 5385057fd03..79285b8cd23 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -37,10 +37,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -55,7 +57,7 @@ #include "TCPHandlerFactory.h" #include "Common/config_version.h" #include - +#include "MySQLHandlerFactory.h" #if defined(OS_LINUX) #include @@ -63,7 +65,6 @@ #endif #if USE_POCO_NETSSL -#include "MySQLHandlerFactory.h" #include #include #endif @@ -920,6 +921,12 @@ int Server::main(const std::vector & /*args*/) global_context->tryCreateEmbeddedDictionaries(); global_context->getExternalDictionariesLoader().enableAlwaysLoadEverything(true); } + + auto dictionaries_repository = std::make_unique(config(), "dictionaries_config"); + global_context->getExternalDictionariesLoader().addConfigRepository("", std::move(dictionaries_repository)); + + auto models_repository = std::make_unique(config(), "models_config"); + global_context->getExternalModelsLoader().addConfigRepository("", std::move(models_repository)); } catch (...) { diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 83749975f34..cb1512a81e8 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -530,7 +530,8 @@ void TCPHandler::processOrdinaryQuery() sendLogs(); } - sendData(block); + if (!block || !state.io.null_format) + sendData(block); if (!block) break; } @@ -850,9 +851,10 @@ bool TCPHandler::receivePacket() return true; case Protocol::Client::Data: + case Protocol::Client::Scalar: if (state.empty()) receiveUnexpectedData(); - return receiveData(); + return receiveData(packet_type == Protocol::Client::Scalar); case Protocol::Client::Ping: writeVarUInt(Protocol::Server::Pong, *out); @@ -957,39 +959,44 @@ void TCPHandler::receiveUnexpectedQuery() throw NetException("Unexpected packet Query received from client", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); } -bool TCPHandler::receiveData() +bool TCPHandler::receiveData(bool scalar) { initBlockInput(); /// The name of the temporary table for writing data, default to empty string - String external_table_name; - readStringBinary(external_table_name, *in); + String name; + readStringBinary(name, *in); /// Read one block from the network and write it down Block block = state.block_in->read(); if (block) { - /// If there is an insert request, then the data should be written directly to `state.io.out`. - /// Otherwise, we write the blocks in the temporary `external_table_name` table. - if (!state.need_receive_data_for_insert && !state.need_receive_data_for_input) - { - StoragePtr storage; - /// If such a table does not exist, create it. - if (!(storage = query_context->tryGetExternalTable(external_table_name))) - { - NamesAndTypesList columns = block.getNamesAndTypesList(); - storage = StorageMemory::create("_external", external_table_name, ColumnsDescription{columns}, ConstraintsDescription{}); - storage->startup(); - query_context->addExternalTable(external_table_name, storage); - } - /// The data will be written directly to the table. - state.io.out = storage->write(ASTPtr(), *query_context); - } - if (state.need_receive_data_for_input) - state.block_for_input = block; + if (scalar) + query_context->addScalar(name, block); else - state.io.out->write(block); + { + /// If there is an insert request, then the data should be written directly to `state.io.out`. + /// Otherwise, we write the blocks in the temporary `external_table_name` table. + if (!state.need_receive_data_for_insert && !state.need_receive_data_for_input) + { + StoragePtr storage; + /// If such a table does not exist, create it. + if (!(storage = query_context->tryGetExternalTable(name))) + { + NamesAndTypesList columns = block.getNamesAndTypesList(); + storage = StorageMemory::create("_external", name, ColumnsDescription{columns}, ConstraintsDescription{}); + storage->startup(); + query_context->addExternalTable(name, storage); + } + /// The data will be written directly to the table. + state.io.out = storage->write(ASTPtr(), *query_context); + } + if (state.need_receive_data_for_input) + state.block_for_input = block; + else + state.io.out->write(block); + } return true; } else diff --git a/dbms/programs/server/TCPHandler.h b/dbms/programs/server/TCPHandler.h index 834d66bba28..561ed4d0eca 100644 --- a/dbms/programs/server/TCPHandler.h +++ b/dbms/programs/server/TCPHandler.h @@ -153,7 +153,7 @@ private: void receiveHello(); bool receivePacket(); void receiveQuery(); - bool receiveData(); + bool receiveData(bool scalar); bool readDataNext(const size_t & poll_interval, const int & receive_timeout); void readData(const Settings & global_settings); std::tuple getReadTimeouts(const Settings & global_settings); diff --git a/dbms/programs/server/config.xml b/dbms/programs/server/config.xml index c8d33922167..6e9bb527c97 100644 --- a/dbms/programs/server/config.xml +++ b/dbms/programs/server/config.xml @@ -180,7 +180,21 @@ 9000 - + + + + + 127.0.0.1 + 9000 + + + + + 127.0.0.2 + 9000 + + + diff --git a/dbms/src/Interpreters/SettingsConstraints.cpp b/dbms/src/Access/SettingsConstraints.cpp similarity index 61% rename from dbms/src/Interpreters/SettingsConstraints.cpp rename to dbms/src/Access/SettingsConstraints.cpp index 35c431d7ed7..a044b7a0dc1 100644 --- a/dbms/src/Interpreters/SettingsConstraints.cpp +++ b/dbms/src/Access/SettingsConstraints.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -29,22 +29,118 @@ void SettingsConstraints::clear() } -void SettingsConstraints::setReadOnly(const String & name, bool read_only) +void SettingsConstraints::setMinValue(const StringRef & name, const Field & min_value) +{ + size_t setting_index = Settings::findIndexStrict(name); + getConstraintRef(setting_index).min_value = Settings::valueToCorrespondingType(setting_index, min_value); +} + + +Field SettingsConstraints::getMinValue(const StringRef & name) const +{ + size_t setting_index = Settings::findIndexStrict(name); + const auto * ptr = tryGetConstraint(setting_index); + if (ptr) + return ptr->min_value; + else + return {}; +} + + +void SettingsConstraints::setMaxValue(const StringRef & name, const Field & max_value) +{ + size_t setting_index = Settings::findIndexStrict(name); + getConstraintRef(setting_index).max_value = Settings::valueToCorrespondingType(setting_index, max_value); +} + + +Field SettingsConstraints::getMaxValue(const StringRef & name) const +{ + size_t setting_index = Settings::findIndexStrict(name); + const auto * ptr = tryGetConstraint(setting_index); + if (ptr) + return ptr->max_value; + else + return {}; +} + + +void SettingsConstraints::setReadOnly(const StringRef & name, bool read_only) { size_t setting_index = Settings::findIndexStrict(name); getConstraintRef(setting_index).read_only = read_only; } -void SettingsConstraints::setMinValue(const String & name, const Field & min_value) + +bool SettingsConstraints::isReadOnly(const StringRef & name) const { size_t setting_index = Settings::findIndexStrict(name); - getConstraintRef(setting_index).min_value = Settings::castValueWithoutApplying(setting_index, min_value); + const auto * ptr = tryGetConstraint(setting_index); + if (ptr) + return ptr->read_only; + else + return false; } -void SettingsConstraints::setMaxValue(const String & name, const Field & max_value) + +void SettingsConstraints::set(const StringRef & name, const Field & min_value, const Field & max_value, bool read_only) { size_t setting_index = Settings::findIndexStrict(name); - getConstraintRef(setting_index).max_value = Settings::castValueWithoutApplying(setting_index, max_value); + auto & ref = getConstraintRef(setting_index); + ref.min_value = min_value; + ref.max_value = max_value; + ref.read_only = read_only; +} + + +void SettingsConstraints::get(const StringRef & name, Field & min_value, Field & max_value, bool & read_only) const +{ + size_t setting_index = Settings::findIndexStrict(name); + const auto * ptr = tryGetConstraint(setting_index); + if (ptr) + { + min_value = ptr->min_value; + max_value = ptr->max_value; + read_only = ptr->read_only; + } + else + { + min_value = Field{}; + max_value = Field{}; + read_only = false; + } +} + + +void SettingsConstraints::merge(const SettingsConstraints & other) +{ + for (const auto & [setting_index, other_constraint] : other.constraints_by_index) + { + auto & constraint = constraints_by_index[setting_index]; + if (!other_constraint.min_value.isNull()) + constraint.min_value = other_constraint.min_value; + if (!other_constraint.max_value.isNull()) + constraint.max_value = other_constraint.max_value; + if (other_constraint.read_only) + constraint.read_only = true; + } +} + + +SettingsConstraints::Infos SettingsConstraints::getInfo() const +{ + Infos result; + result.reserve(constraints_by_index.size()); + for (const auto & [setting_index, constraint] : constraints_by_index) + { + result.emplace_back(); + Info & info = result.back(); + info.name = Settings::getName(setting_index); + info.min = constraint.min_value; + info.max = constraint.max_value; + info.read_only = constraint.read_only; + } + return result; } @@ -55,7 +151,7 @@ void SettingsConstraints::check(const Settings & current_settings, const Setting if (setting_index == Settings::npos) return; - Field new_value = Settings::castValueWithoutApplying(setting_index, change.value); + Field new_value = Settings::valueToCorrespondingType(setting_index, change.value); Field current_value = current_settings.get(setting_index); /// Setting isn't checked if value wasn't changed. @@ -159,4 +255,15 @@ void SettingsConstraints::loadFromConfig(const String & path_to_constraints, con } } + +bool SettingsConstraints::Constraint::operator==(const Constraint & rhs) const +{ + return (read_only == rhs.read_only) && (min_value == rhs.min_value) && (max_value == rhs.max_value); +} + + +bool operator ==(const SettingsConstraints & lhs, const SettingsConstraints & rhs) +{ + return lhs.constraints_by_index == rhs.constraints_by_index; +} } diff --git a/dbms/src/Interpreters/SettingsConstraints.h b/dbms/src/Access/SettingsConstraints.h similarity index 70% rename from dbms/src/Interpreters/SettingsConstraints.h rename to dbms/src/Access/SettingsConstraints.h index f0ba3b7eb16..3b4d0c28800 100644 --- a/dbms/src/Interpreters/SettingsConstraints.h +++ b/dbms/src/Access/SettingsConstraints.h @@ -58,10 +58,32 @@ public: ~SettingsConstraints(); void clear(); + bool empty() const { return constraints_by_index.empty(); } - void setMinValue(const String & name, const Field & min_value); - void setMaxValue(const String & name, const Field & max_value); - void setReadOnly(const String & name, bool read_only); + void setMinValue(const StringRef & name, const Field & min_value); + Field getMinValue(const StringRef & name) const; + + void setMaxValue(const StringRef & name, const Field & max_value); + Field getMaxValue(const StringRef & name) const; + + void setReadOnly(const StringRef & name, bool read_only); + bool isReadOnly(const StringRef & name) const; + + void set(const StringRef & name, const Field & min_value, const Field & max_value, bool read_only); + void get(const StringRef & name, Field & min_value, Field & max_value, bool & read_only) const; + + void merge(const SettingsConstraints & other); + + struct Info + { + StringRef name; + Field min; + Field max; + bool read_only = false; + }; + using Infos = std::vector; + + Infos getInfo() const; void check(const Settings & current_settings, const SettingChange & change) const; void check(const Settings & current_settings, const SettingsChanges & changes) const; @@ -74,12 +96,18 @@ public: /// Loads the constraints from configuration file, at "path" prefix in configuration. void loadFromConfig(const String & path, const Poco::Util::AbstractConfiguration & config); + friend bool operator ==(const SettingsConstraints & lhs, const SettingsConstraints & rhs); + friend bool operator !=(const SettingsConstraints & lhs, const SettingsConstraints & rhs) { return !(lhs == rhs); } + private: struct Constraint { bool read_only = false; Field min_value; Field max_value; + + bool operator ==(const Constraint & rhs) const; + bool operator !=(const Constraint & rhs) const { return !(*this == rhs); } }; Constraint & getConstraintRef(size_t index); diff --git a/dbms/src/AggregateFunctions/AggregateFunctionAvg.h b/dbms/src/AggregateFunctions/AggregateFunctionAvg.h index 185dbc38c51..13aa9157706 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionAvg.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionAvg.h @@ -31,9 +31,9 @@ struct AggregateFunctionAvgData if constexpr (std::numeric_limits::is_iec559) return static_cast(sum) / count; /// allow division by zero - if (!count) - throw Exception("AggregateFunctionAvg with zero values", ErrorCodes::LOGICAL_ERROR); - return static_cast(sum) / count; + if (count == 0) + return static_cast(0); + return static_cast(sum / count); } }; @@ -43,10 +43,10 @@ template class AggregateFunctionAvg final : public IAggregateFunctionDataHelper> { public: - using ResultType = std::conditional_t, Decimal128, Float64>; - using ResultDataType = std::conditional_t, DataTypeDecimal, DataTypeNumber>; + using ResultType = std::conditional_t, T, Float64>; + using ResultDataType = std::conditional_t, DataTypeDecimal, DataTypeNumber>; using ColVecType = std::conditional_t, ColumnDecimal, ColumnVector>; - using ColVecResult = std::conditional_t, ColumnDecimal, ColumnVector>; + using ColVecResult = std::conditional_t, ColumnDecimal, ColumnVector>; /// ctor for native types AggregateFunctionAvg(const DataTypes & argument_types_) diff --git a/dbms/src/AggregateFunctions/AggregateFunctionCount.h b/dbms/src/AggregateFunctions/AggregateFunctionCount.h index c1691e665b3..edf6277cef5 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionCount.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionCount.h @@ -64,6 +64,12 @@ public: } const char * getHeaderFilePath() const override { return __FILE__; } + + /// Reset the state to specified value. This function is not the part of common interface. + void set(AggregateDataPtr place, UInt64 new_count) + { + data(place).count = new_count; + } }; diff --git a/dbms/src/AggregateFunctions/AggregateFunctionGroupBitmap.h b/dbms/src/AggregateFunctions/AggregateFunctionGroupBitmap.h index 65a450bfbaf..6479eaf3c1f 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionGroupBitmap.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionGroupBitmap.h @@ -90,6 +90,10 @@ public: { Data & data_lhs = this->data(place); const Data & data_rhs = this->data(rhs); + + if (!data_rhs.doneFirst) + return; + if (!data_lhs.doneFirst) { data_lhs.doneFirst = true; diff --git a/dbms/src/AggregateFunctions/AggregateFunctionMinMaxAny.h b/dbms/src/AggregateFunctions/AggregateFunctionMinMaxAny.h index 00869c846d4..db2978db6a0 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionMinMaxAny.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionMinMaxAny.h @@ -673,15 +673,15 @@ struct AggregateFunctionAnyHeavyData : Data }; -template -class AggregateFunctionsSingleValue final : public IAggregateFunctionDataHelper> +template +class AggregateFunctionsSingleValue final : public IAggregateFunctionDataHelper> { private: DataTypePtr & type; public: AggregateFunctionsSingleValue(const DataTypePtr & type_) - : IAggregateFunctionDataHelper>({type_}, {}) + : IAggregateFunctionDataHelper>({type_}, {}) , type(this->argument_types[0]) { if (StringRef(Data::name()) == StringRef("min") @@ -722,7 +722,7 @@ public: bool allocatesMemoryInArena() const override { - return AllocatesMemoryInArena; + return use_arena; } void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override diff --git a/dbms/src/AggregateFunctions/AggregateFunctionQuantile.cpp b/dbms/src/AggregateFunctions/AggregateFunctionQuantile.cpp index 2439120d169..d96bb82d6f5 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionQuantile.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionQuantile.cpp @@ -16,11 +16,11 @@ namespace ErrorCodes namespace { -template using FuncQuantile = AggregateFunctionQuantile, NameQuantile, false, std::conditional_t, false>; -template using FuncQuantiles = AggregateFunctionQuantile, NameQuantiles, false, std::conditional_t, true>; +template using FuncQuantile = AggregateFunctionQuantile, NameQuantile, false, std::conditional_t, false>; +template using FuncQuantiles = AggregateFunctionQuantile, NameQuantiles, false, std::conditional_t, true>; -template using FuncQuantileDeterministic = AggregateFunctionQuantile, NameQuantileDeterministic, true, std::conditional_t, false>; -template using FuncQuantilesDeterministic = AggregateFunctionQuantile, NameQuantilesDeterministic, true, std::conditional_t, true>; +template using FuncQuantileDeterministic = AggregateFunctionQuantile, NameQuantileDeterministic, true, std::conditional_t, false>; +template using FuncQuantilesDeterministic = AggregateFunctionQuantile, NameQuantilesDeterministic, true, std::conditional_t, true>; template using FuncQuantileExact = AggregateFunctionQuantile, NameQuantileExact, false, void, false>; template using FuncQuantilesExact = AggregateFunctionQuantile, NameQuantilesExact, false, void, true>; @@ -40,11 +40,11 @@ template using FuncQuantilesTiming = AggregateFunctionQ template using FuncQuantileTimingWeighted = AggregateFunctionQuantile, NameQuantileTimingWeighted, true, Float32, false>; template using FuncQuantilesTimingWeighted = AggregateFunctionQuantile, NameQuantilesTimingWeighted, true, Float32, true>; -template using FuncQuantileTDigest = AggregateFunctionQuantile, NameQuantileTDigest, false, std::conditional_t, false>; -template using FuncQuantilesTDigest = AggregateFunctionQuantile, NameQuantilesTDigest, false, std::conditional_t, true>; +template using FuncQuantileTDigest = AggregateFunctionQuantile, NameQuantileTDigest, false, std::conditional_t, false>; +template using FuncQuantilesTDigest = AggregateFunctionQuantile, NameQuantilesTDigest, false, std::conditional_t, true>; -template using FuncQuantileTDigestWeighted = AggregateFunctionQuantile, NameQuantileTDigestWeighted, true, std::conditional_t, false>; -template using FuncQuantilesTDigestWeighted = AggregateFunctionQuantile, NameQuantilesTDigestWeighted, true, std::conditional_t, true>; +template using FuncQuantileTDigestWeighted = AggregateFunctionQuantile, NameQuantileTDigestWeighted, true, std::conditional_t, false>; +template using FuncQuantilesTDigestWeighted = AggregateFunctionQuantile, NameQuantilesTDigestWeighted, true, std::conditional_t, true>; template