diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c93accf1009..257040c68b7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,4 @@ dbms/* @ClickHouse/core-assigner +utils/* @ClickHouse/core-assigner docs/* @ClickHouse/docs docs/zh/* @ClickHouse/docs-zh diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 926a7b4e7ef..0e914c656fc 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,8 +1,6 @@ I hereby agree to the terms of the CLA available at: https://yandex.ru/legal/cla/?lang=en -For changelog. Remove if this is non-significant change. - -Category (leave one): +Changelog category (leave one): - New Feature - Bug Fix - Improvement @@ -11,11 +9,14 @@ Category (leave one): - Build/Testing/Packaging Improvement - Documentation - Other +- Non-significant (changelog entry is not needed) -Short description (up to few sentences): + +Changelog entry (up to few sentences, required except for Non-significant/Documentation categories): ... + Detailed description (optional): ... diff --git a/.github/stale.yml b/.github/stale.yml deleted file mode 100644 index 6628bbbd305..00000000000 --- a/.github/stale.yml +++ /dev/null @@ -1,70 +0,0 @@ -# Configuration for probot-stale - https://github.com/probot/stale - -# Number of days of inactivity before an Issue or Pull Request becomes stale -daysUntilStale: 60 - -# Number of days of inactivity before an Issue or Pull Request with the stale label is closed. -# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale. -daysUntilClose: 30 - -# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled) -onlyLabels: [] - -# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable -exemptLabels: - - bug - - feature - - memory - - performance - - prio-crit - - prio-major - - st-accepted - - st-in-progress - - st-waiting-for-fix - - segfault - - crash - -# Set to true to ignore issues in a project (defaults to false) -exemptProjects: false - -# Set to true to ignore issues in a milestone (defaults to false) -exemptMilestones: false - -# Set to true to ignore issues with an assignee (defaults to false) -exemptAssignees: false - -# Label to use when marking as stale -staleLabel: stale - -# Comment to post when marking as stale. Set to `false` to disable -markComment: > - This issue has been automatically marked as stale because it has not had - recent activity. Please post a comment if this issue is still relevant to you. - Thank you for your contributions. - -# Comment to post when removing the stale label. -# unmarkComment: > -# Your comment here. - -# Comment to post when closing a stale Issue or Pull Request. -# closeComment: > -# Your comment here. - -# Limit the number of actions per hour, from 1-30. Default is 30 -limitPerRun: 30 - -# Limit to only `issues` or `pulls` -# only: issues - -# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls': -pulls: - daysUntilStale: 365 - daysUntilClose: 60 - markComment: > - This pull request has been automatically marked as stale because it has not had - any activity for over a year. Please post a comment about whether you intend to continue working on it. - Thank you for your contributions. - -# issues: -# exemptLabels: -# - confirmed diff --git a/.gitmodules b/.gitmodules index 0b80743cadb..7d975031c54 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,7 @@ [submodule "contrib/poco"] path = contrib/poco url = https://github.com/ClickHouse-Extras/poco + branch = clickhouse [submodule "contrib/zstd"] path = contrib/zstd url = https://github.com/facebook/zstd.git @@ -28,9 +29,6 @@ [submodule "contrib/re2"] path = contrib/re2 url = https://github.com/google/re2.git -[submodule "contrib/ssl"] - path = contrib/ssl - url = https://github.com/ClickHouse-Extras/ssl.git [submodule "contrib/llvm"] path = contrib/llvm url = https://github.com/ClickHouse-Extras/llvm @@ -69,10 +67,10 @@ url = https://github.com/ClickHouse-Extras/libgsasl.git [submodule "contrib/libcxx"] path = contrib/libcxx - url = https://github.com/llvm-mirror/libcxx.git + url = https://github.com/ClickHouse-Extras/libcxx.git [submodule "contrib/libcxxabi"] path = contrib/libcxxabi - url = https://github.com/llvm-mirror/libcxxabi.git + url = https://github.com/ClickHouse-Extras/libcxxabi.git [submodule "contrib/snappy"] path = contrib/snappy url = https://github.com/google/snappy @@ -106,3 +104,30 @@ [submodule "contrib/sparsehash-c11"] path = contrib/sparsehash-c11 url = https://github.com/sparsehash/sparsehash-c11.git +[submodule "contrib/aws"] + path = contrib/aws + url = https://github.com/aws/aws-sdk-cpp.git +[submodule "aws-c-event-stream"] + path = contrib/aws-c-event-stream + url = https://github.com/awslabs/aws-c-event-stream.git +[submodule "aws-c-common"] + path = contrib/aws-c-common + url = https://github.com/awslabs/aws-c-common.git +[submodule "aws-checksums"] + path = contrib/aws-checksums + url = https://github.com/awslabs/aws-checksums.git +[submodule "contrib/curl"] + path = contrib/curl + url = https://github.com/curl/curl.git +[submodule "contrib/openssl"] + path = contrib/openssl + url = https://github.com/ClickHouse-Extras/openssl.git +[submodule "contrib/icudata"] + path = contrib/icudata + url = https://github.com/ClickHouse-Extras/icudata.git +[submodule "contrib/icu"] + path = contrib/icu + url = https://github.com/unicode-org/icu.git +[submodule "contrib/libc-headers"] + path = contrib/libc-headers + url = https://github.com/ClickHouse-Extras/libc-headers.git diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a8c32b588a..305021728a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,375 @@ +## ClickHouse release v19.17.4.11, 2019-11-22 + +### Backward Incompatible Change +* Using column instead of AST to store scalar subquery results for better performance. Setting `enable_scalar_subquery_optimization` was added in 19.17 and it was enabled by default. It leads to errors like [this](https://github.com/ClickHouse/ClickHouse/issues/7851) during upgrade to 19.17.2 or 19.17.3 from previous versions. This setting was disabled by default in 19.17.4, to make possible upgrading from 19.16 and older versions without errors. [#7392](https://github.com/ClickHouse/ClickHouse/pull/7392) ([Amos Bird](https://github.com/amosbird)) + +### New Feature +* Add the ability to create dictionaries with DDL queries. [#7360](https://github.com/ClickHouse/ClickHouse/pull/7360) ([alesapin](https://github.com/alesapin)) +* Make `bloom_filter` type of index supporting `LowCardinality` and `Nullable` [#7363](https://github.com/ClickHouse/ClickHouse/issues/7363) [#7561](https://github.com/ClickHouse/ClickHouse/pull/7561) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Add function `isValidJSON` to check that passed string is a valid json. [#5910](https://github.com/ClickHouse/ClickHouse/issues/5910) [#7293](https://github.com/ClickHouse/ClickHouse/pull/7293) ([Vdimir](https://github.com/Vdimir)) +* Implement `arrayCompact` function [#7328](https://github.com/ClickHouse/ClickHouse/pull/7328) ([Memo](https://github.com/Joeywzr)) +* Created function `hex` for Decimal numbers. It works like `hex(reinterpretAsString())`, but doesn't delete last zero bytes. [#7355](https://github.com/ClickHouse/ClickHouse/pull/7355) ([Mikhail Korotov](https://github.com/millb)) +* Add `arrayFill` and `arrayReverseFill` functions, which replace elements by other elements in front/back of them in the array. [#7380](https://github.com/ClickHouse/ClickHouse/pull/7380) ([hcz](https://github.com/hczhcz)) +* Add `CRC32IEEE()`/`CRC64()` support [#7480](https://github.com/ClickHouse/ClickHouse/pull/7480) ([Azat Khuzhin](https://github.com/azat)) +* Implement `char` function similar to one in [mysql](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_char) [#7486](https://github.com/ClickHouse/ClickHouse/pull/7486) ([sundyli](https://github.com/sundy-li)) +* Add `bitmapTransform` function. It transforms an array of values in a bitmap to another array of values, the result is a new bitmap [#7598](https://github.com/ClickHouse/ClickHouse/pull/7598) ([Zhichang Yu](https://github.com/yuzhichang)) +* Implemented `javaHashUTF16LE()` function [#7651](https://github.com/ClickHouse/ClickHouse/pull/7651) ([achimbab](https://github.com/achimbab)) +* Add `_shard_num` virtual column for the Distributed engine [#7624](https://github.com/ClickHouse/ClickHouse/pull/7624) ([Azat Khuzhin](https://github.com/azat)) + +### Experimental Feature +* Support for processors (new query execution pipeline) in `MergeTree`. [#7181](https://github.com/ClickHouse/ClickHouse/pull/7181) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) + +### Bug Fix +* Fix incorrect float parsing in `Values` [#7817](https://github.com/ClickHouse/ClickHouse/issues/7817) [#7870](https://github.com/ClickHouse/ClickHouse/pull/7870) ([tavplubix](https://github.com/tavplubix)) +* Fix rare deadlock which can happen when trace_log is enabled. [#7838](https://github.com/ClickHouse/ClickHouse/pull/7838) ([filimonov](https://github.com/filimonov)) +* Prevent message duplication when producing Kafka table has any MVs selecting from it [#7265](https://github.com/ClickHouse/ClickHouse/pull/7265) ([Ivan](https://github.com/abyss7)) +* Support for `Array(LowCardinality(Nullable(String)))` in `IN`. Resolves [#7364](https://github.com/ClickHouse/ClickHouse/issues/7364) [#7366](https://github.com/ClickHouse/ClickHouse/pull/7366) ([achimbab](https://github.com/achimbab)) +* Add handling of `SQL_TINYINT` and `SQL_BIGINT`, and fix handling of `SQL_FLOAT` data source types in ODBC Bridge. [#7491](https://github.com/ClickHouse/ClickHouse/pull/7491) ([Denis Glazachev](https://github.com/traceon)) +* Fix aggregation (`avg` and quantiles) over empty decimal columns [#7431](https://github.com/ClickHouse/ClickHouse/pull/7431) ([Andrey Konyaev](https://github.com/akonyaev90)) +* Fix `INSERT` into Distributed with `MATERIALIZED` columns [#7377](https://github.com/ClickHouse/ClickHouse/pull/7377) ([Azat Khuzhin](https://github.com/azat)) +* Make `MOVE PARTITION` work if some parts of partition are already on destination disk or volume [#7434](https://github.com/ClickHouse/ClickHouse/pull/7434) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fixed bug with hardlinks failing to be created during mutations in `ReplicatedMergeTree` in multi-disk configurations. [#7558](https://github.com/ClickHouse/ClickHouse/pull/7558) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fixed a bug with a mutation on a MergeTree when whole part remains unchanged and best space is being found on another disk [#7602](https://github.com/ClickHouse/ClickHouse/pull/7602) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fixed bug with `keep_free_space_ratio` not being read from disks configuration [#7645](https://github.com/ClickHouse/ClickHouse/pull/7645) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix bug with table contains only `Tuple` columns or columns with complex paths. Fixes [7541](https://github.com/ClickHouse/ClickHouse/issues/7541). [#7545](https://github.com/ClickHouse/ClickHouse/pull/7545) ([alesapin](https://github.com/alesapin)) +* Do not account memory for Buffer engine in max_memory_usage limit [#7552](https://github.com/ClickHouse/ClickHouse/pull/7552) ([Azat Khuzhin](https://github.com/azat)) +* Fix final mark usage in `MergeTree` tables ordered by `tuple()`. In rare cases it could lead to `Can't adjust last granule` error while select. [#7639](https://github.com/ClickHouse/ClickHouse/pull/7639) ([Anton Popov](https://github.com/CurtizJ)) +* Fix bug in mutations that have predicate with actions that require context (for example functions for json), which may lead to crashes or strange exceptions. [#7664](https://github.com/ClickHouse/ClickHouse/pull/7664) ([alesapin](https://github.com/alesapin)) +* Fix mismatch of database and table names escaping in `data/` and `shadow/` directories [#7575](https://github.com/ClickHouse/ClickHouse/pull/7575) ([Alexander Burmak](https://github.com/Alex-Burmak)) +* Support duplicated keys in RIGHT|FULL JOINs, e.g. ```ON t.x = u.x AND t.x = u.y```. Fix crash in this case. [#7586](https://github.com/ClickHouse/ClickHouse/pull/7586) ([Artem Zuikov](https://github.com/4ertus2)) +* Fix `Not found column in block` when joining on expression with RIGHT or FULL JOIN. [#7641](https://github.com/ClickHouse/ClickHouse/pull/7641) ([Artem Zuikov](https://github.com/4ertus2)) +* One more attempt to fix infinite loop in `PrettySpace` format [#7591](https://github.com/ClickHouse/ClickHouse/pull/7591) ([Olga Khvostikova](https://github.com/stavrolia)) +* Fix bug in `concat` function when all arguments were `FixedString` of the same size. [#7635](https://github.com/ClickHouse/ClickHouse/pull/7635) ([alesapin](https://github.com/alesapin)) +* Fixed exception in case of using 1 argument while defining S3, URL and HDFS storages. [#7618](https://github.com/ClickHouse/ClickHouse/pull/7618) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix scope of the InterpreterSelectQuery for views with query [#7601](https://github.com/ClickHouse/ClickHouse/pull/7601) ([Azat Khuzhin](https://github.com/azat)) + +### Improvement +* `Nullable` columns recognized and NULL-values handled correctly by ODBC-bridge [#7402](https://github.com/ClickHouse/ClickHouse/pull/7402) ([Vasily Nemkov](https://github.com/Enmk)) +* Write current batch for distributed send atomically [#7600](https://github.com/ClickHouse/ClickHouse/pull/7600) ([Azat Khuzhin](https://github.com/azat)) +* Throw an exception if we cannot detect table for column name in query. [#7358](https://github.com/ClickHouse/ClickHouse/pull/7358) ([Artem Zuikov](https://github.com/4ertus2)) +* Add `merge_max_block_size` setting to `MergeTreeSettings` [#7412](https://github.com/ClickHouse/ClickHouse/pull/7412) ([Artem Zuikov](https://github.com/4ertus2)) +* Queries with `HAVING` and without `GROUP BY` assume group by constant. So, `SELECT 1 HAVING 1` now returns a result. [#7496](https://github.com/ClickHouse/ClickHouse/pull/7496) ([Amos Bird](https://github.com/amosbird)) +* Support parsing `(X,)` as tuple similar to python. [#7501](https://github.com/ClickHouse/ClickHouse/pull/7501), [#7562](https://github.com/ClickHouse/ClickHouse/pull/7562) ([Amos Bird](https://github.com/amosbird)) +* Make `range` function behaviors almost like pythonic one. [#7518](https://github.com/ClickHouse/ClickHouse/pull/7518) ([sundyli](https://github.com/sundy-li)) +* Add `constraints` columns to table `system.settings` [#7553](https://github.com/ClickHouse/ClickHouse/pull/7553) ([Vitaly Baranov](https://github.com/vitlibar)) +* Better Null format for tcp handler, so that it's possible to use `select ignore() from table format Null` for perf measure via clickhouse-client [#7606](https://github.com/ClickHouse/ClickHouse/pull/7606) ([Amos Bird](https://github.com/amosbird)) +* Queries like `CREATE TABLE ... AS (SELECT (1, 2))` are parsed correctly [#7542](https://github.com/ClickHouse/ClickHouse/pull/7542) ([hcz](https://github.com/hczhcz)) + +### Performance Improvement +* The performance of aggregation over short string keys is improved. [#6243](https://github.com/ClickHouse/ClickHouse/pull/6243) ([Alexander Kuzmenkov](https://github.com/akuzm), [Amos Bird](https://github.com/amosbird)) +* Run another pass of syntax/expression analysis to get potential optimizations after constant predicates are folded. [#7497](https://github.com/ClickHouse/ClickHouse/pull/7497) ([Amos Bird](https://github.com/amosbird)) +* Use storage meta info to evaluate trivial `SELECT count() FROM table;` [#7510](https://github.com/ClickHouse/ClickHouse/pull/7510) ([Amos Bird](https://github.com/amosbird), [alexey-milovidov](https://github.com/alexey-milovidov)) +* Vectorize processing `arrayReduce` similar to Aggregator `addBatch`. [#7608](https://github.com/ClickHouse/ClickHouse/pull/7608) ([Amos Bird](https://github.com/amosbird)) +* Minor improvements in performance of `Kafka` consumption [#7475](https://github.com/ClickHouse/ClickHouse/pull/7475) ([Ivan](https://github.com/abyss7)) + +### Build/Testing/Packaging Improvement +* Add support for cross-compiling to the CPU architecture AARCH64. Refactor packager script. [#7370](https://github.com/ClickHouse/ClickHouse/pull/7370) [#7539](https://github.com/ClickHouse/ClickHouse/pull/7539) ([Ivan](https://github.com/abyss7)) +* Unpack darwin-x86_64 and linux-aarch64 toolchains into mounted Docker volume when building packages [#7534](https://github.com/ClickHouse/ClickHouse/pull/7534) ([Ivan](https://github.com/abyss7)) +* Update Docker Image for Binary Packager [#7474](https://github.com/ClickHouse/ClickHouse/pull/7474) ([Ivan](https://github.com/abyss7)) +* Fixed compile errors on MacOS Catalina [#7585](https://github.com/ClickHouse/ClickHouse/pull/7585) ([Ernest Poletaev](https://github.com/ernestp)) +* Some refactoring in query analysis logic: split complex class into several simple ones. [#7454](https://github.com/ClickHouse/ClickHouse/pull/7454) ([Artem Zuikov](https://github.com/4ertus2)) +* Fix build without submodules [#7295](https://github.com/ClickHouse/ClickHouse/pull/7295) ([proller](https://github.com/proller)) +* Better `add_globs` in CMake files [#7418](https://github.com/ClickHouse/ClickHouse/pull/7418) ([Amos Bird](https://github.com/amosbird)) +* Remove hardcoded paths in `unwind` target [#7460](https://github.com/ClickHouse/ClickHouse/pull/7460) ([Konstantin Podshumok](https://github.com/podshumok)) +* Allow to use mysql format without ssl [#7524](https://github.com/ClickHouse/ClickHouse/pull/7524) ([proller](https://github.com/proller)) + +### Other +* Added ANTLR4 grammar for ClickHouse SQL dialect [#7595](https://github.com/ClickHouse/ClickHouse/issues/7595) [#7596](https://github.com/ClickHouse/ClickHouse/pull/7596) ([alexey-milovidov](https://github.com/alexey-milovidov)) + + +## ClickHouse release v19.16.2.2, 2019-10-30 + +### Backward Incompatible Change +* Add missing arity validation for count/counIf. + [#7095](https://github.com/ClickHouse/ClickHouse/issues/7095) +[#7298](https://github.com/ClickHouse/ClickHouse/pull/7298) ([Vdimir](https://github.com/Vdimir)) +* Remove legacy `asterisk_left_columns_only` setting (it was disabled by default). + [#7335](https://github.com/ClickHouse/ClickHouse/pull/7335) ([Artem +Zuikov](https://github.com/4ertus2)) +* Format strings for Template data format are now specified in files. + [#7118](https://github.com/ClickHouse/ClickHouse/pull/7118) +([tavplubix](https://github.com/tavplubix)) + +### New Feature +* Introduce uniqCombined64() to calculate cardinality greater than UINT_MAX. + [#7213](https://github.com/ClickHouse/ClickHouse/pull/7213), +[#7222](https://github.com/ClickHouse/ClickHouse/pull/7222) ([Azat +Khuzhin](https://github.com/azat)) +* Support Bloom filter indexes on Array columns. + [#6984](https://github.com/ClickHouse/ClickHouse/pull/6984) +([achimbab](https://github.com/achimbab)) +* Add a function `getMacro(name)` that returns String with the value of corresponding `` + from server configuration. [#7240](https://github.com/ClickHouse/ClickHouse/pull/7240) +([alexey-milovidov](https://github.com/alexey-milovidov)) +* Set two configuration options for a dictionary based on an HTTP source: `credentials` and + `http-headers`. [#7092](https://github.com/ClickHouse/ClickHouse/pull/7092) ([Guillaume +Tassery](https://github.com/YiuRULE)) +* Add a new ProfileEvent `Merge` that counts the number of launched background merges. + [#7093](https://github.com/ClickHouse/ClickHouse/pull/7093) ([Mikhail +Korotov](https://github.com/millb)) +* Add fullHostName function that returns a fully qualified domain name. + [#7263](https://github.com/ClickHouse/ClickHouse/issues/7263) +[#7291](https://github.com/ClickHouse/ClickHouse/pull/7291) ([sundyli](https://github.com/sundy-li)) +* Add function `arraySplit` and `arrayReverseSplit` which split an array by "cut off" + conditions. They are useful in time sequence handling. +[#7294](https://github.com/ClickHouse/ClickHouse/pull/7294) ([hcz](https://github.com/hczhcz)) +* Add new functions that return the Array of all matched indices in multiMatch family of functions. + [#7299](https://github.com/ClickHouse/ClickHouse/pull/7299) ([Danila +Kutenin](https://github.com/danlark1)) +* Add a new database engine `Lazy` that is optimized for storing a large number of small -Log + tables. [#7171](https://github.com/ClickHouse/ClickHouse/pull/7171) ([Nikita +Vasilev](https://github.com/nikvas0)) +* Add aggregate functions groupBitmapAnd, -Or, -Xor for bitmap columns. [#7109](https://github.com/ClickHouse/ClickHouse/pull/7109) ([Zhichang +Yu](https://github.com/yuzhichang)) +* Add aggregate function combinators -OrNull and -OrDefault, which return null + or default values when there is nothing to aggregate. +[#7331](https://github.com/ClickHouse/ClickHouse/pull/7331) +([hcz](https://github.com/hczhcz)) +* Introduce CustomSeparated data format that supports custom escaping and + delimiter rules. [#7118](https://github.com/ClickHouse/ClickHouse/pull/7118) +([tavplubix](https://github.com/tavplubix)) +* Support Redis as source of external dictionary. [#4361](https://github.com/ClickHouse/ClickHouse/pull/4361) [#6962](https://github.com/ClickHouse/ClickHouse/pull/6962) ([comunodi](https://github.com/comunodi), [Anton +Popov](https://github.com/CurtizJ)) + +### Bug Fix +* Fix wrong query result if it has `WHERE IN (SELECT ...)` section and `optimize_read_in_order` is + used. [#7371](https://github.com/ClickHouse/ClickHouse/pull/7371) ([Anton +Popov](https://github.com/CurtizJ)) +* Disabled MariaDB authentication plugin, which depends on files outside of project. + [#7140](https://github.com/ClickHouse/ClickHouse/pull/7140) ([Yuriy +Baranov](https://github.com/yurriy)) +* Fix exception `Cannot convert column ... because it is constant but values of constants are + different in source and result` which could rarely happen when functions `now()`, `today()`, +`yesterday()`, `randConstant()` are used. +[#7156](https://github.com/ClickHouse/ClickHouse/pull/7156) ([Nikolai +Kochetov](https://github.com/KochetovNicolai)) +* Fixed issue of using HTTP keep alive timeout instead of TCP keep alive timeout. + [#7351](https://github.com/ClickHouse/ClickHouse/pull/7351) ([Vasily +Nemkov](https://github.com/Enmk)) +* Fixed a segmentation fault in groupBitmapOr (issue [#7109](https://github.com/ClickHouse/ClickHouse/issues/7109)). + [#7289](https://github.com/ClickHouse/ClickHouse/pull/7289) ([Zhichang +Yu](https://github.com/yuzhichang)) +* For materialized views the commit for Kafka is called after all data were written. + [#7175](https://github.com/ClickHouse/ClickHouse/pull/7175) ([Ivan](https://github.com/abyss7)) +* Fixed wrong `duration_ms` value in `system.part_log` table. It was ten times off. + [#7172](https://github.com/ClickHouse/ClickHouse/pull/7172) ([Vladimir +Chebotarev](https://github.com/excitoon)) +* A quick fix to resolve crash in LIVE VIEW table and re-enabling all LIVE VIEW tests. + [#7201](https://github.com/ClickHouse/ClickHouse/pull/7201) +([vzakaznikov](https://github.com/vzakaznikov)) +* Serialize NULL values correctly in min/max indexes of MergeTree parts. + [#7234](https://github.com/ClickHouse/ClickHouse/pull/7234) ([Alexander +Kuzmenkov](https://github.com/akuzm)) +* Don't put virtual columns to .sql metadata when table is created as `CREATE TABLE AS`. + [#7183](https://github.com/ClickHouse/ClickHouse/pull/7183) ([Ivan](https://github.com/abyss7)) +* Fix segmentation fault in `ATTACH PART` query. + [#7185](https://github.com/ClickHouse/ClickHouse/pull/7185) +([alesapin](https://github.com/alesapin)) +* Fix wrong result for some queries given by the optimization of empty IN subqueries and empty + INNER/RIGHT JOIN. [#7284](https://github.com/ClickHouse/ClickHouse/pull/7284) ([Nikolai +Kochetov](https://github.com/KochetovNicolai)) +* Fixing AddressSanitizer error in the LIVE VIEW getHeader() method. + [#7271](https://github.com/ClickHouse/ClickHouse/pull/7271) +([vzakaznikov](https://github.com/vzakaznikov)) + +### Improvement +* Add a message in case of queue_wait_max_ms wait takes place. + [#7390](https://github.com/ClickHouse/ClickHouse/pull/7390) ([Azat +Khuzhin](https://github.com/azat)) +* Made setting `s3_min_upload_part_size` table-level. + [#7059](https://github.com/ClickHouse/ClickHouse/pull/7059) ([Vladimir +Chebotarev](https://github.com/excitoon)) +* Check TTL in StorageFactory. [#7304](https://github.com/ClickHouse/ClickHouse/pull/7304) + ([sundyli](https://github.com/sundy-li)) +* Squash left-hand blocks in partial merge join (optimization). + [#7122](https://github.com/ClickHouse/ClickHouse/pull/7122) ([Artem +Zuikov](https://github.com/4ertus2)) +* Do not allow non-deterministic functions in mutations of Replicated table engines, because this + can introduce inconsistencies between replicas. +[#7247](https://github.com/ClickHouse/ClickHouse/pull/7247) ([Alexander +Kazakov](https://github.com/Akazz)) +* Disable memory tracker while converting exception stack trace to string. It can prevent the loss + of error messages of type `Memory limit exceeded` on server, which caused the `Attempt to read +after eof` exception on client. [#7264](https://github.com/ClickHouse/ClickHouse/pull/7264) +([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Miscellaneous format improvements. Resolves + [#6033](https://github.com/ClickHouse/ClickHouse/issues/6033), +[#2633](https://github.com/ClickHouse/ClickHouse/issues/2633), +[#6611](https://github.com/ClickHouse/ClickHouse/issues/6611), +[#6742](https://github.com/ClickHouse/ClickHouse/issues/6742) +[#7215](https://github.com/ClickHouse/ClickHouse/pull/7215) +([tavplubix](https://github.com/tavplubix)) +* ClickHouse ignores values on the right side of IN operator that are not convertible to the left + side type. Make it work properly for compound types -- Array and Tuple. +[#7283](https://github.com/ClickHouse/ClickHouse/pull/7283) ([Alexander +Kuzmenkov](https://github.com/akuzm)) +* Support missing inequalities for ASOF JOIN. It's possible to join less-or-equal variant and strict + greater and less variants for ASOF column in ON syntax. +[#7282](https://github.com/ClickHouse/ClickHouse/pull/7282) ([Artem +Zuikov](https://github.com/4ertus2)) +* Optimize partial merge join. [#7070](https://github.com/ClickHouse/ClickHouse/pull/7070) + ([Artem Zuikov](https://github.com/4ertus2)) +* Do not use more than 98K of memory in uniqCombined functions. + [#7236](https://github.com/ClickHouse/ClickHouse/pull/7236), +[#7270](https://github.com/ClickHouse/ClickHouse/pull/7270) ([Azat +Khuzhin](https://github.com/azat)) +* Flush parts of right-hand joining table on disk in PartialMergeJoin (if there is not enough + memory). Load data back when needed. [#7186](https://github.com/ClickHouse/ClickHouse/pull/7186) +([Artem Zuikov](https://github.com/4ertus2)) + +### Performance Improvement +* Speed up joinGet with const arguments by avoiding data duplication. + [#7359](https://github.com/ClickHouse/ClickHouse/pull/7359) ([Amos +Bird](https://github.com/amosbird)) +* Return early if the subquery is empty. + [#7007](https://github.com/ClickHouse/ClickHouse/pull/7007) ([小路](https://github.com/nicelulu)) +* Optimize parsing of SQL expression in Values. + [#6781](https://github.com/ClickHouse/ClickHouse/pull/6781) +([tavplubix](https://github.com/tavplubix)) + +### Build/Testing/Packaging Improvement +* Disable some contribs for cross-compilation to Mac OS. + [#7101](https://github.com/ClickHouse/ClickHouse/pull/7101) ([Ivan](https://github.com/abyss7)) +* Add missing linking with PocoXML for clickhouse_common_io. + [#7200](https://github.com/ClickHouse/ClickHouse/pull/7200) ([Azat +Khuzhin](https://github.com/azat)) +* Accept multiple test filter arguments in clickhouse-test. + [#7226](https://github.com/ClickHouse/ClickHouse/pull/7226) ([Alexander +Kuzmenkov](https://github.com/akuzm)) +* Enable musl and jemalloc for ARM. [#7300](https://github.com/ClickHouse/ClickHouse/pull/7300) + ([Amos Bird](https://github.com/amosbird)) +* Added `--client-option` parameter to `clickhouse-test` to pass additional parameters to client. + [#7277](https://github.com/ClickHouse/ClickHouse/pull/7277) ([Nikolai +Kochetov](https://github.com/KochetovNicolai)) +* Preserve existing configs on rpm package upgrade. + [#7103](https://github.com/ClickHouse/ClickHouse/pull/7103) +([filimonov](https://github.com/filimonov)) +* Fix errors detected by PVS. [#7153](https://github.com/ClickHouse/ClickHouse/pull/7153) ([Artem + Zuikov](https://github.com/4ertus2)) +* Fix build for Darwin. [#7149](https://github.com/ClickHouse/ClickHouse/pull/7149) + ([Ivan](https://github.com/abyss7)) +* glibc 2.29 compatibility. [#7142](https://github.com/ClickHouse/ClickHouse/pull/7142) ([Amos + Bird](https://github.com/amosbird)) +* Make sure dh_clean does not touch potential source files. + [#7205](https://github.com/ClickHouse/ClickHouse/pull/7205) ([Amos +Bird](https://github.com/amosbird)) +* Attempt to avoid conflict when updating from altinity rpm - it has config file packaged separately + in clickhouse-server-common. [#7073](https://github.com/ClickHouse/ClickHouse/pull/7073) +([filimonov](https://github.com/filimonov)) +* Optimize some header files for faster rebuilds. + [#7212](https://github.com/ClickHouse/ClickHouse/pull/7212), +[#7231](https://github.com/ClickHouse/ClickHouse/pull/7231) ([Alexander +Kuzmenkov](https://github.com/akuzm)) +* Add performance tests for Date and DateTime. [#7332](https://github.com/ClickHouse/ClickHouse/pull/7332) ([Vasily + Nemkov](https://github.com/Enmk)) +* Fix some tests that contained non-deterministic mutations. + [#7132](https://github.com/ClickHouse/ClickHouse/pull/7132) ([Alexander +Kazakov](https://github.com/Akazz)) +* Add build with MemorySanitizer to CI. [#7066](https://github.com/ClickHouse/ClickHouse/pull/7066) + ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Avoid use of uninitialized values in MetricsTransmitter. + [#7158](https://github.com/ClickHouse/ClickHouse/pull/7158) ([Azat +Khuzhin](https://github.com/azat)) +* Fix some issues in Fields found by MemorySanitizer. + [#7135](https://github.com/ClickHouse/ClickHouse/pull/7135), +[#7179](https://github.com/ClickHouse/ClickHouse/pull/7179) ([Alexander +Kuzmenkov](https://github.com/akuzm)), [#7376](https://github.com/ClickHouse/ClickHouse/pull/7376) +([Amos Bird](https://github.com/amosbird)) +* Fix undefined behavior in murmurhash32. [#7388](https://github.com/ClickHouse/ClickHouse/pull/7388) ([Amos + Bird](https://github.com/amosbird)) +* Fix undefined behavior in StoragesInfoStream. [#7384](https://github.com/ClickHouse/ClickHouse/pull/7384) + ([tavplubix](https://github.com/tavplubix)) +* Fixed constant expressions folding for external database engines (MySQL, ODBC, JDBC). In previous + versions it wasn't working for multiple constant expressions and was not working at all for Date, +DateTime and UUID. This fixes [#7245](https://github.com/ClickHouse/ClickHouse/issues/7245) +[#7252](https://github.com/ClickHouse/ClickHouse/pull/7252) +([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixing ThreadSanitizer data race error in the LIVE VIEW when accessing no_users_thread variable. + [#7353](https://github.com/ClickHouse/ClickHouse/pull/7353) +([vzakaznikov](https://github.com/vzakaznikov)) +* Get rid of malloc symbols in libcommon + [#7134](https://github.com/ClickHouse/ClickHouse/pull/7134), +[#7065](https://github.com/ClickHouse/ClickHouse/pull/7065) ([Amos +Bird](https://github.com/amosbird)) +* Add global flag ENABLE_LIBRARIES for disabling all libraries. + [#7063](https://github.com/ClickHouse/ClickHouse/pull/7063) +([proller](https://github.com/proller)) + +### Code cleanup +* Generalize configuration repository to prepare for DDL for Dictionaries. [#7155](https://github.com/ClickHouse/ClickHouse/pull/7155) + ([alesapin](https://github.com/alesapin)) +* Parser for dictionaries DDL without any semantic. + [#7209](https://github.com/ClickHouse/ClickHouse/pull/7209) +([alesapin](https://github.com/alesapin)) +* Split ParserCreateQuery into different smaller parsers. + [#7253](https://github.com/ClickHouse/ClickHouse/pull/7253) +([alesapin](https://github.com/alesapin)) +* Small refactoring and renaming near external dictionaries. + [#7111](https://github.com/ClickHouse/ClickHouse/pull/7111) +([alesapin](https://github.com/alesapin)) +* Refactor some code to prepare for role-based access control. [#7235](https://github.com/ClickHouse/ClickHouse/pull/7235) ([Vitaly + Baranov](https://github.com/vitlibar)) +* Some improvements in DatabaseOrdinary code. + [#7086](https://github.com/ClickHouse/ClickHouse/pull/7086) ([Nikita +Vasilev](https://github.com/nikvas0)) +* Do not use iterators in find() and emplace() methods of hash tables. +[#7026](https://github.com/ClickHouse/ClickHouse/pull/7026) ([Alexander +Kuzmenkov](https://github.com/akuzm)) +* Fix getMultipleValuesFromConfig in case when parameter root is not empty. [#7374](https://github.com/ClickHouse/ClickHouse/pull/7374) +([Mikhail Korotov](https://github.com/millb)) +* Remove some copy-paste (TemporaryFile and TemporaryFileStream) + [#7166](https://github.com/ClickHouse/ClickHouse/pull/7166) ([Artem +Zuikov](https://github.com/4ertus2)) +* Improved code readability a little bit (`MergeTreeData::getActiveContainingPart`). + [#7361](https://github.com/ClickHouse/ClickHouse/pull/7361) ([Vladimir +Chebotarev](https://github.com/excitoon)) +* Wait for all scheduled jobs, which are using local objects, if `ThreadPool::schedule(...)` throws + an exception. Rename `ThreadPool::schedule(...)` to `ThreadPool::scheduleOrThrowOnError(...)` and +fix comments to make obvious that it may throw. +[#7350](https://github.com/ClickHouse/ClickHouse/pull/7350) +([tavplubix](https://github.com/tavplubix)) + +## ClickHouse release 19.15.4.10, 2019-10-31 + +### Bug Fix +* Added handling of SQL_TINYINT and SQL_BIGINT, and fix handling of SQL_FLOAT data source types in ODBC Bridge. +[#7491](https://github.com/ClickHouse/ClickHouse/pull/7491) ([Denis Glazachev](https://github.com/traceon)) +* Allowed to have some parts on destination disk or volume in MOVE PARTITION. +[#7434](https://github.com/ClickHouse/ClickHouse/pull/7434) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fixed NULL-values in nullable columns through ODBC-bridge. +[#7402](https://github.com/ClickHouse/ClickHouse/pull/7402) ([Vasily Nemkov](https://github.com/Enmk)) +* Fixed INSERT into Distributed non local node with MATERIALIZED columns. +[#7377](https://github.com/ClickHouse/ClickHouse/pull/7377) ([Azat Khuzhin](https://github.com/azat)) +* Fixed function getMultipleValuesFromConfig. +[#7374](https://github.com/ClickHouse/ClickHouse/pull/7374) ([Mikhail Korotov](https://github.com/millb)) +* Fixed issue of using HTTP keep alive timeout instead of TCP keep alive timeout. +[#7351](https://github.com/ClickHouse/ClickHouse/pull/7351) ([Vasily Nemkov](https://github.com/Enmk)) +* Wait for all jobs to finish on exception (fixes rare segfaults). +[#7350](https://github.com/ClickHouse/ClickHouse/pull/7350) ([tavplubix](https://github.com/tavplubix)) +* Don't push to MVs when inserting into Kafka table. +[#7265](https://github.com/ClickHouse/ClickHouse/pull/7265) ([Ivan](https://github.com/abyss7)) +* Disable memory tracker for exception stack. +[#7264](https://github.com/ClickHouse/ClickHouse/pull/7264) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fixed bad code in transforming query for external database. +[#7252](https://github.com/ClickHouse/ClickHouse/pull/7252) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Avoid use of uninitialized values in MetricsTransmitter. +[#7158](https://github.com/ClickHouse/ClickHouse/pull/7158) ([Azat Khuzhin](https://github.com/azat)) +* Added example config with macros for tests ([alexey-milovidov](https://github.com/alexey-milovidov)) + +## ClickHouse release 19.15.3.6, 2019-10-09 + +### Bug Fix +* Fixed bad_variant in hashed dictionary. +([alesapin](https://github.com/alesapin)) +* Fixed up bug with segmentation fault in ATTACH PART query. +([alesapin](https://github.com/alesapin)) +* Fixed time calculation in `MergeTreeData`. +([Vladimir Chebotarev](https://github.com/excitoon)) +* Commit to Kafka explicitly after the writing is finalized. +[#7175](https://github.com/ClickHouse/ClickHouse/pull/7175) ([Ivan](https://github.com/abyss7)) +* Serialize NULL values correctly in min/max indexes of MergeTree parts. +[#7234](https://github.com/ClickHouse/ClickHouse/pull/7234) ([Alexander Kuzmenkov](https://github.com/akuzm)) + ## ClickHouse release 19.15.2.2, 2019-10-01 ### New Feature @@ -105,7 +477,7 @@ * Fix segfault with enabled `optimize_skip_unused_shards` and missing sharding key. [#6384](https://github.com/ClickHouse/ClickHouse/pull/6384) ([Anton Popov](https://github.com/CurtizJ)) * Fixed wrong code in mutations that may lead to memory corruption. Fixed segfault with read of address `0x14c0` that may happed due to concurrent `DROP TABLE` and `SELECT` from `system.parts` or `system.parts_columns`. Fixed race condition in preparation of mutation queries. Fixed deadlock caused by `OPTIMIZE` of Replicated tables and concurrent modification operations like ALTERs. [#6514](https://github.com/ClickHouse/ClickHouse/pull/6514) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Removed extra verbose logging in MySQL interface [#6389](https://github.com/ClickHouse/ClickHouse/pull/6389) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Return ability to parse boolean settings from 'true' and 'false' in configuration file. [#6278](https://github.com/ClickHouse/ClickHouse/pull/6278) ([alesapin](https://github.com/alesapin)) +* Return the ability to parse boolean settings from 'true' and 'false' in the configuration file. [#6278](https://github.com/ClickHouse/ClickHouse/pull/6278) ([alesapin](https://github.com/alesapin)) * Fix crash in `quantile` and `median` function over `Nullable(Decimal128)`. [#6378](https://github.com/ClickHouse/ClickHouse/pull/6378) ([Artem Zuikov](https://github.com/4ertus2)) * Fixed possible incomplete result returned by `SELECT` query with `WHERE` condition on primary key contained conversion to Float type. It was caused by incorrect checking of monotonicity in `toFloat` function. [#6248](https://github.com/ClickHouse/ClickHouse/issues/6248) [#6374](https://github.com/ClickHouse/ClickHouse/pull/6374) ([dimarub2000](https://github.com/dimarub2000)) * Check `max_expanded_ast_elements` setting for mutations. Clear mutations after `TRUNCATE TABLE`. [#6205](https://github.com/ClickHouse/ClickHouse/pull/6205) ([Winter Zhang](https://github.com/zhang2014)) @@ -133,8 +505,8 @@ * Fix bug with writing secondary indices marks with adaptive granularity. [#6126](https://github.com/ClickHouse/ClickHouse/pull/6126) ([alesapin](https://github.com/alesapin)) * Fix initialization order while server startup. Since `StorageMergeTree::background_task_handle` is initialized in `startup()` the `MergeTreeBlockOutputStream::write()` may try to use it before initialization. Just check if it is initialized. [#6080](https://github.com/ClickHouse/ClickHouse/pull/6080) ([Ivan](https://github.com/abyss7)) * Clearing the data buffer from the previous read operation that was completed with an error. [#6026](https://github.com/ClickHouse/ClickHouse/pull/6026) ([Nikolay](https://github.com/bopohaa)) -* Fix bug with enabling adaptive granularity when creating new replica for Replicated*MergeTree table. [#6394](https://github.com/ClickHouse/ClickHouse/issues/6394) [#6452](https://github.com/ClickHouse/ClickHouse/pull/6452) ([alesapin](https://github.com/alesapin)) -* Fixed possible crash during server startup in case of exception happened in `libunwind` during exception at access to uninitialised `ThreadStatus` structure. [#6456](https://github.com/ClickHouse/ClickHouse/pull/6456) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Fix bug with enabling adaptive granularity when creating a new replica for Replicated*MergeTree table. [#6394](https://github.com/ClickHouse/ClickHouse/issues/6394) [#6452](https://github.com/ClickHouse/ClickHouse/pull/6452) ([alesapin](https://github.com/alesapin)) +* Fixed possible crash during server startup in case of exception happened in `libunwind` during exception at access to uninitialized `ThreadStatus` structure. [#6456](https://github.com/ClickHouse/ClickHouse/pull/6456) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) * Fix crash in `yandexConsistentHash` function. Found by fuzz test. [#6304](https://github.com/ClickHouse/ClickHouse/issues/6304) [#6305](https://github.com/ClickHouse/ClickHouse/pull/6305) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fixed the possibility of hanging queries when server is overloaded and global thread pool becomes near full. This have higher chance to happen on clusters with large number of shards (hundreds), because distributed queries allocate a thread per connection to each shard. For example, this issue may reproduce if a cluster of 330 shards is processing 30 concurrent distributed queries. This issue affects all versions starting from 19.2. [#6301](https://github.com/ClickHouse/ClickHouse/pull/6301) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fixed logic of `arrayEnumerateUniqRanked` function. [#6423](https://github.com/ClickHouse/ClickHouse/pull/6423) ([alexey-milovidov](https://github.com/alexey-milovidov)) @@ -307,7 +679,7 @@ ### Backward Incompatible Change * Removed rarely used table function `catBoostPool` and storage `CatBoostPool`. If you have used this table function, please write email to `clickhouse-feedback@yandex-team.com`. Note that CatBoost integration remains and will be supported. [#6279](https://github.com/ClickHouse/ClickHouse/pull/6279) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Disable `ANY RIGHT JOIN` and `ANY FULL JOIN` by default. Set `any_join_get_any_from_right_table` setting to enable them. [#5126](https://github.com/ClickHouse/ClickHouse/issues/5126) [#6351](https://github.com/ClickHouse/ClickHouse/pull/6351) ([Artem Zuikov](https://github.com/4ertus2)) +* Disable `ANY RIGHT JOIN` and `ANY FULL JOIN` by default. Set `any_join_distinct_right_table_keys` setting to enable them. [#5126](https://github.com/ClickHouse/ClickHouse/issues/5126) [#6351](https://github.com/ClickHouse/ClickHouse/pull/6351) ([Artem Zuikov](https://github.com/4ertus2)) ## ClickHouse release 19.13.6.51, 2019-10-02 @@ -345,6 +717,13 @@ ### Security Fix * Fix two vulnerabilities in codecs in decompression phase (malicious user can fabricate compressed data that will lead to buffer overflow in decompression). [#6670](https://github.com/ClickHouse/ClickHouse/pull/6670) ([Artem Zuikov](https://github.com/4ertus2)) +## ClickHouse release 19.11.13.74, 2019-11-01 + +### Bug Fix +* Fixed rare crash in `ALTER MODIFY COLUMN` and vertical merge when one of merged/altered parts is empty (0 rows). [#6780](https://github.com/ClickHouse/ClickHouse/pull/6780) ([alesapin](https://github.com/alesapin)) +* Manual update of `SIMDJSON`. This fixes possible flooding of stderr files with bogus json diagnostic messages. [#7548](https://github.com/ClickHouse/ClickHouse/pull/7548) ([Alexander Kazakov](https://github.com/Akazz)) +* Fixed bug with `mrk` file extension for mutations ([alesapin](https://github.com/alesapin)) + ## ClickHouse release 19.11.12.69, 2019-10-02 ### Bug Fix @@ -371,7 +750,7 @@ * Fix kafka tests. [#6805](https://github.com/ClickHouse/ClickHouse/pull/6805) ([Ivan](https://github.com/abyss7)) ### Security Fix -* If the attacker has write access to ZooKeeper and is able to run custom server available from the network where ClickHouse run, it can create custom-built malicious server that will act as ClickHouse replica and register it in ZooKeeper. When another replica will fetch data part from malicious replica, it can force clickhouse-server to write to arbitrary path on filesystem. Found by Eldar Zaitov, information security team at Yandex. [#6247](https://github.com/ClickHouse/ClickHouse/pull/6247) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* If the attacker has write access to ZooKeeper and is able to run custom server available from the network where ClickHouse runs, it can create custom-built malicious server that will act as ClickHouse replica and register it in ZooKeeper. When another replica will fetch data part from malicious replica, it can force clickhouse-server to write to arbitrary path on filesystem. Found by Eldar Zaitov, information security team at Yandex. [#6247](https://github.com/ClickHouse/ClickHouse/pull/6247) ([alexey-milovidov](https://github.com/alexey-milovidov)) ## ClickHouse release 19.13.3.26, 2019-08-22 @@ -399,7 +778,7 @@ * Now client receive logs from server with any desired level by setting `send_logs_level` regardless to the log level specified in server settings. [#5964](https://github.com/ClickHouse/ClickHouse/pull/5964) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) ### Backward Incompatible Change -* The setting `input_format_defaults_for_omitted_fields` is enabled by default. Inserts in Distibuted tables need this setting to be the same on cluster (you need to set it before rolling update). It enables calculation of complex default expressions for omitted fields in `JSONEachRow` and `CSV*` formats. It should be the expected behaviour but may lead to negligible performance difference. [#6043](https://github.com/ClickHouse/ClickHouse/pull/6043) ([Artem Zuikov](https://github.com/4ertus2)), [#5625](https://github.com/ClickHouse/ClickHouse/pull/5625) ([akuzm](https://github.com/akuzm)) +* The setting `input_format_defaults_for_omitted_fields` is enabled by default. Inserts in Distributed tables need this setting to be the same on cluster (you need to set it before rolling update). It enables calculation of complex default expressions for omitted fields in `JSONEachRow` and `CSV*` formats. It should be the expected behavior but may lead to negligible performance difference. [#6043](https://github.com/ClickHouse/ClickHouse/pull/6043) ([Artem Zuikov](https://github.com/4ertus2)), [#5625](https://github.com/ClickHouse/ClickHouse/pull/5625) ([akuzm](https://github.com/akuzm)) ### Experimental features * New query processing pipeline. Use `experimental_use_processors=1` option to enable it. Use for your own trouble. [#4914](https://github.com/ClickHouse/ClickHouse/pull/4914) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) @@ -1180,7 +1559,7 @@ lee](https://github.com/neverlee)) ### Bug fixes -* Fixed error in #3920. This error manifestate itself as random cache corruption (messages `Unknown codec family code`, `Cannot seek through file`) and segfaults. This bug first appeared in version 19.1 and is present in versions up to 19.1.10 and 19.3.6. [#4623](https://github.com/ClickHouse/ClickHouse/pull/4623) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed error in #3920. This error manifests itself as random cache corruption (messages `Unknown codec family code`, `Cannot seek through file`) and segfaults. This bug first appeared in version 19.1 and is present in versions up to 19.1.10 and 19.3.6. [#4623](https://github.com/ClickHouse/ClickHouse/pull/4623) ([alexey-milovidov](https://github.com/alexey-milovidov)) ## ClickHouse release 19.3.6, 2019-03-02 @@ -2037,7 +2416,7 @@ The expression must be a chain of equalities joined by the AND operator. Each si ### Improvements: -* Changed the numbering scheme for release versions. Now the first part contains the year of release (A.D., Moscow timezone, minus 2000), the second part contains the number for major changes (increases for most releases), and the third part is the patch version. Releases are still backwards compatible, unless otherwise stated in the changelog. +* Changed the numbering scheme for release versions. Now the first part contains the year of release (A.D., Moscow timezone, minus 2000), the second part contains the number for major changes (increases for most releases), and the third part is the patch version. Releases are still backward compatible, unless otherwise stated in the changelog. * Faster conversions of floating-point numbers to a string ([Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/2664)). * If some rows were skipped during an insert due to parsing errors (this is possible with the `input_allow_errors_num` and `input_allow_errors_ratio` settings enabled), the number of skipped rows is now written to the server log ([Leonardo Cecchi](https://github.com/ClickHouse/ClickHouse/pull/2669)). @@ -2236,7 +2615,7 @@ The expression must be a chain of equalities joined by the AND operator. Each si * Configuration of the table level for the `ReplicatedMergeTree` family in order to minimize the amount of data stored in Zookeeper: : `use_minimalistic_checksums_in_zookeeper = 1` * Configuration of the `clickhouse-client` prompt. By default, server names are now output to the prompt. The server's display name can be changed. It's also sent in the `X-ClickHouse-Display-Name` HTTP header (Kirill Shvakov). * Multiple comma-separated `topics` can be specified for the `Kafka` engine (Tobias Adamson) -* When a query is stopped by `KILL QUERY` or `replace_running_query`, the client receives the `Query was cancelled` exception instead of an incomplete result. +* When a query is stopped by `KILL QUERY` or `replace_running_query`, the client receives the `Query was canceled` exception instead of an incomplete result. ### Improvements: diff --git a/CMakeLists.txt b/CMakeLists.txt index 986096ba9e8..c6ae23c0955 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -198,14 +198,21 @@ if(WITH_COVERAGE AND COMPILER_GCC) endif() set (CMAKE_BUILD_COLOR_MAKEFILE ON) -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS}") +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS}") set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_CXX_FLAGS_ADD}") set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_CXX_FLAGS_ADD}") -set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CMAKE_C_FLAGS_ADD}") +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} ${COMMON_WARNING_FLAGS} ${CMAKE_C_FLAGS_ADD}") set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_C_FLAGS_ADD}") set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}") +if (COMPILER_CLANG) + # Exception unwinding doesn't work in clang release build without this option + # TODO investigate if contrib/libcxxabi is out of date + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer") +endif () + option (ENABLE_LIBRARIES "Enable all libraries (Global default switch)" ON) option (UNBUNDLED "Try find all libraries in system. We recommend to avoid this mode for production builds, because we cannot guarantee exact versions and variants of libraries your system has installed. This mode exists for enthusiastic developers who search for trouble. Also it is useful for maintainers of OS packages." OFF) @@ -215,7 +222,7 @@ else () set(NOT_UNBUNDLED 1) endif () -# Using system libs can cause lot of warnings in includes. +# Using system libs can cause lot of warnings in includes (on macro expansion). if (UNBUNDLED OR NOT (OS_LINUX OR APPLE) OR ARCH_32) option (NO_WERROR "Disable -Werror compiler option" ON) endif () @@ -325,6 +332,7 @@ include (cmake/find/brotli.cmake) include (cmake/find/protobuf.cmake) include (cmake/find/pdqsort.cmake) include (cmake/find/hdfs3.cmake) # uses protobuf +include (cmake/find/s3.cmake) include (cmake/find/consistent-hashing.cmake) include (cmake/find/base64.cmake) include (cmake/find/parquet.cmake) @@ -344,7 +352,6 @@ if (ENABLE_TESTS) endif () # Need to process before "contrib" dir: -include (libs/libcommon/cmake/find_gperftools.cmake) include (libs/libcommon/cmake/find_jemalloc.cmake) include (libs/libcommon/cmake/find_cctz.cmake) include (libs/libmysqlxx/cmake/find_mysqlclient.cmake) @@ -354,18 +361,6 @@ include (libs/libmysqlxx/cmake/find_mysqlclient.cmake) if (USE_JEMALLOC) message (STATUS "Link jemalloc: ${JEMALLOC_LIBRARIES}") set (MALLOC_LIBRARIES ${JEMALLOC_LIBRARIES}) -elseif (USE_TCMALLOC) - if (DEBUG_TCMALLOC AND NOT GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG) - message (FATAL_ERROR "Requested DEBUG_TCMALLOC but debug library is not found. You should install Google Perftools. Example: sudo apt-get install libgoogle-perftools-dev") - endif () - - if (DEBUG_TCMALLOC AND GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG) - message (STATUS "Link libtcmalloc_minimal_debug for testing: ${GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG}") - set (MALLOC_LIBRARIES ${GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG}) - else () - message (STATUS "Link libtcmalloc_minimal: ${GPERFTOOLS_TCMALLOC_MINIMAL}") - set (MALLOC_LIBRARIES ${GPERFTOOLS_TCMALLOC_MINIMAL}) - endif () elseif (SANITIZE) message (STATUS "Will use ${SANITIZE} sanitizer.") elseif (OS_LINUX) @@ -382,16 +377,12 @@ add_subdirectory (contrib EXCLUDE_FROM_ALL) macro (add_executable target) # invoke built-in add_executable - _add_executable (${ARGV}) + # explicitly acquire and interpose malloc symbols by clickhouse_malloc + _add_executable (${ARGV} $) get_target_property (type ${target} TYPE) if (${type} STREQUAL EXECUTABLE) - file (RELATIVE_PATH dir ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) - if (${dir} MATCHES "^dbms") - # Only interpose operator::new/delete for dbms executables (MemoryTracker stuff) - target_link_libraries (${target} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES}) - else () - target_link_libraries (${target} PRIVATE ${MALLOC_LIBRARIES}) - endif () + # operator::new/delete for executables (MemoryTracker stuff) + target_link_libraries (${target} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES}) endif() endmacro() diff --git a/README.md b/README.md index 08be0b9ed07..a545c91886f 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,3 @@ ClickHouse is an open-source column-oriented database management system that all * [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events. * [Contacts](https://clickhouse.yandex/#contacts) can help to get your questions answered if there are any. * You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person. - -## Upcoming Events -* [ClickHouse Meetup in Tokyo](https://clickhouse.connpass.com/event/147001/) on November 14. -* [ClickHouse Meetup in Istanbul](https://www.eventbrite.com/e/clickhouse-meetup-istanbul-create-blazing-fast-experiences-w-clickhouse-tickets-73101120419) on November 19. -* [ClickHouse Meetup in Ankara](https://www.eventbrite.com/e/clickhouse-meetup-ankara-create-blazing-fast-experiences-w-clickhouse-tickets-73100530655) on November 21. -* [ClickHouse Meetup in Singapore](https://www.meetup.com/Singapore-Clickhouse-Meetup-Group/events/265085331/) on November 23. -* [ClickHouse Meetup in San Francisco](https://www.eventbrite.com/e/clickhouse-december-meetup-registration-78642047481) on December 3. - diff --git a/cmake/Modules/FindGperftools.cmake b/cmake/Modules/FindGperftools.cmake deleted file mode 100644 index 1cb8d42343f..00000000000 --- a/cmake/Modules/FindGperftools.cmake +++ /dev/null @@ -1,61 +0,0 @@ -# https://github.com/vast-io/vast/blob/master/cmake/FindGperftools.cmake - -# Tries to find Gperftools. -# -# Usage of this module as follows: -# -# find_package(Gperftools) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# Gperftools_ROOT_DIR Set this variable to the root installation of -# Gperftools if the module has problems finding -# the proper installation path. -# -# Variables defined by this module: -# -# GPERFTOOLS_FOUND System has Gperftools libs/headers -# GPERFTOOLS_LIBRARIES The Gperftools libraries (tcmalloc & profiler) -# GPERFTOOLS_INCLUDE_DIR The location of Gperftools headers - -find_library(GPERFTOOLS_TCMALLOC - NAMES tcmalloc - HINTS ${Gperftools_ROOT_DIR}/lib) - -find_library(GPERFTOOLS_TCMALLOC_MINIMAL - NAMES tcmalloc_minimal - HINTS ${Gperftools_ROOT_DIR}/lib) - -find_library(GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG - NAMES tcmalloc_minimal_debug - HINTS ${Gperftools_ROOT_DIR}/lib) - -find_library(GPERFTOOLS_PROFILER - NAMES profiler - HINTS ${Gperftools_ROOT_DIR}/lib) - -find_library(GPERFTOOLS_TCMALLOC_AND_PROFILER - NAMES tcmalloc_and_profiler - HINTS ${Gperftools_ROOT_DIR}/lib) - -find_path(GPERFTOOLS_INCLUDE_DIR - NAMES gperftools/heap-profiler.h - HINTS ${Gperftools_ROOT_DIR}/include) - -set(GPERFTOOLS_LIBRARIES ${GPERFTOOLS_TCMALLOC_AND_PROFILER}) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args( - Gperftools - DEFAULT_MSG - GPERFTOOLS_LIBRARIES - GPERFTOOLS_INCLUDE_DIR) - -mark_as_advanced( - Gperftools_ROOT_DIR - GPERFTOOLS_TCMALLOC - GPERFTOOLS_PROFILER - GPERFTOOLS_TCMALLOC_AND_PROFILER - GPERFTOOLS_LIBRARIES - GPERFTOOLS_INCLUDE_DIR) diff --git a/cmake/arch.cmake b/cmake/arch.cmake index 79fe92c03e5..ec644b6fe77 100644 --- a/cmake/arch.cmake +++ b/cmake/arch.cmake @@ -19,6 +19,6 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64le.*|PPC64LE.*)") set (ARCH_PPC64LE 1) # FIXME: move this check into tools.cmake if (COMPILER_CLANG OR (COMPILER_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8)) - message(FATAL_ERROR "Only gcc-8 is supported for powerpc architecture") + message(FATAL_ERROR "Only gcc-8 or higher is supported for powerpc architecture") endif () endif () diff --git a/cmake/find/capnp.cmake b/cmake/find/capnp.cmake index 1ac6241a5f1..0620a66808b 100644 --- a/cmake/find/capnp.cmake +++ b/cmake/find/capnp.cmake @@ -4,6 +4,14 @@ if (ENABLE_CAPNP) option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library instead of bundled" ${NOT_UNBUNDLED}) +if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/capnproto/CMakeLists.txt") + if(USE_INTERNAL_CAPNP_LIBRARY) + message(WARNING "submodule contrib/capnproto is missing. to fix try run: \n git submodule update --init --recursive") + endif() + set(MISSING_INTERNAL_CAPNP_LIBRARY 1) + set(USE_INTERNAL_CAPNP_LIBRARY 0) +endif() + # FIXME: refactor to use `add_library(… IMPORTED)` if possible. if (NOT USE_INTERNAL_CAPNP_LIBRARY) find_library (KJ kj) @@ -11,7 +19,7 @@ if (NOT USE_INTERNAL_CAPNP_LIBRARY) find_library (CAPNPC capnpc) set (CAPNP_LIBRARIES ${CAPNPC} ${CAPNP} ${KJ}) -else () +elseif(NOT MISSING_INTERNAL_CAPNP_LIBRARY) add_subdirectory(contrib/capnproto-cmake) set (CAPNP_LIBRARIES capnpc) @@ -23,4 +31,4 @@ endif () endif () -message (STATUS "Using capnp: ${CAPNP_LIBRARIES}") +message (STATUS "Using capnp=${USE_CAPNP}: ${CAPNP_LIBRARIES}") diff --git a/cmake/find/icu.cmake b/cmake/find/icu.cmake index 9062b573514..8ebe2f9befd 100644 --- a/cmake/find/icu.cmake +++ b/cmake/find/icu.cmake @@ -1,6 +1,18 @@ option(ENABLE_ICU "Enable ICU" ${ENABLE_LIBRARIES}) -if(ENABLE_ICU) +if (ENABLE_ICU) + +option (USE_INTERNAL_ICU_LIBRARY "Set to FALSE to use system ICU library instead of bundled" ${NOT_UNBUNDLED}) + +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/icu/icu4c/LICENSE") + if (USE_INTERNAL_ICU_LIBRARY) + message (WARNING "submodule contrib/icu is missing. to fix try run: \n git submodule update --init --recursive") + set (USE_INTERNAL_ICU_LIBRARY 0) + endif () + set (MISSING_INTERNAL_ICU_LIBRARY 1) +endif () + +if(NOT USE_INTERNAL_ICU_LIBRARY) if (APPLE) set(ICU_ROOT "/usr/local/opt/icu4c" CACHE STRING "") endif() @@ -11,6 +23,16 @@ if(ENABLE_ICU) endif() endif() +if (ICU_LIBRARY AND ICU_INCLUDE_DIR) + set (USE_ICU 1) +elseif (NOT MISSING_INTERNAL_ICU_LIBRARY) + set (USE_INTERNAL_ICU_LIBRARY 1) + set (ICU_LIBRARIES icui18n icuuc icudata) + set (USE_ICU 1) +endif () + +endif() + if(USE_ICU) message(STATUS "Using icu=${USE_ICU}: ${ICU_INCLUDE_DIR} : ${ICU_LIBRARIES}") else() diff --git a/cmake/find/llvm.cmake b/cmake/find/llvm.cmake index 8dfd26ec581..7cb67d1a990 100644 --- a/cmake/find/llvm.cmake +++ b/cmake/find/llvm.cmake @@ -1,7 +1,7 @@ # Broken in macos. TODO: update clang, re-test, enable if (NOT APPLE) - option (ENABLE_EMBEDDED_COMPILER "Set to TRUE to enable support for 'compile' option for query execution" ${ENABLE_LIBRARIES}) - option (USE_INTERNAL_LLVM_LIBRARY "Use bundled or system LLVM library. Default: system library for quicker developer builds." 0) + option (ENABLE_EMBEDDED_COMPILER "Set to TRUE to enable support for 'compile_expressions' option for query execution" ${ENABLE_LIBRARIES}) + option (USE_INTERNAL_LLVM_LIBRARY "Use bundled or system LLVM library." ${NOT_UNBUNDLED}) endif () if (ENABLE_EMBEDDED_COMPILER) @@ -13,27 +13,11 @@ if (ENABLE_EMBEDDED_COMPILER) if (NOT USE_INTERNAL_LLVM_LIBRARY) set (LLVM_PATHS "/usr/local/lib/llvm") - if (LLVM_VERSION) - find_package(LLVM ${LLVM_VERSION} CONFIG PATHS ${LLVM_PATHS}) - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - find_package(LLVM ${CMAKE_CXX_COMPILER_VERSION} CONFIG PATHS ${LLVM_PATHS}) - else () - # TODO: 9 8 - foreach(llvm_v 7.1 7 6 5) - if (NOT LLVM_FOUND) - find_package (LLVM ${llvm_v} CONFIG PATHS ${LLVM_PATHS}) - endif () - endforeach () - endif () - - if (LLVM_FOUND) - find_library (LLD_LIBRARY_TEST lldCore PATHS ${LLVM_LIBRARY_DIRS}) - find_path (LLD_INCLUDE_DIR_TEST NAMES lld/Core/AbsoluteAtom.h PATHS ${LLVM_INCLUDE_DIRS}) - if (NOT LLD_LIBRARY_TEST OR NOT LLD_INCLUDE_DIR_TEST) - set (LLVM_FOUND 0) - message(WARNING "liblld (${LLD_LIBRARY_TEST}, ${LLD_INCLUDE_DIR_TEST}) not found in ${LLVM_INCLUDE_DIRS} ${LLVM_LIBRARY_DIRS}. Disabling internal compiler.") + foreach(llvm_v 9 8) + if (NOT LLVM_FOUND) + find_package (LLVM ${llvm_v} CONFIG PATHS ${LLVM_PATHS}) endif () - endif () + endforeach () if (LLVM_FOUND) # Remove dynamically-linked zlib and libedit from LLVM's dependencies: @@ -51,30 +35,39 @@ if (ENABLE_EMBEDDED_COMPILER) set (LLVM_FOUND 0) set (USE_EMBEDDED_COMPILER 0) endif () - - # TODO: fix llvm 8+ and remove: - if (LLVM_FOUND AND LLVM_VERSION_MAJOR GREATER 7) - message(WARNING "LLVM 8+ not supported yet, disabling.") - set (USE_EMBEDDED_COMPILER 0) - endif () else() - set (LLVM_FOUND 1) - set (USE_EMBEDDED_COMPILER 1) - set (LLVM_VERSION "7.0.0bundled") - set (LLVM_INCLUDE_DIRS - ${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm/include - ${ClickHouse_BINARY_DIR}/contrib/llvm/llvm/include - ${ClickHouse_SOURCE_DIR}/contrib/llvm/clang/include - ${ClickHouse_BINARY_DIR}/contrib/llvm/clang/include - ${ClickHouse_BINARY_DIR}/contrib/llvm/llvm/tools/clang/include - ${ClickHouse_SOURCE_DIR}/contrib/llvm/lld/include - ${ClickHouse_BINARY_DIR}/contrib/llvm/lld/include - ${ClickHouse_BINARY_DIR}/contrib/llvm/llvm/tools/lld/include) - set (LLVM_LIBRARY_DIRS ${ClickHouse_BINARY_DIR}/contrib/llvm/llvm) + if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR) + message(WARNING "Option ENABLE_EMBEDDED_COMPILER is set but LLVM library cannot build if build directory is the same as source directory.") + set (LLVM_FOUND 0) + set (USE_EMBEDDED_COMPILER 0) + elseif (SPLIT_SHARED_LIBRARIES) + # llvm-tablegen cannot find shared libraries that we build. Probably can be easily fixed. + message(WARNING "Option ENABLE_EMBEDDED_COMPILER is not compatible with SPLIT_SHARED_LIBRARIES. Build of LLVM will be disabled.") + set (LLVM_FOUND 0) + set (USE_EMBEDDED_COMPILER 0) + elseif (NOT ARCH_AMD64) + # It's not supported yet, but you can help. + message(WARNING "Option ENABLE_EMBEDDED_COMPILER is only available for x86_64. Build of LLVM will be disabled.") + set (LLVM_FOUND 0) + set (USE_EMBEDDED_COMPILER 0) + elseif (SANITIZE STREQUAL "undefined") + # llvm-tblgen, that is used during LLVM build, doesn't work with UBSan. + message(WARNING "Option ENABLE_EMBEDDED_COMPILER does not work with UBSan, because 'llvm-tblgen' tool from LLVM has undefined behaviour. Build of LLVM will be disabled.") + set (LLVM_FOUND 0) + set (USE_EMBEDDED_COMPILER 0) + else () + set (LLVM_FOUND 1) + set (USE_EMBEDDED_COMPILER 1) + set (LLVM_VERSION "9.0.0bundled") + set (LLVM_INCLUDE_DIRS + ${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm/include + ${ClickHouse_BINARY_DIR}/contrib/llvm/llvm/include + ) + set (LLVM_LIBRARY_DIRS ${ClickHouse_BINARY_DIR}/contrib/llvm/llvm) + endif() endif() if (LLVM_FOUND) - message(STATUS "LLVM version: ${LLVM_PACKAGE_VERSION}") message(STATUS "LLVM include Directory: ${LLVM_INCLUDE_DIRS}") message(STATUS "LLVM library Directory: ${LLVM_LIBRARY_DIRS}") message(STATUS "LLVM C++ compiler flags: ${LLVM_CXXFLAGS}") @@ -82,16 +75,53 @@ if (ENABLE_EMBEDDED_COMPILER) endif() -function(llvm_libs_all REQUIRED_LLVM_LIBRARIES) - llvm_map_components_to_libnames (result all) - if (USE_STATIC_LIBRARIES OR NOT "LLVM" IN_LIST result) - list (REMOVE_ITEM result "LTO" "LLVM") - else() - set (result "LLVM") - endif () - if (TERMCAP_LIBRARY) - list (APPEND result ${TERMCAP_LIBRARY}) - endif () - list (APPEND result ${CMAKE_DL_LIBS} ${ZLIB_LIBRARIES}) - set (${REQUIRED_LLVM_LIBRARIES} ${result} PARENT_SCOPE) -endfunction() +# This list was generated by listing all LLVM libraries, compiling the binary and removing all libraries while it still compiles. +set (REQUIRED_LLVM_LIBRARIES +LLVMOrcJIT +LLVMExecutionEngine +LLVMRuntimeDyld +LLVMX86CodeGen +LLVMX86Desc +LLVMX86Info +LLVMX86Utils +LLVMAsmPrinter +LLVMDebugInfoDWARF +LLVMGlobalISel +LLVMSelectionDAG +LLVMMCDisassembler +LLVMPasses +LLVMCodeGen +LLVMipo +LLVMBitWriter +LLVMInstrumentation +LLVMScalarOpts +LLVMAggressiveInstCombine +LLVMInstCombine +LLVMVectorize +LLVMTransformUtils +LLVMTarget +LLVMAnalysis +LLVMProfileData +LLVMObject +LLVMBitReader +LLVMCore +LLVMRemarks +LLVMBitstreamReader +LLVMMCParser +LLVMMC +LLVMBinaryFormat +LLVMDebugInfoCodeView +LLVMSupport +LLVMDemangle +) + +#function(llvm_libs_all REQUIRED_LLVM_LIBRARIES) +# llvm_map_components_to_libnames (result all) +# if (USE_STATIC_LIBRARIES OR NOT "LLVM" IN_LIST result) +# list (REMOVE_ITEM result "LTO" "LLVM") +# else() +# set (result "LLVM") +# endif () +# list (APPEND result ${CMAKE_DL_LIBS} ${ZLIB_LIBRARIES}) +# set (${REQUIRED_LLVM_LIBRARIES} ${result} PARENT_SCOPE) +#endfunction() diff --git a/cmake/find/orc.cmake b/cmake/find/orc.cmake index 613016c6ba7..26253687c80 100644 --- a/cmake/find/orc.cmake +++ b/cmake/find/orc.cmake @@ -1,7 +1,8 @@ option (ENABLE_ORC "Enable ORC" ${ENABLE_LIBRARIES}) if(ENABLE_ORC) -option (USE_INTERNAL_ORC_LIBRARY "Set to FALSE to use system ORC instead of bundled" ${NOT_UNBUNDLED}) +include(cmake/find/snappy.cmake) +option(USE_INTERNAL_ORC_LIBRARY "Set to FALSE to use system ORC instead of bundled" ${NOT_UNBUNDLED}) if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include/orc/OrcFile.hh") if(USE_INTERNAL_ORC_LIBRARY) @@ -25,7 +26,7 @@ endif () if (ORC_LIBRARY AND ORC_INCLUDE_DIR) set(USE_ORC 1) -elseif(NOT MISSING_INTERNAL_ORC_LIBRARY AND ARROW_LIBRARY) # (LIBGSASL_LIBRARY AND LIBXML2_LIBRARY) +elseif(NOT MISSING_INTERNAL_ORC_LIBRARY AND ARROW_LIBRARY AND SNAPPY_LIBRARY) # (LIBGSASL_LIBRARY AND LIBXML2_LIBRARY) set(ORC_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include") set(ORC_LIBRARY orc) set(USE_ORC 1) diff --git a/cmake/find/parquet.cmake b/cmake/find/parquet.cmake index 14332fc9caf..d302fd64e1e 100644 --- a/cmake/find/parquet.cmake +++ b/cmake/find/parquet.cmake @@ -24,7 +24,10 @@ endif() if(ARROW_INCLUDE_DIR AND PARQUET_INCLUDE_DIR) elseif(NOT MISSING_INTERNAL_PARQUET_LIBRARY AND NOT OS_FREEBSD) include(cmake/find/snappy.cmake) - set(CAN_USE_INTERNAL_PARQUET_LIBRARY 1) + if(SNAPPY_LIBRARY) + set(CAN_USE_INTERNAL_PARQUET_LIBRARY 1) + endif() + include(CheckCXXSourceCompiles) if(NOT USE_INTERNAL_DOUBLE_CONVERSION_LIBRARY) set(CMAKE_REQUIRED_LIBRARIES ${DOUBLE_CONVERSION_LIBRARIES}) diff --git a/cmake/find/poco.cmake b/cmake/find/poco.cmake index ee2f5d9df1f..b44d2932276 100644 --- a/cmake/find/poco.cmake +++ b/cmake/find/poco.cmake @@ -8,6 +8,14 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/poco/CMakeLists.txt") set (MISSING_INTERNAL_POCO_LIBRARY 1) endif () +if (NOT ENABLE_LIBRARIES) + set (ENABLE_POCO_NETSSL ${ENABLE_LIBRARIES} CACHE BOOL "") + set (ENABLE_POCO_MONGODB ${ENABLE_LIBRARIES} CACHE BOOL "") + set (ENABLE_POCO_REDIS ${ENABLE_LIBRARIES} CACHE BOOL "") + set (ENABLE_POCO_ODBC ${ENABLE_LIBRARIES} CACHE BOOL "") + set (ENABLE_POCO_SQL ${ENABLE_LIBRARIES} CACHE BOOL "") +endif () + set (POCO_COMPONENTS Net XML SQL Data) if (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL) list (APPEND POCO_COMPONENTS Crypto NetSSL) diff --git a/cmake/find/s3.cmake b/cmake/find/s3.cmake new file mode 100644 index 00000000000..af53dc80feb --- /dev/null +++ b/cmake/find/s3.cmake @@ -0,0 +1,26 @@ +if(NOT OS_FREEBSD AND NOT APPLE AND NOT ARCH_ARM) + option(ENABLE_S3 "Enable S3" ${ENABLE_LIBRARIES}) +endif() + +if(ENABLE_S3) + option(USE_INTERNAL_AWS_S3_LIBRARY "Set to FALSE to use system S3 instead of bundled" ${NOT_UNBUNDLED}) + + if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3") + message (WARNING "submodule contrib/aws is missing. to fix try run: \n git submodule update --init --recursive") + set (MISSING_AWS_S3 1) + endif () + + if (USE_INTERNAL_AWS_S3_LIBRARY AND NOT MISSING_AWS_S3) + set(AWS_S3_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3/include") + set(AWS_S3_CORE_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-core/include") + set(AWS_S3_LIBRARY aws_s3) + set(USE_INTERNAL_AWS_S3_LIBRARY 1) + set(USE_AWS_S3 1) + else() + set(USE_INTERNAL_AWS_S3_LIBRARY 0) + set(USE_AWS_S3 0) + endif () + +endif() + +message (STATUS "Using aws_s3=${USE_AWS_S3}: ${AWS_S3_INCLUDE_DIR} : ${AWS_S3_LIBRARY}") diff --git a/cmake/find/ssl.cmake b/cmake/find/ssl.cmake index af879406aab..36f9d1e67ec 100644 --- a/cmake/find/ssl.cmake +++ b/cmake/find/ssl.cmake @@ -6,9 +6,9 @@ if(NOT ARCH_32) option(USE_INTERNAL_SSL_LIBRARY "Set to FALSE to use system *ssl library instead of bundled" ${NOT_UNBUNDLED}) endif() -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/ssl/CMakeLists.txt") +if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/openssl/README") if(USE_INTERNAL_SSL_LIBRARY) - message(WARNING "submodule contrib/ssl is missing. to fix try run: \n git submodule update --init --recursive") + message(WARNING "submodule contrib/openssl is missing. to fix try run: \n git submodule update --init --recursive") endif() set(USE_INTERNAL_SSL_LIBRARY 0) set(MISSING_INTERNAL_SSL_LIBRARY 1) @@ -42,17 +42,17 @@ endif () if (NOT OPENSSL_FOUND AND NOT MISSING_INTERNAL_SSL_LIBRARY) set (USE_INTERNAL_SSL_LIBRARY 1) - set (OPENSSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/ssl") - set (OPENSSL_INCLUDE_DIR "${OPENSSL_ROOT_DIR}/include") - if (NOT USE_STATIC_LIBRARIES AND TARGET crypto-shared AND TARGET ssl-shared) - set (OPENSSL_CRYPTO_LIBRARY crypto-shared) - set (OPENSSL_SSL_LIBRARY ssl-shared) - else () - set (OPENSSL_CRYPTO_LIBRARY crypto) - set (OPENSSL_SSL_LIBRARY ssl) + set (OPENSSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/openssl") + + if (ARCH_AMD64) + set (OPENSSL_INCLUDE_DIR "${OPENSSL_ROOT_DIR}/include" "${ClickHouse_SOURCE_DIR}/contrib/openssl-cmake/linux_x86_64/include") + elseif (ARCH_AARCH64) + set (OPENSSL_INCLUDE_DIR "${OPENSSL_ROOT_DIR}/include" "${ClickHouse_SOURCE_DIR}/contrib/openssl-cmake/linux_aarch64/include") endif () - set (OPENSSL_LIBRARIES ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY}) + set (OPENSSL_CRYPTO_LIBRARY crypto) + set (OPENSSL_SSL_LIBRARY ssl) set (OPENSSL_FOUND 1) + set (OPENSSL_LIBRARIES ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY}) endif () if(OPENSSL_FOUND) diff --git a/cmake/find/termcap.cmake b/cmake/find/termcap.cmake index 47b772331bb..7564b7134e7 100644 --- a/cmake/find/termcap.cmake +++ b/cmake/find/termcap.cmake @@ -1,5 +1,8 @@ -find_library (TERMCAP_LIBRARY termcap) +find_library (TERMCAP_LIBRARY tinfo) if (NOT TERMCAP_LIBRARY) - find_library (TERMCAP_LIBRARY tinfo) + find_library (TERMCAP_LIBRARY ncurses) +endif() +if (NOT TERMCAP_LIBRARY) + find_library (TERMCAP_LIBRARY termcap) endif() message (STATUS "Using termcap: ${TERMCAP_LIBRARY}") diff --git a/cmake/find/unwind.cmake b/cmake/find/unwind.cmake index efa7618a53b..d3653973082 100644 --- a/cmake/find/unwind.cmake +++ b/cmake/find/unwind.cmake @@ -4,6 +4,11 @@ if (NOT CMAKE_SYSTEM MATCHES "Linux" OR ARCH_ARM OR ARCH_32) set (USE_UNWIND OFF) endif () +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libunwind/CMakeLists.txt") + message(WARNING "submodule contrib/libunwind is missing. to fix try run: \n git submodule update --init --recursive") + set (USE_UNWIND OFF) +endif () + if (USE_UNWIND) add_subdirectory(contrib/libunwind-cmake) set (UNWIND_LIBRARIES unwind) diff --git a/cmake/linux/default_libs.cmake b/cmake/linux/default_libs.cmake index ef1354628fe..6ecc3e96593 100644 --- a/cmake/linux/default_libs.cmake +++ b/cmake/linux/default_libs.cmake @@ -18,6 +18,14 @@ message(STATUS "Default libraries: ${DEFAULT_LIBS}") set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS}) set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS}) +# glibc-compatibility library relies to fixed version of libc headers +# (because minor changes in function attributes between different glibc versions will introduce incompatibilities) +# This is for x86_64. For other architectures we have separate toolchains. +if (ARCH_AMD64) + set(CMAKE_C_STANDARD_INCLUDE_DIRECTORIES ${ClickHouse_SOURCE_DIR}/contrib/libc-headers/x86_64-linux-gnu ${ClickHouse_SOURCE_DIR}/contrib/libc-headers) + set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES ${ClickHouse_SOURCE_DIR}/contrib/libc-headers/x86_64-linux-gnu ${ClickHouse_SOURCE_DIR}/contrib/libc-headers) +endif () + # Global libraries add_library(global-libs INTERFACE) diff --git a/cmake/linux/toolchain-aarch64.cmake b/cmake/linux/toolchain-aarch64.cmake index e93f0f47faf..e3924fdc537 100644 --- a/cmake/linux/toolchain-aarch64.cmake +++ b/cmake/linux/toolchain-aarch64.cmake @@ -2,13 +2,18 @@ set (CMAKE_SYSTEM_NAME "Linux") set (CMAKE_SYSTEM_PROCESSOR "aarch64") set (CMAKE_C_COMPILER_TARGET "aarch64-linux-gnu") set (CMAKE_CXX_COMPILER_TARGET "aarch64-linux-gnu") +set (CMAKE_ASM_COMPILER_TARGET "aarch64-linux-gnu") set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64/aarch64-linux-gnu/libc") # We don't use compiler from toolchain because it's gcc-8, and we provide support only for gcc-9. -set (CMAKE_AR "${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64/bin/aarch64-linux-gnu-ar") +set (CMAKE_AR "${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64/bin/aarch64-linux-gnu-ar" CACHE FILEPATH "" FORCE) +set (CMAKE_RANLIB "${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64/bin/aarch64-linux-gnu-ranlib" CACHE FILEPATH "" FORCE) -set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64") -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64") +set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64") +set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64") +set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64") + +set (LINKER_NAME "lld" CACHE STRING "" FORCE) set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld") set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld") diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake index a5d1d0d3055..cb099ade7f5 100644 --- a/cmake/sanitize.cmake +++ b/cmake/sanitize.cmake @@ -23,7 +23,7 @@ if (SANITIZE) # RelWithDebInfo, and downgrade optimizations to -O1 but not to -Og, to # keep the binary size down. # TODO: try compiling with -Og and with ld.gold. - set (MSAN_FLAGS "-fsanitize=memory -fsanitize-memory-track-origins -fno-optimize-sibling-calls") + set (MSAN_FLAGS "-fsanitize=memory -fsanitize-memory-track-origins -fno-optimize-sibling-calls -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/dbms/tests/msan_suppressions.txt") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}") @@ -40,7 +40,6 @@ if (SANITIZE) set (ENABLE_HDFS 0 CACHE BOOL "") set (ENABLE_CAPNP 0 CACHE BOOL "") set (ENABLE_RDKAFKA 0 CACHE BOOL "") - set (ENABLE_ICU 0 CACHE BOOL "") set (ENABLE_POCO_MONGODB 0 CACHE BOOL "") set (ENABLE_POCO_NETSSL 0 CACHE BOOL "") set (ENABLE_POCO_ODBC 0 CACHE BOOL "") @@ -77,6 +76,9 @@ if (SANITIZE) set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libubsan") endif () + # llvm-tblgen, that is used during LLVM build, doesn't work with UBSan. + set (ENABLE_EMBEDDED_COMPILER 0 CACHE BOOL "") + elseif (SANITIZE STREQUAL "libfuzzer") # NOTE: Eldar Zaitov decided to name it "libfuzzer" instead of "fuzzer" to keep in mind another possible fuzzer backends. # NOTE: no-link means that all the targets are built with instrumentation for fuzzer, but only some of them (tests) have entry point for fuzzer and it's not checked. diff --git a/cmake/toolchain/darwin-x86_64/README.txt b/cmake/toolchain/darwin-x86_64/README.txt index 4ece240f029..0626c9e886c 100644 --- a/cmake/toolchain/darwin-x86_64/README.txt +++ b/cmake/toolchain/darwin-x86_64/README.txt @@ -1,2 +1,2 @@ wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz -tar --strip-components=1 xJf MacOSX10.14.sdk.tar.xz +tar xJf MacOSX10.14.sdk.tar.xz --strip-components=1 diff --git a/cmake/toolchain/linux-aarch64/README.txt b/cmake/toolchain/linux-aarch64/README.txt index 7146c1d2b9a..3183d30b70d 100644 --- a/cmake/toolchain/linux-aarch64/README.txt +++ b/cmake/toolchain/linux-aarch64/README.txt @@ -1,2 +1,2 @@ -wget https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en -O gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -tar --strip-components=1 xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz +wget 'https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en' -O gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz +tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz --strip-components=1 diff --git a/cmake/tools.cmake b/cmake/tools.cmake index 04e0946ee73..025aac79b53 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -1,6 +1,6 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set (COMPILER_GCC 1) -elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") +elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang|AppleClang") set (COMPILER_CLANG 1) endif () @@ -20,16 +20,38 @@ else () message (WARNING "You are using an unsupported compiler. Compilation has only been tested with Clang 6+ and GCC 7+.") endif () +STRING(REGEX MATCHALL "[0-9]+" COMPILER_VERSION_LIST ${CMAKE_CXX_COMPILER_VERSION}) +LIST(GET COMPILER_VERSION_LIST 0 COMPILER_VERSION_MAJOR) + option (LINKER_NAME "Linker name or full path") +if (COMPILER_GCC) + find_program (LLD_PATH NAMES "ld.lld") + find_program (GOLD_PATH NAMES "ld.gold") +else () + find_program (LLD_PATH NAMES "ld.lld-${COMPILER_VERSION_MAJOR}" "lld-${COMPILER_VERSION_MAJOR}" "ld.lld" "lld") + find_program (GOLD_PATH NAMES "ld.gold" "gold") +endif () -find_program (LLD_PATH NAMES "ld.lld" "lld") -find_program (GOLD_PATH NAMES "ld.gold" "gold") - +# We prefer LLD linker over Gold or BFD. if (NOT LINKER_NAME) if (LLD_PATH) - set (LINKER_NAME "lld") - elseif (GOLD_PATH) - set (LINKER_NAME "gold") + if (COMPILER_GCC) + # GCC driver requires one of supported linker names like "lld". + set (LINKER_NAME "lld") + else () + # Clang driver simply allows full linker path. + set (LINKER_NAME ${LLD_PATH}) + endif () + endif () +endif () + +if (NOT LINKER_NAME) + if (GOLD_PATH) + if (COMPILER_GCC) + set (LINKER_NAME "gold") + else () + set (LINKER_NAME ${GOLD_PATH}) + endif () endif () endif () diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b0a271b21ac..415d3a88703 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -52,6 +52,7 @@ if (USE_INTERNAL_BTRIE_LIBRARY) endif () if (USE_INTERNAL_ZLIB_LIBRARY) + unset (BUILD_SHARED_LIBS CACHE) set (ZLIB_ENABLE_TESTS 0 CACHE INTERNAL "") set (SKIP_INSTALL_ALL 1 CACHE INTERNAL "") set (ZLIB_COMPAT 1 CACHE INTERNAL "") # also enables WITH_GZFILEOP @@ -65,34 +66,19 @@ if (USE_INTERNAL_ZLIB_LIBRARY) endif () add_subdirectory (${INTERNAL_ZLIB_NAME}) - # TODO: make pull to Dead2/zlib-ng and remove: # We should use same defines when including zlib.h as used when zlib compiled target_compile_definitions (zlib PUBLIC ZLIB_COMPAT WITH_GZFILEOP) target_compile_definitions (zlibstatic PUBLIC ZLIB_COMPAT WITH_GZFILEOP) - if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "AMD64") + if (ARCH_AMD64 OR ARCH_AARCH64) target_compile_definitions (zlib PUBLIC X86_64 UNALIGNED_OK) target_compile_definitions (zlibstatic PUBLIC X86_64 UNALIGNED_OK) endif () - - #set_target_properties(example PROPERTIES EXCLUDE_FROM_ALL 1) - #if (TARGET example64) - # set_target_properties(example64 PROPERTIES EXCLUDE_FROM_ALL 1) - #endif () - - #set_target_properties(minigzip PROPERTIES EXCLUDE_FROM_ALL 1) - #if (TARGET minigzip64) - # set_target_properties(minigzip64 PROPERTIES EXCLUDE_FROM_ALL 1) - #endif () endif () if (USE_INTERNAL_CCTZ_LIBRARY) add_subdirectory (cctz-cmake) endif () -if (ENABLE_TCMALLOC AND USE_INTERNAL_GPERFTOOLS_LIBRARY) - add_subdirectory (libtcmalloc) -endif () - if (ENABLE_JEMALLOC AND USE_INTERNAL_JEMALLOC_LIBRARY) add_subdirectory (jemalloc-cmake) endif () @@ -106,20 +92,10 @@ if (USE_INTERNAL_H3_LIBRARY) endif () if (USE_INTERNAL_SSL_LIBRARY) - if (NOT MAKE_STATIC_LIBRARIES) - set (BUILD_SHARED 1) - endif () + add_subdirectory (openssl-cmake) - # By default, ${CMAKE_INSTALL_PREFIX}/etc/ssl is selected - that is not what we need. - # We need to use system wide ssl directory. - set (OPENSSLDIR "/etc/ssl") - - set (LIBRESSL_SKIP_INSTALL 1 CACHE INTERNAL "") - add_subdirectory (ssl) - target_include_directories(${OPENSSL_CRYPTO_LIBRARY} SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR}) - target_include_directories(${OPENSSL_SSL_LIBRARY} SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR}) + # This is for Poco library set (POCO_SKIP_OPENSSL_FIND 1) - add_library(OpenSSL::Crypto ALIAS ${OPENSSL_CRYPTO_LIBRARY}) add_library(OpenSSL::SSL ALIAS ${OPENSSL_SSL_LIBRARY}) endif () @@ -164,6 +140,10 @@ if (ENABLE_ODBC AND USE_INTERNAL_ODBC_LIBRARY) add_library(ODBC::ODBC ALIAS ${ODBC_LIBRARIES}) endif () +if (ENABLE_ICU AND USE_INTERNAL_ICU_LIBRARY) + add_subdirectory (icu-cmake) +endif () + if (USE_INTERNAL_PARQUET_LIBRARY) if (USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE) # We dont use arrow's cmakefiles because they uses too many depends and download some libs in compile time @@ -180,10 +160,7 @@ if (USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE) set (ARROW_VERBOSE_THIRDPARTY_BUILD ON CACHE INTERNAL "") set (ARROW_BUILD_SHARED 1 CACHE INTERNAL "") set (ARROW_BOOST_HEADER_ONLY ON CACHE INTERNAL "") - #set (BOOST_INCLUDEDIR Boost_INCLUDE_DIRS) set (Boost_FOUND 1 CACHE INTERNAL "") - #set (ZLIB_HOME ${ZLIB_INCLUDE_DIR}) - #set (ZLIB_FOUND 1) if (MAKE_STATIC_LIBRARIES) set (PARQUET_ARROW_LINKAGE "static" CACHE INTERNAL "") set (ARROW_TEST_LINKAGE "static" CACHE INTERNAL "") @@ -223,6 +200,11 @@ else() endif() add_subdirectory(arrow-cmake) + + # The library is large - avoid bloat. + target_compile_options (${ARROW_LIBRARY} PRIVATE -g0) + target_compile_options (${THRIFT_LIBRARY} PRIVATE -g0) + target_compile_options (${PARQUET_LIBRARY} PRIVATE -g0) endif() endif() @@ -259,28 +241,14 @@ elseif(GTEST_SRC_DIR) target_compile_definitions(gtest INTERFACE GTEST_HAS_POSIX_RE=0) endif() -if (USE_INTERNAL_LLVM_LIBRARY) - file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/empty.cpp CONTENT " ") - add_library(LLVM0 ${CMAKE_CURRENT_BINARY_DIR}/empty.cpp) # silly cmake bug fix - add_library(LLVMOFF ${CMAKE_CURRENT_BINARY_DIR}/empty.cpp) +if (USE_EMBEDDED_COMPILER AND USE_INTERNAL_LLVM_LIBRARY) # ld: unknown option: --color-diagnostics if (APPLE) set (LINKER_SUPPORTS_COLOR_DIAGNOSTICS 0 CACHE INTERNAL "") endif () set (LLVM_ENABLE_EH 1 CACHE INTERNAL "") set (LLVM_ENABLE_RTTI 1 CACHE INTERNAL "") - set (LLVM_INCLUDE_TESTS 0 CACHE INTERNAL "") - set (LLVM_INCLUDE_EXAMPLES 0 CACHE INTERNAL "") - set (LLVM_INCLUDE_TOOLS 0 CACHE INTERNAL "") - set (LLVM_INSTALL_TOOLCHAIN_ONLY 0 CACHE INTERNAL "") - set (CLANG_BUILT_STANDALONE 0 CACHE INTERNAL "") - set (LLDB_BUILT_STANDALONE 0 CACHE INTERNAL "") - set (CLANG_ENABLE_STATIC_ANALYZER 0 CACHE INTERNAL "") - set (CLANG_ENABLE_ARCMT 0 CACHE INTERNAL "") - set (CLANG_BUILD_TOOLS 0 CACHE INTERNAL "") - set (BENCHMARK_ENABLE_GTEST_TESTS 0 CACHE INTERNAL "") - set (BENCHMARK_ENABLE_ASSEMBLY_TESTS 0 CACHE INTERNAL "") - set (LLVM_TARGETS_TO_BUILD "X86;AArch64" CACHE INTERNAL "") + set (LLVM_TARGETS_TO_BUILD "X86;AArch64" CACHE STRING "") add_subdirectory (llvm/llvm) endif () @@ -312,12 +280,39 @@ if (USE_INTERNAL_HDFS3_LIBRARY) add_subdirectory(libhdfs3-cmake) endif () +if (USE_INTERNAL_AWS_S3_LIBRARY) + set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) + set (save_CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES}) + set (save_CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES}) + set (save_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS}) + set (save_CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}) + add_subdirectory(curl-cmake) + set (CMAKE_C_FLAGS ${save_CMAKE_C_FLAGS}) + set (CMAKE_REQUIRED_LIBRARIES ${save_CMAKE_REQUIRED_LIBRARIES}) + set (CMAKE_CMAKE_REQUIRED_INCLUDES ${save_CMAKE_REQUIRED_INCLUDES}) + set (CMAKE_REQUIRED_FLAGS ${save_CMAKE_REQUIRED_FLAGS}) + set (CMAKE_CMAKE_MODULE_PATH ${save_CMAKE_MODULE_PATH}) + add_subdirectory(aws-s3-cmake) + + # The library is large - avoid bloat. + target_compile_options (aws_s3 PRIVATE -g0) + target_compile_options (aws_s3_checksums PRIVATE -g0) + target_compile_options (libcurl PRIVATE -g0) +endif () + if (USE_BASE64) add_subdirectory (base64-cmake) endif() if (USE_INTERNAL_HYPERSCAN_LIBRARY) add_subdirectory (hyperscan) + + # The library is large - avoid bloat. + if (USE_STATIC_LIBRARIES) + target_compile_options (hs PRIVATE -g0) + else () + target_compile_options (hs_shared PRIVATE -g0) + endif () endif() if (USE_SIMDJSON) @@ -331,7 +326,3 @@ endif() if (USE_FASTOPS) add_subdirectory (fastops-cmake) endif() - -#if (USE_INTERNAL_ORC_LIBRARY) -# add_subdirectory(orc-cmake) -#endif () diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index a564f62384d..3ba24f49498 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -70,6 +70,14 @@ add_custom_command(OUTPUT orc_proto.pb.h orc_proto.pb.cc --cpp_out="${CMAKE_CURRENT_BINARY_DIR}" "${PROTO_DIR}/orc_proto.proto") +# arrow-cmake cmake file calling orc cmake subroutine which detects certain compiler features. +# Apple Clang compiler failed to compile this code without specifying c++11 standard. +# As result these compiler features detected as absent. In result it failed to compile orc itself. +# In orc makefile there is code that sets flags, but arrow-cmake ignores these flags. +if (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") + set (CXX11_FLAGS "-std=c++0x") +endif() + include(${ClickHouse_SOURCE_DIR}/contrib/orc/cmake_modules/CheckSourceCompiles.cmake) include(orc_check.cmake) configure_file("${ORC_INCLUDE_DIR}/orc/orc-config.hh.in" "${ORC_BUILD_INCLUDE_DIR}/orc/orc-config.hh") diff --git a/contrib/aws b/contrib/aws new file mode 160000 index 00000000000..45dd8552d3c --- /dev/null +++ b/contrib/aws @@ -0,0 +1 @@ +Subproject commit 45dd8552d3c492defca79d2720bcc809e35654da diff --git a/contrib/aws-c-common b/contrib/aws-c-common new file mode 160000 index 00000000000..736a82d1697 --- /dev/null +++ b/contrib/aws-c-common @@ -0,0 +1 @@ +Subproject commit 736a82d1697c108b04a277e66438a7f4e19b6857 diff --git a/contrib/aws-c-event-stream b/contrib/aws-c-event-stream new file mode 160000 index 00000000000..3bc33662f9c --- /dev/null +++ b/contrib/aws-c-event-stream @@ -0,0 +1 @@ +Subproject commit 3bc33662f9ccff4f4cbcf9509cc78c26e022fde0 diff --git a/contrib/aws-checksums b/contrib/aws-checksums new file mode 160000 index 00000000000..519d6d90938 --- /dev/null +++ b/contrib/aws-checksums @@ -0,0 +1 @@ +Subproject commit 519d6d9093819b6cf89ffff589a27ef8f83d0f65 diff --git a/contrib/aws-s3-cmake/CMakeLists.txt b/contrib/aws-s3-cmake/CMakeLists.txt new file mode 100644 index 00000000000..667ca43c501 --- /dev/null +++ b/contrib/aws-s3-cmake/CMakeLists.txt @@ -0,0 +1,107 @@ +SET(AWS_S3_LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3) +SET(AWS_CORE_LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-core) +SET(AWS_CHECKSUMS_LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/aws-checksums) +SET(AWS_COMMON_LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/aws-c-common) +SET(AWS_EVENT_STREAM_LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/aws-c-event-stream) + +OPTION(USE_AWS_MEMORY_MANAGEMENT "Aws memory management" OFF) +configure_file("${AWS_CORE_LIBRARY_DIR}/include/aws/core/SDKConfig.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/include/aws/core/SDKConfig.h" @ONLY) + +configure_file("${AWS_COMMON_LIBRARY_DIR}/include/aws/common/config.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/include/aws/common/config.h" @ONLY) + + +file(GLOB AWS_CORE_SOURCES + "${AWS_CORE_LIBRARY_DIR}/source/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/auth/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/client/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/http/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/http/standard/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/http/curl/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/config/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/external/cjson/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/external/tinyxml2/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/internal/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/monitoring/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/net/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/linux-shared/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/platform/linux-shared/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/utils/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/utils/base64/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/utils/event/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/utils/crypto/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/utils/crypto/openssl/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/utils/crypto/factory/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/utils/json/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/utils/logging/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/utils/memory/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/utils/memory/stl/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/utils/stream/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/utils/threading/*.cpp" + "${AWS_CORE_LIBRARY_DIR}/source/utils/xml/*.cpp" + ) + +file(GLOB AWS_S3_SOURCES + "${AWS_S3_LIBRARY_DIR}/source/*.cpp" + ) + +file(GLOB AWS_S3_MODEL_SOURCES + "${AWS_S3_LIBRARY_DIR}/source/model/*.cpp" + ) + +file(GLOB AWS_EVENT_STREAM_SOURCES + "${AWS_EVENT_STREAM_LIBRARY_DIR}/source/*.c" + ) + +file(GLOB AWS_COMMON_SOURCES + "${AWS_COMMON_LIBRARY_DIR}/source/*.c" + "${AWS_COMMON_LIBRARY_DIR}/source/posix/*.c" + ) + +file(GLOB AWS_CHECKSUMS_SOURCES + "${AWS_CHECKSUMS_LIBRARY_DIR}/source/*.c" + "${AWS_CHECKSUMS_LIBRARY_DIR}/source/intel/*.c" + "${AWS_CHECKSUMS_LIBRARY_DIR}/source/arm/*.c" + ) + +file(GLOB S3_UNIFIED_SRC + ${AWS_EVENT_STREAM_SOURCES} + ${AWS_COMMON_SOURCES} + ${AWS_S3_SOURCES} + ${AWS_S3_MODEL_SOURCES} + ${AWS_CORE_SOURCES} + ) + +set(S3_INCLUDES + "${CMAKE_CURRENT_SOURCE_DIR}/include/" + "${AWS_COMMON_LIBRARY_DIR}/include/" + "${AWS_EVENT_STREAM_LIBRARY_DIR}/include/" + "${AWS_S3_LIBRARY_DIR}/include/" + "${AWS_CORE_LIBRARY_DIR}/include/" + "${CMAKE_CURRENT_BINARY_DIR}/include/" + ) + +add_library(aws_s3_checksums ${AWS_CHECKSUMS_SOURCES}) +target_include_directories(aws_s3_checksums PUBLIC "${AWS_CHECKSUMS_LIBRARY_DIR}/include/") +if(CMAKE_BUILD_TYPE STREQUAL "" OR CMAKE_BUILD_TYPE STREQUAL "Debug") + target_compile_definitions(aws_s3_checksums PRIVATE "-DDEBUG_BUILD") +endif() +set_target_properties(aws_s3_checksums PROPERTIES COMPILE_OPTIONS -fPIC) +set_target_properties(aws_s3_checksums PROPERTIES LINKER_LANGUAGE C) +set_property(TARGET aws_s3_checksums PROPERTY C_STANDARD 99) + +add_library(aws_s3 ${S3_UNIFIED_SRC}) + +target_compile_definitions(aws_s3 PUBLIC -DENABLE_CURL_CLIENT) +target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_MAJOR=1") +target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_MINOR=7") +target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_PATCH=231") +target_include_directories(aws_s3 PUBLIC ${S3_INCLUDES} "${CMAKE_BINARY_DIR}/install") + +if (OPENSSL_FOUND) + target_compile_definitions(aws_s3 PUBLIC -DENABLE_OPENSSL_ENCRYPTION) + target_link_libraries(aws_s3 PRIVATE ${OPENSSL_LIBRARIES}) +endif() + +target_link_libraries(aws_s3 PRIVATE aws_s3_checksums libcurl) diff --git a/contrib/capnproto-cmake/CMakeLists.txt b/contrib/capnproto-cmake/CMakeLists.txt index d92a5a282ce..c54b4e8eae5 100644 --- a/contrib/capnproto-cmake/CMakeLists.txt +++ b/contrib/capnproto-cmake/CMakeLists.txt @@ -28,8 +28,7 @@ set (KJ_SRCS ) add_library(kj ${KJ_SRCS}) -target_include_directories(kj PUBLIC ${CAPNPROTO_SOURCE_DIR}) -target_compile_options(kj PUBLIC -Wno-non-virtual-dtor) +target_include_directories(kj SYSTEM PUBLIC ${CAPNPROTO_SOURCE_DIR}) set (CAPNP_SRCS ${CAPNPROTO_SOURCE_DIR}/capnp/c++.capnp.c++ @@ -50,6 +49,9 @@ set (CAPNP_SRCS ) add_library(capnp ${CAPNP_SRCS}) +set_target_properties(capnp + PROPERTIES LINKER_LANGUAGE CXX + ) target_link_libraries(capnp PUBLIC kj) set (CAPNPC_SRCS @@ -67,3 +69,15 @@ set (CAPNPC_SRCS add_library(capnpc ${CAPNPC_SRCS}) target_link_libraries(capnpc PUBLIC capnp) + +# The library has substandard code +if (COMPILER_GCC) + set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor -Wno-sign-compare -Wno-strict-aliasing -Wno-maybe-uninitialized + -Wno-deprecated-declarations -Wno-class-memaccess) +elseif (COMPILER_CLANG) + set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor -Wno-sign-compare -Wno-strict-aliasing -Wno-deprecated-declarations) +endif () + +target_compile_options(kj PRIVATE ${SUPPRESS_WARNINGS}) +target_compile_options(capnp PRIVATE ${SUPPRESS_WARNINGS}) +target_compile_options(capnpc PRIVATE ${SUPPRESS_WARNINGS}) diff --git a/contrib/croaring/CMakeLists.txt b/contrib/croaring/CMakeLists.txt index eeffb1e0a34..da19911487f 100644 --- a/contrib/croaring/CMakeLists.txt +++ b/contrib/croaring/CMakeLists.txt @@ -1,6 +1,6 @@ add_library(roaring - roaring.c - roaring/roaring.h - roaring/roaring.hh) + roaring.c + roaring/roaring.h + roaring/roaring.hh) -target_include_directories (roaring PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_include_directories (roaring SYSTEM PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/contrib/curl b/contrib/curl new file mode 160000 index 00000000000..3b8bbbbd160 --- /dev/null +++ b/contrib/curl @@ -0,0 +1 @@ +Subproject commit 3b8bbbbd1609c638a3d3d0acb148a33dedb67be3 diff --git a/contrib/curl-cmake/CMake/CurlSymbolHiding.cmake b/contrib/curl-cmake/CMake/CurlSymbolHiding.cmake new file mode 100644 index 00000000000..224755a314c --- /dev/null +++ b/contrib/curl-cmake/CMake/CurlSymbolHiding.cmake @@ -0,0 +1,61 @@ +include(CheckCSourceCompiles) + +option(CURL_HIDDEN_SYMBOLS "Set to ON to hide libcurl internal symbols (=hide all symbols that aren't officially external)." ON) +mark_as_advanced(CURL_HIDDEN_SYMBOLS) + +if(CURL_HIDDEN_SYMBOLS) + set(SUPPORTS_SYMBOL_HIDING FALSE) + + if(CMAKE_C_COMPILER_ID MATCHES "Clang") + set(SUPPORTS_SYMBOL_HIDING TRUE) + set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))") + set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden") + elseif(CMAKE_COMPILER_IS_GNUCC) + if(NOT CMAKE_VERSION VERSION_LESS 2.8.10) + set(GCC_VERSION ${CMAKE_C_COMPILER_VERSION}) + else() + execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion + OUTPUT_VARIABLE GCC_VERSION) + endif() + if(NOT GCC_VERSION VERSION_LESS 3.4) + # note: this is considered buggy prior to 4.0 but the autotools don't care, so let's ignore that fact + set(SUPPORTS_SYMBOL_HIDING TRUE) + set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))") + set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden") + endif() + elseif(CMAKE_C_COMPILER_ID MATCHES "SunPro" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 8.0) + set(SUPPORTS_SYMBOL_HIDING TRUE) + set(_SYMBOL_EXTERN "__global") + set(_CFLAG_SYMBOLS_HIDE "-xldscope=hidden") + elseif(CMAKE_C_COMPILER_ID MATCHES "Intel" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 9.0) + # note: this should probably just check for version 9.1.045 but I'm not 100% sure + # so let's do it the same way autotools do. + set(SUPPORTS_SYMBOL_HIDING TRUE) + set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))") + set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden") + check_c_source_compiles("#include + int main (void) { printf(\"icc fvisibility bug test\"); return 0; }" _no_bug) + if(NOT _no_bug) + set(SUPPORTS_SYMBOL_HIDING FALSE) + set(_SYMBOL_EXTERN "") + set(_CFLAG_SYMBOLS_HIDE "") + endif() + elseif(MSVC) + set(SUPPORTS_SYMBOL_HIDING TRUE) + endif() + + set(HIDES_CURL_PRIVATE_SYMBOLS ${SUPPORTS_SYMBOL_HIDING}) +elseif(MSVC) + if(NOT CMAKE_VERSION VERSION_LESS 3.7) + set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE) #present since 3.4.3 but broken + set(HIDES_CURL_PRIVATE_SYMBOLS FALSE) + else() + message(WARNING "Hiding private symbols regardless CURL_HIDDEN_SYMBOLS being disabled.") + set(HIDES_CURL_PRIVATE_SYMBOLS TRUE) + endif() +else() + set(HIDES_CURL_PRIVATE_SYMBOLS FALSE) +endif() + +set(CURL_CFLAG_SYMBOLS_HIDE ${_CFLAG_SYMBOLS_HIDE}) +set(CURL_EXTERN_SYMBOL ${_SYMBOL_EXTERN}) diff --git a/contrib/curl-cmake/CMake/CurlTests.c b/contrib/curl-cmake/CMake/CurlTests.c new file mode 100644 index 00000000000..3ef35f02535 --- /dev/null +++ b/contrib/curl-cmake/CMake/CurlTests.c @@ -0,0 +1,617 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2019, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ +#ifdef TIME_WITH_SYS_TIME +/* Time with sys/time test */ + +#include +#include +#include + +int +main () +{ +if ((struct tm *) 0) +return 0; + ; + return 0; +} + +#endif + +#ifdef HAVE_FCNTL_O_NONBLOCK + +/* headers for FCNTL_O_NONBLOCK test */ +#include +#include +#include +/* */ +#if defined(sun) || defined(__sun__) || \ + defined(__SUNPRO_C) || defined(__SUNPRO_CC) +# if defined(__SVR4) || defined(__srv4__) +# define PLATFORM_SOLARIS +# else +# define PLATFORM_SUNOS4 +# endif +#endif +#if (defined(_AIX) || defined(__xlC__)) && !defined(_AIX41) +# define PLATFORM_AIX_V3 +#endif +/* */ +#if defined(PLATFORM_SUNOS4) || defined(PLATFORM_AIX_V3) || defined(__BEOS__) +#error "O_NONBLOCK does not work on this platform" +#endif + +int +main () +{ + /* O_NONBLOCK source test */ + int flags = 0; + if(0 != fcntl(0, F_SETFL, flags | O_NONBLOCK)) + return 1; + return 0; +} +#endif + +/* tests for gethostbyaddr_r or gethostbyname_r */ +#if defined(HAVE_GETHOSTBYADDR_R_5_REENTRANT) || \ + defined(HAVE_GETHOSTBYADDR_R_7_REENTRANT) || \ + defined(HAVE_GETHOSTBYADDR_R_8_REENTRANT) || \ + defined(HAVE_GETHOSTBYNAME_R_3_REENTRANT) || \ + defined(HAVE_GETHOSTBYNAME_R_5_REENTRANT) || \ + defined(HAVE_GETHOSTBYNAME_R_6_REENTRANT) +# define _REENTRANT + /* no idea whether _REENTRANT is always set, just invent a new flag */ +# define TEST_GETHOSTBYFOO_REENTRANT +#endif +#if defined(HAVE_GETHOSTBYADDR_R_5) || \ + defined(HAVE_GETHOSTBYADDR_R_7) || \ + defined(HAVE_GETHOSTBYADDR_R_8) || \ + defined(HAVE_GETHOSTBYNAME_R_3) || \ + defined(HAVE_GETHOSTBYNAME_R_5) || \ + defined(HAVE_GETHOSTBYNAME_R_6) || \ + defined(TEST_GETHOSTBYFOO_REENTRANT) +#include +#include +int main(void) +{ + char *address = "example.com"; + int length = 0; + int type = 0; + struct hostent h; + int rc = 0; +#if defined(HAVE_GETHOSTBYADDR_R_5) || \ + defined(HAVE_GETHOSTBYADDR_R_5_REENTRANT) || \ + \ + defined(HAVE_GETHOSTBYNAME_R_3) || \ + defined(HAVE_GETHOSTBYNAME_R_3_REENTRANT) + struct hostent_data hdata; +#elif defined(HAVE_GETHOSTBYADDR_R_7) || \ + defined(HAVE_GETHOSTBYADDR_R_7_REENTRANT) || \ + defined(HAVE_GETHOSTBYADDR_R_8) || \ + defined(HAVE_GETHOSTBYADDR_R_8_REENTRANT) || \ + \ + defined(HAVE_GETHOSTBYNAME_R_5) || \ + defined(HAVE_GETHOSTBYNAME_R_5_REENTRANT) || \ + defined(HAVE_GETHOSTBYNAME_R_6) || \ + defined(HAVE_GETHOSTBYNAME_R_6_REENTRANT) + char buffer[8192]; + int h_errnop; + struct hostent *hp; +#endif + +#ifndef gethostbyaddr_r + (void)gethostbyaddr_r; +#endif + +#if defined(HAVE_GETHOSTBYADDR_R_5) || \ + defined(HAVE_GETHOSTBYADDR_R_5_REENTRANT) + rc = gethostbyaddr_r(address, length, type, &h, &hdata); + (void)rc; +#elif defined(HAVE_GETHOSTBYADDR_R_7) || \ + defined(HAVE_GETHOSTBYADDR_R_7_REENTRANT) + hp = gethostbyaddr_r(address, length, type, &h, buffer, 8192, &h_errnop); + (void)hp; +#elif defined(HAVE_GETHOSTBYADDR_R_8) || \ + defined(HAVE_GETHOSTBYADDR_R_8_REENTRANT) + rc = gethostbyaddr_r(address, length, type, &h, buffer, 8192, &hp, &h_errnop); + (void)rc; +#endif + +#if defined(HAVE_GETHOSTBYNAME_R_3) || \ + defined(HAVE_GETHOSTBYNAME_R_3_REENTRANT) + rc = gethostbyname_r(address, &h, &hdata); +#elif defined(HAVE_GETHOSTBYNAME_R_5) || \ + defined(HAVE_GETHOSTBYNAME_R_5_REENTRANT) + rc = gethostbyname_r(address, &h, buffer, 8192, &h_errnop); + (void)hp; /* not used for test */ +#elif defined(HAVE_GETHOSTBYNAME_R_6) || \ + defined(HAVE_GETHOSTBYNAME_R_6_REENTRANT) + rc = gethostbyname_r(address, &h, buffer, 8192, &hp, &h_errnop); +#endif + + (void)length; + (void)type; + (void)rc; + return 0; +} +#endif + +#ifdef HAVE_SOCKLEN_T +#ifdef _WIN32 +#include +#else +#include +#include +#endif +int +main () +{ +if ((socklen_t *) 0) + return 0; +if (sizeof (socklen_t)) + return 0; + ; + return 0; +} +#endif +#ifdef HAVE_IN_ADDR_T +#include +#include +#include + +int +main () +{ +if ((in_addr_t *) 0) + return 0; +if (sizeof (in_addr_t)) + return 0; + ; + return 0; +} +#endif + +#ifdef HAVE_BOOL_T +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_STDBOOL_H +#include +#endif +int +main () +{ +if (sizeof (bool *) ) + return 0; + ; + return 0; +} +#endif + +#ifdef STDC_HEADERS +#include +#include +#include +#include +int main() { return 0; } +#endif +#ifdef RETSIGTYPE_TEST +#include +#include +#ifdef signal +# undef signal +#endif +#ifdef __cplusplus +extern "C" void (*signal (int, void (*)(int)))(int); +#else +void (*signal ()) (); +#endif + +int +main () +{ + return 0; +} +#endif +#ifdef HAVE_INET_NTOA_R_DECL +#include + +typedef void (*func_type)(); + +int main() +{ +#ifndef inet_ntoa_r + func_type func; + func = (func_type)inet_ntoa_r; + (void)func; +#endif + return 0; +} +#endif +#ifdef HAVE_INET_NTOA_R_DECL_REENTRANT +#define _REENTRANT +#include + +typedef void (*func_type)(); + +int main() +{ +#ifndef inet_ntoa_r + func_type func; + func = (func_type)&inet_ntoa_r; + (void)func; +#endif + return 0; +} +#endif +#ifdef HAVE_GETADDRINFO +#include +#include +#include + +int main(void) { + struct addrinfo hints, *ai; + int error; + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; +#ifndef getaddrinfo + (void)getaddrinfo; +#endif + error = getaddrinfo("127.0.0.1", "8080", &hints, &ai); + if (error) { + return 1; + } + return 0; +} +#endif +#ifdef HAVE_FILE_OFFSET_BITS +#ifdef _FILE_OFFSET_BITS +#undef _FILE_OFFSET_BITS +#endif +#define _FILE_OFFSET_BITS 64 +#include + /* Check that off_t can represent 2**63 - 1 correctly. + We can't simply define LARGE_OFF_T to be 9223372036854775807, + since some C++ compilers masquerading as C compilers + incorrectly reject 9223372036854775807. */ +#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) + int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 + && LARGE_OFF_T % 2147483647 == 1) + ? 1 : -1]; +int main () { ; return 0; } +#endif +#ifdef HAVE_IOCTLSOCKET +/* includes start */ +#ifdef HAVE_WINDOWS_H +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include +# ifdef HAVE_WINSOCK2_H +# include +# else +# ifdef HAVE_WINSOCK_H +# include +# endif +# endif +#endif + +int +main () +{ + +/* ioctlsocket source code */ + int socket; + unsigned long flags = ioctlsocket(socket, FIONBIO, &flags); + + ; + return 0; +} + +#endif +#ifdef HAVE_IOCTLSOCKET_CAMEL +/* includes start */ +#ifdef HAVE_WINDOWS_H +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include +# ifdef HAVE_WINSOCK2_H +# include +# else +# ifdef HAVE_WINSOCK_H +# include +# endif +# endif +#endif + +int +main () +{ + +/* IoctlSocket source code */ + if(0 != IoctlSocket(0, 0, 0)) + return 1; + ; + return 0; +} +#endif +#ifdef HAVE_IOCTLSOCKET_CAMEL_FIONBIO +/* includes start */ +#ifdef HAVE_WINDOWS_H +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include +# ifdef HAVE_WINSOCK2_H +# include +# else +# ifdef HAVE_WINSOCK_H +# include +# endif +# endif +#endif + +int +main () +{ + +/* IoctlSocket source code */ + long flags = 0; + if(0 != ioctlsocket(0, FIONBIO, &flags)) + return 1; + ; + return 0; +} +#endif +#ifdef HAVE_IOCTLSOCKET_FIONBIO +/* includes start */ +#ifdef HAVE_WINDOWS_H +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include +# ifdef HAVE_WINSOCK2_H +# include +# else +# ifdef HAVE_WINSOCK_H +# include +# endif +# endif +#endif + +int +main () +{ + + int flags = 0; + if(0 != ioctlsocket(0, FIONBIO, &flags)) + return 1; + + ; + return 0; +} +#endif +#ifdef HAVE_IOCTL_FIONBIO +/* headers for FIONBIO test */ +/* includes start */ +#ifdef HAVE_SYS_TYPES_H +# include +#endif +#ifdef HAVE_UNISTD_H +# include +#endif +#ifdef HAVE_SYS_SOCKET_H +# include +#endif +#ifdef HAVE_SYS_IOCTL_H +# include +#endif +#ifdef HAVE_STROPTS_H +# include +#endif + +int +main () +{ + + int flags = 0; + if(0 != ioctl(0, FIONBIO, &flags)) + return 1; + + ; + return 0; +} +#endif +#ifdef HAVE_IOCTL_SIOCGIFADDR +/* headers for FIONBIO test */ +/* includes start */ +#ifdef HAVE_SYS_TYPES_H +# include +#endif +#ifdef HAVE_UNISTD_H +# include +#endif +#ifdef HAVE_SYS_SOCKET_H +# include +#endif +#ifdef HAVE_SYS_IOCTL_H +# include +#endif +#ifdef HAVE_STROPTS_H +# include +#endif +#include + +int +main () +{ + struct ifreq ifr; + if(0 != ioctl(0, SIOCGIFADDR, &ifr)) + return 1; + + ; + return 0; +} +#endif +#ifdef HAVE_SETSOCKOPT_SO_NONBLOCK +/* includes start */ +#ifdef HAVE_WINDOWS_H +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include +# ifdef HAVE_WINSOCK2_H +# include +# else +# ifdef HAVE_WINSOCK_H +# include +# endif +# endif +#endif +/* includes start */ +#ifdef HAVE_SYS_TYPES_H +# include +#endif +#ifdef HAVE_SYS_SOCKET_H +# include +#endif +/* includes end */ + +int +main () +{ + if(0 != setsockopt(0, SOL_SOCKET, SO_NONBLOCK, 0, 0)) + return 1; + ; + return 0; +} +#endif +#ifdef HAVE_GLIBC_STRERROR_R +#include +#include + +void check(char c) {} + +int +main () { + char buffer[1024]; + /* This will not compile if strerror_r does not return a char* */ + check(strerror_r(EACCES, buffer, sizeof(buffer))[0]); + return 0; +} +#endif +#ifdef HAVE_POSIX_STRERROR_R +#include +#include + +/* float, because a pointer can't be implicitly cast to float */ +void check(float f) {} + +int +main () { + char buffer[1024]; + /* This will not compile if strerror_r does not return an int */ + check(strerror_r(EACCES, buffer, sizeof(buffer))); + return 0; +} +#endif +#ifdef HAVE_FSETXATTR_6 +#include /* header from libc, not from libattr */ +int +main() { + fsetxattr(0, 0, 0, 0, 0, 0); + return 0; +} +#endif +#ifdef HAVE_FSETXATTR_5 +#include /* header from libc, not from libattr */ +int +main() { + fsetxattr(0, 0, 0, 0, 0); + return 0; +} +#endif +#ifdef HAVE_CLOCK_GETTIME_MONOTONIC +#include +int +main() { + struct timespec ts = {0, 0}; + clock_gettime(CLOCK_MONOTONIC, &ts); + return 0; +} +#endif +#ifdef HAVE_BUILTIN_AVAILABLE +int +main() { + if(__builtin_available(macOS 10.12, *)) {} + return 0; +} +#endif +#ifdef HAVE_VARIADIC_MACROS_C99 +#define c99_vmacro3(first, ...) fun3(first, __VA_ARGS__) +#define c99_vmacro2(first, ...) fun2(first, __VA_ARGS__) + +int fun3(int arg1, int arg2, int arg3); +int fun2(int arg1, int arg2); + +int fun3(int arg1, int arg2, int arg3) { + return arg1 + arg2 + arg3; +} +int fun2(int arg1, int arg2) { + return arg1 + arg2; +} + +int +main() { + int res3 = c99_vmacro3(1, 2, 3); + int res2 = c99_vmacro2(1, 2); + (void)res3; + (void)res2; + return 0; +} +#endif +#ifdef HAVE_VARIADIC_MACROS_GCC +#define gcc_vmacro3(first, args...) fun3(first, args) +#define gcc_vmacro2(first, args...) fun2(first, args) + +int fun3(int arg1, int arg2, int arg3); +int fun2(int arg1, int arg2); + +int fun3(int arg1, int arg2, int arg3) { + return arg1 + arg2 + arg3; +} +int fun2(int arg1, int arg2) { + return arg1 + arg2; +} + +int +main() { + int res3 = gcc_vmacro3(1, 2, 3); + int res2 = gcc_vmacro2(1, 2); + (void)res3; + (void)res2; + return 0; +} +#endif diff --git a/contrib/curl-cmake/CMake/Macros.cmake b/contrib/curl-cmake/CMake/Macros.cmake new file mode 100644 index 00000000000..39a60b04df9 --- /dev/null +++ b/contrib/curl-cmake/CMake/Macros.cmake @@ -0,0 +1,84 @@ +#File defines convenience macros for available feature testing + +# This macro checks if the symbol exists in the library and if it +# does, it prepends library to the list. It is intended to be called +# multiple times with a sequence of possibly dependent libraries in +# order of least-to-most-dependent. Some libraries depend on others +# to link correctly. +macro(check_library_exists_concat LIBRARY SYMBOL VARIABLE) + check_library_exists("${LIBRARY};${CURL_LIBS}" ${SYMBOL} "${CMAKE_LIBRARY_PATH}" + ${VARIABLE}) + if(${VARIABLE}) + set(CURL_LIBS ${LIBRARY} ${CURL_LIBS}) + endif() +endmacro() + +# Check if header file exists and add it to the list. +# This macro is intended to be called multiple times with a sequence of +# possibly dependent header files. Some headers depend on others to be +# compiled correctly. +macro(check_include_file_concat FILE VARIABLE) + check_include_files("${CURL_INCLUDES};${FILE}" ${VARIABLE}) + if(${VARIABLE}) + set(CURL_INCLUDES ${CURL_INCLUDES} ${FILE}) + set(CURL_TEST_DEFINES "${CURL_TEST_DEFINES} -D${VARIABLE}") + endif() +endmacro() + +# For other curl specific tests, use this macro. +macro(curl_internal_test CURL_TEST) + if(NOT DEFINED "${CURL_TEST}") + set(MACRO_CHECK_FUNCTION_DEFINITIONS + "-D${CURL_TEST} ${CURL_TEST_DEFINES} ${CMAKE_REQUIRED_FLAGS}") + if(CMAKE_REQUIRED_LIBRARIES) + set(CURL_TEST_ADD_LIBRARIES + "-DLINK_LIBRARIES:STRING=${CMAKE_REQUIRED_LIBRARIES}") + endif() + + try_compile(${CURL_TEST} + ${CMAKE_BINARY_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/CMake/CurlTests.c + CMAKE_FLAGS -DCOMPILE_DEFINITIONS:STRING=${MACRO_CHECK_FUNCTION_DEFINITIONS} + "${CURL_TEST_ADD_LIBRARIES}" + OUTPUT_VARIABLE OUTPUT) + if(${CURL_TEST}) + set(${CURL_TEST} 1 CACHE INTERNAL "Curl test ${FUNCTION}") + file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log + "Performing Curl Test ${CURL_TEST} passed with the following output:\n" + "${OUTPUT}\n") + else() + set(${CURL_TEST} "" CACHE INTERNAL "Curl test ${FUNCTION}") + file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log + "Performing Curl Test ${CURL_TEST} failed with the following output:\n" + "${OUTPUT}\n") + endif() + endif() +endmacro() + +macro(curl_nroff_check) + find_program(NROFF NAMES gnroff nroff) + if(NROFF) + # Need a way to write to stdin, this will do + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/nroff-input.txt" "test") + # Tests for a valid nroff option to generate a manpage + foreach(_MANOPT "-man" "-mandoc") + execute_process(COMMAND "${NROFF}" ${_MANOPT} + OUTPUT_VARIABLE NROFF_MANOPT_OUTPUT + INPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/nroff-input.txt" + ERROR_QUIET) + # Save the option if it was valid + if(NROFF_MANOPT_OUTPUT) + set(NROFF_MANOPT ${_MANOPT}) + set(NROFF_USEFUL ON) + break() + endif() + endforeach() + # No need for the temporary file + file(REMOVE "${CMAKE_CURRENT_BINARY_DIR}/nroff-input.txt") + if(NOT NROFF_USEFUL) + message(WARNING "Found no *nroff option to get plaintext from man pages") + endif() + else() + message(WARNING "Found no *nroff program") + endif() +endmacro() diff --git a/contrib/curl-cmake/CMake/OtherTests.cmake b/contrib/curl-cmake/CMake/OtherTests.cmake new file mode 100644 index 00000000000..5ad70a62fb2 --- /dev/null +++ b/contrib/curl-cmake/CMake/OtherTests.cmake @@ -0,0 +1,260 @@ +include(CheckCSourceCompiles) +# The begin of the sources (macros and includes) +set(_source_epilogue "#undef inline") + +macro(add_header_include check header) + if(${check}) + set(_source_epilogue "${_source_epilogue}\n#include <${header}>") + endif() +endmacro() + +set(signature_call_conv) +if(HAVE_WINDOWS_H) + add_header_include(HAVE_WINSOCK2_H "winsock2.h") + add_header_include(HAVE_WINDOWS_H "windows.h") + add_header_include(HAVE_WINSOCK_H "winsock.h") + set(_source_epilogue + "${_source_epilogue}\n#ifndef WIN32_LEAN_AND_MEAN\n#define WIN32_LEAN_AND_MEAN\n#endif") + set(signature_call_conv "PASCAL") + if(HAVE_LIBWS2_32) + set(CMAKE_REQUIRED_LIBRARIES ws2_32) + endif() +else() + add_header_include(HAVE_SYS_TYPES_H "sys/types.h") + add_header_include(HAVE_SYS_SOCKET_H "sys/socket.h") +endif() + +set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) + +check_c_source_compiles("${_source_epilogue} +int main(void) { + recv(0, 0, 0, 0); + return 0; +}" curl_cv_recv) +if(curl_cv_recv) + if(NOT DEFINED curl_cv_func_recv_args OR "${curl_cv_func_recv_args}" STREQUAL "unknown") + foreach(recv_retv "int" "ssize_t" ) + foreach(recv_arg1 "SOCKET" "int" ) + foreach(recv_arg2 "char *" "void *" ) + foreach(recv_arg3 "int" "size_t" "socklen_t" "unsigned int") + foreach(recv_arg4 "int" "unsigned int") + if(NOT curl_cv_func_recv_done) + unset(curl_cv_func_recv_test CACHE) + check_c_source_compiles(" + ${_source_epilogue} + extern ${recv_retv} ${signature_call_conv} + recv(${recv_arg1}, ${recv_arg2}, ${recv_arg3}, ${recv_arg4}); + int main(void) { + ${recv_arg1} s=0; + ${recv_arg2} buf=0; + ${recv_arg3} len=0; + ${recv_arg4} flags=0; + ${recv_retv} res = recv(s, buf, len, flags); + (void) res; + return 0; + }" + curl_cv_func_recv_test) + if(curl_cv_func_recv_test) + set(curl_cv_func_recv_args + "${recv_arg1},${recv_arg2},${recv_arg3},${recv_arg4},${recv_retv}") + set(RECV_TYPE_ARG1 "${recv_arg1}") + set(RECV_TYPE_ARG2 "${recv_arg2}") + set(RECV_TYPE_ARG3 "${recv_arg3}") + set(RECV_TYPE_ARG4 "${recv_arg4}") + set(RECV_TYPE_RETV "${recv_retv}") + set(HAVE_RECV 1) + set(curl_cv_func_recv_done 1) + endif() + endif() + endforeach() + endforeach() + endforeach() + endforeach() + endforeach() + else() + string(REGEX REPLACE "^([^,]*),[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG1 "${curl_cv_func_recv_args}") + string(REGEX REPLACE "^[^,]*,([^,]*),[^,]*,[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG2 "${curl_cv_func_recv_args}") + string(REGEX REPLACE "^[^,]*,[^,]*,([^,]*),[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG3 "${curl_cv_func_recv_args}") + string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,([^,]*),[^,]*$" "\\1" RECV_TYPE_ARG4 "${curl_cv_func_recv_args}") + string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,[^,]*,([^,]*)$" "\\1" RECV_TYPE_RETV "${curl_cv_func_recv_args}") + endif() + + if("${curl_cv_func_recv_args}" STREQUAL "unknown") + message(FATAL_ERROR "Cannot find proper types to use for recv args") + endif() +else() + message(FATAL_ERROR "Unable to link function recv") +endif() +set(curl_cv_func_recv_args "${curl_cv_func_recv_args}" CACHE INTERNAL "Arguments for recv") +set(HAVE_RECV 1) + +check_c_source_compiles("${_source_epilogue} +int main(void) { + send(0, 0, 0, 0); + return 0; +}" curl_cv_send) +if(curl_cv_send) + if(NOT DEFINED curl_cv_func_send_args OR "${curl_cv_func_send_args}" STREQUAL "unknown") + foreach(send_retv "int" "ssize_t" ) + foreach(send_arg1 "SOCKET" "int" "ssize_t" ) + foreach(send_arg2 "const char *" "const void *" "void *" "char *") + foreach(send_arg3 "int" "size_t" "socklen_t" "unsigned int") + foreach(send_arg4 "int" "unsigned int") + if(NOT curl_cv_func_send_done) + unset(curl_cv_func_send_test CACHE) + check_c_source_compiles(" + ${_source_epilogue} + extern ${send_retv} ${signature_call_conv} + send(${send_arg1}, ${send_arg2}, ${send_arg3}, ${send_arg4}); + int main(void) { + ${send_arg1} s=0; + ${send_arg2} buf=0; + ${send_arg3} len=0; + ${send_arg4} flags=0; + ${send_retv} res = send(s, buf, len, flags); + (void) res; + return 0; + }" + curl_cv_func_send_test) + if(curl_cv_func_send_test) + string(REGEX REPLACE "(const) .*" "\\1" send_qual_arg2 "${send_arg2}") + string(REGEX REPLACE "const (.*)" "\\1" send_arg2 "${send_arg2}") + set(curl_cv_func_send_args + "${send_arg1},${send_arg2},${send_arg3},${send_arg4},${send_retv},${send_qual_arg2}") + set(SEND_TYPE_ARG1 "${send_arg1}") + set(SEND_TYPE_ARG2 "${send_arg2}") + set(SEND_TYPE_ARG3 "${send_arg3}") + set(SEND_TYPE_ARG4 "${send_arg4}") + set(SEND_TYPE_RETV "${send_retv}") + set(HAVE_SEND 1) + set(curl_cv_func_send_done 1) + endif() + endif() + endforeach() + endforeach() + endforeach() + endforeach() + endforeach() + else() + string(REGEX REPLACE "^([^,]*),[^,]*,[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG1 "${curl_cv_func_send_args}") + string(REGEX REPLACE "^[^,]*,([^,]*),[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG2 "${curl_cv_func_send_args}") + string(REGEX REPLACE "^[^,]*,[^,]*,([^,]*),[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG3 "${curl_cv_func_send_args}") + string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,([^,]*),[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG4 "${curl_cv_func_send_args}") + string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,[^,]*,([^,]*),[^,]*$" "\\1" SEND_TYPE_RETV "${curl_cv_func_send_args}") + string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,[^,]*,[^,]*,([^,]*)$" "\\1" SEND_QUAL_ARG2 "${curl_cv_func_send_args}") + endif() + + if("${curl_cv_func_send_args}" STREQUAL "unknown") + message(FATAL_ERROR "Cannot find proper types to use for send args") + endif() + set(SEND_QUAL_ARG2 "const") +else() + message(FATAL_ERROR "Unable to link function send") +endif() +set(curl_cv_func_send_args "${curl_cv_func_send_args}" CACHE INTERNAL "Arguments for send") +set(HAVE_SEND 1) + +check_c_source_compiles("${_source_epilogue} + int main(void) { + int flag = MSG_NOSIGNAL; + (void)flag; + return 0; + }" HAVE_MSG_NOSIGNAL) + +if(NOT HAVE_WINDOWS_H) + add_header_include(HAVE_SYS_TIME_H "sys/time.h") + add_header_include(TIME_WITH_SYS_TIME "time.h") + add_header_include(HAVE_TIME_H "time.h") +endif() +check_c_source_compiles("${_source_epilogue} +int main(void) { + struct timeval ts; + ts.tv_sec = 0; + ts.tv_usec = 0; + (void)ts; + return 0; +}" HAVE_STRUCT_TIMEVAL) + +set(HAVE_SIG_ATOMIC_T 1) +set(CMAKE_REQUIRED_FLAGS) +if(HAVE_SIGNAL_H) + set(CMAKE_REQUIRED_FLAGS "-DHAVE_SIGNAL_H") + set(CMAKE_EXTRA_INCLUDE_FILES "signal.h") +endif() +check_type_size("sig_atomic_t" SIZEOF_SIG_ATOMIC_T) +if(HAVE_SIZEOF_SIG_ATOMIC_T) + check_c_source_compiles(" + #ifdef HAVE_SIGNAL_H + # include + #endif + int main(void) { + static volatile sig_atomic_t dummy = 0; + (void)dummy; + return 0; + }" HAVE_SIG_ATOMIC_T_NOT_VOLATILE) + if(NOT HAVE_SIG_ATOMIC_T_NOT_VOLATILE) + set(HAVE_SIG_ATOMIC_T_VOLATILE 1) + endif() +endif() + +if(HAVE_WINDOWS_H) + set(CMAKE_EXTRA_INCLUDE_FILES winsock2.h) +else() + set(CMAKE_EXTRA_INCLUDE_FILES) + if(HAVE_SYS_SOCKET_H) + set(CMAKE_EXTRA_INCLUDE_FILES sys/socket.h) + endif() +endif() + +check_type_size("struct sockaddr_storage" SIZEOF_STRUCT_SOCKADDR_STORAGE) +if(HAVE_SIZEOF_STRUCT_SOCKADDR_STORAGE) + set(HAVE_STRUCT_SOCKADDR_STORAGE 1) +endif() + +unset(CMAKE_TRY_COMPILE_TARGET_TYPE) + +if(NOT DEFINED CMAKE_TOOLCHAIN_FILE) + # if not cross-compilation... + include(CheckCSourceRuns) + set(CMAKE_REQUIRED_FLAGS "") + if(HAVE_SYS_POLL_H) + set(CMAKE_REQUIRED_FLAGS "-DHAVE_SYS_POLL_H") + elseif(HAVE_POLL_H) + set(CMAKE_REQUIRED_FLAGS "-DHAVE_POLL_H") + endif() + check_c_source_runs(" + #include + #include + + #ifdef HAVE_SYS_POLL_H + # include + #elif HAVE_POLL_H + # include + #endif + + int main(void) + { + if(0 != poll(0, 0, 10)) { + return 1; /* fail */ + } + else { + /* detect the 10.12 poll() breakage */ + struct timeval before, after; + int rc; + size_t us; + + gettimeofday(&before, NULL); + rc = poll(NULL, 0, 500); + gettimeofday(&after, NULL); + + us = (after.tv_sec - before.tv_sec) * 1000000 + + (after.tv_usec - before.tv_usec); + + if(us < 400000) { + return 1; + } + } + return 0; + }" HAVE_POLL_FINE) +endif() + diff --git a/contrib/curl-cmake/CMake/README.md b/contrib/curl-cmake/CMake/README.md new file mode 100644 index 00000000000..a96fe6849ed --- /dev/null +++ b/contrib/curl-cmake/CMake/README.md @@ -0,0 +1,28 @@ +#*************************************************************************** +# _ _ ____ _ +# Project ___| | | | _ \| | +# / __| | | | |_) | | +# | (__| |_| | _ <| |___ +# \___|\___/|_| \_\_____| +# +# Copyright (C) 1998 - 2019, Daniel Stenberg, , et al. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at https://curl.haxx.se/docs/copyright.html. +# +# You may opt to use, copy, modify, merge, publish, distribute and/or sell +# copies of the Software, and permit persons to whom the Software is +# furnished to do so, under the terms of the COPYING file. +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY +# KIND, either express or implied. +# +########################################################################### + +CMake files under this directory were reused from project curl. +Here are links to original source files: +https://github.com/curl/curl/blob/master/CMake/CurlSymbolHiding.cmake +https://github.com/curl/curl/blob/master/CMake/CurlTests,c +https://github.com/curl/curl/blob/master/CMake/Macros.cmake +https://github.com/curl/curl/blob/master/CMake/OtherTests.cmake diff --git a/contrib/curl-cmake/CMakeLists.patch b/contrib/curl-cmake/CMakeLists.patch new file mode 100644 index 00000000000..3b4ef3bf9ab --- /dev/null +++ b/contrib/curl-cmake/CMakeLists.patch @@ -0,0 +1,1509 @@ +--- ../curl/CMakeLists.txt 2019-12-09 13:33:38.140856959 +0300 ++++ CMakeLists.txt 2019-12-12 19:50:48.001094425 +0300 +@@ -19,220 +19,100 @@ + # KIND, either express or implied. + # + ########################################################################### +-# curl/libcurl CMake script +-# by Tetetest and Sukender (Benoit Neil) +- +-# TODO: +-# The output .so file lacks the soname number which we currently have within the lib/Makefile.am file +-# Add full (4 or 5 libs) SSL support +-# Add INSTALL target (EXTRA_DIST variables in Makefile.am may be moved to Makefile.inc so that CMake/CPack is aware of what's to include). +-# Add CTests(?) +-# Check on all possible platforms +-# Test with as many configurations possible (With or without any option) +-# Create scripts that help keeping the CMake build system up to date (to reduce maintenance). According to Tetetest: +-# - lists of headers that 'configure' checks for; +-# - curl-specific tests (the ones that are in m4/curl-*.m4 files); +-# - (most obvious thing:) curl version numbers. +-# Add documentation subproject +-# +-# To check: +-# (From Daniel Stenberg) The cmake build selected to run gcc with -fPIC on my box while the plain configure script did not. +-# (From Daniel Stenberg) The gcc command line use neither -g nor any -O options. As a developer, I also treasure our configure scripts's --enable-debug option that sets a long range of "picky" compiler options. ++# This file is mostly copy-pasted from original curl CMakeLists.txt + cmake_minimum_required(VERSION 3.0 FATAL_ERROR) ++ ++SET(CURL_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl) ++SET(CURL_LIBRARY_DIR ${CURL_SOURCE_DIR}/lib) + set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake;${CMAKE_MODULE_PATH}") +-include(Utilities) + include(Macros) + include(CMakeDependentOption) + include(CheckCCompilerFlag) + +-project(CURL C) +- +-message(WARNING "the curl cmake build system is poorly maintained. Be aware") +- + file(READ ${CURL_SOURCE_DIR}/include/curl/curlver.h CURL_VERSION_H_CONTENTS) + string(REGEX MATCH "#define LIBCURL_VERSION \"[^\"]*" +- CURL_VERSION ${CURL_VERSION_H_CONTENTS}) ++ CURL_VERSION ${CURL_VERSION_H_CONTENTS}) + string(REGEX REPLACE "[^\"]+\"" "" CURL_VERSION ${CURL_VERSION}) + string(REGEX MATCH "#define LIBCURL_VERSION_NUM 0x[0-9a-fA-F]+" +- CURL_VERSION_NUM ${CURL_VERSION_H_CONTENTS}) ++ CURL_VERSION_NUM ${CURL_VERSION_H_CONTENTS}) + string(REGEX REPLACE "[^0]+0x" "" CURL_VERSION_NUM ${CURL_VERSION_NUM}) + +- +-# Setup package meta-data +-# SET(PACKAGE "curl") +-message(STATUS "curl version=[${CURL_VERSION}]") +-# SET(PACKAGE_TARNAME "curl") +-# SET(PACKAGE_NAME "curl") +-# SET(PACKAGE_VERSION "-") +-# SET(PACKAGE_STRING "curl-") +-# SET(PACKAGE_BUGREPORT "a suitable curl mailing list => https://curl.haxx.se/mail/") ++message(STATUS "Use curl version=[${CURL_VERSION}]") + set(OPERATING_SYSTEM "${CMAKE_SYSTEM_NAME}") + set(OS "\"${CMAKE_SYSTEM_NAME}\"") + +-include_directories(${CURL_SOURCE_DIR}/include) +- +-option(CURL_WERROR "Turn compiler warnings into errors" OFF) + option(PICKY_COMPILER "Enable picky compiler options" ON) +-option(BUILD_CURL_EXE "Set to ON to build curl executable." ON) +-option(BUILD_SHARED_LIBS "Build shared libraries" ON) +-option(ENABLE_ARES "Set to ON to enable c-ares support" OFF) +-if(WIN32) +- option(CURL_STATIC_CRT "Set to ON to build libcurl with static CRT on Windows (/MT)." OFF) +- option(ENABLE_INET_PTON "Set to OFF to prevent usage of inet_pton when building against modern SDKs while still requiring compatibility with older Windows versions, such as Windows XP, Windows Server 2003 etc." ON) +-endif() +- +-cmake_dependent_option(ENABLE_THREADED_RESOLVER "Set to ON to enable threaded DNS lookup" +- ON "NOT ENABLE_ARES" +- OFF) +- +-option(ENABLE_DEBUG "Set to ON to enable curl debug features" OFF) +-option(ENABLE_CURLDEBUG "Set to ON to build with TrackMemory feature enabled" OFF) ++option(ENABLE_THREADED_RESOLVER "Set to ON to enable threaded DNS lookup" ON) + + if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG) +- if(PICKY_COMPILER) +- foreach(_CCOPT -pedantic -Wall -W -Wpointer-arith -Wwrite-strings -Wunused -Wshadow -Winline -Wnested-externs -Wmissing-declarations -Wmissing-prototypes -Wno-long-long -Wfloat-equal -Wno-multichar -Wsign-compare -Wundef -Wno-format-nonliteral -Wendif-labels -Wstrict-prototypes -Wdeclaration-after-statement -Wstrict-aliasing=3 -Wcast-align -Wtype-limits -Wold-style-declaration -Wmissing-parameter-type -Wempty-body -Wclobbered -Wignored-qualifiers -Wconversion -Wno-sign-conversion -Wvla -Wdouble-promotion -Wno-system-headers -Wno-pedantic-ms-format) +- # surprisingly, CHECK_C_COMPILER_FLAG needs a new variable to store each new +- # test result in. +- check_c_compiler_flag(${_CCOPT} OPT${_CCOPT}) +- if(OPT${_CCOPT}) +- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_CCOPT}") +- endif() +- endforeach() +- endif() +-endif() +- +-if(ENABLE_DEBUG) +- # DEBUGBUILD will be defined only for Debug builds +- set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS $<$:DEBUGBUILD>) +- set(ENABLE_CURLDEBUG ON) +-endif() +- +-if(ENABLE_CURLDEBUG) +- set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS CURLDEBUG) ++ if(PICKY_COMPILER) ++ foreach(_CCOPT -pedantic -Wall -W -Wpointer-arith -Wwrite-strings -Wunused -Wshadow -Winline -Wnested-externs -Wmissing-declarations -Wmissing-prototypes -Wno-long-long -Wfloat-equal -Wno-multichar -Wsign-compare -Wundef -Wno-format-nonliteral -Wendif-labels -Wstrict-prototypes -Wdeclaration-after-statement -Wstrict-aliasing=3 -Wcast-align -Wtype-limits -Wold-style-declaration -Wmissing-parameter-type -Wempty-body -Wclobbered -Wignored-qualifiers -Wconversion -Wno-sign-conversion -Wvla -Wdouble-promotion -Wno-system-headers -Wno-pedantic-ms-format) ++ # surprisingly, CHECK_C_COMPILER_FLAG needs a new variable to store each new ++ # test result in. ++ check_c_compiler_flag(${_CCOPT} OPT${_CCOPT}) ++ if(OPT${_CCOPT}) ++ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_CCOPT}") ++ endif() ++ endforeach() ++ endif() + endif() + + # For debug libs and exes, add "-d" postfix + if(NOT DEFINED CMAKE_DEBUG_POSTFIX) +- set(CMAKE_DEBUG_POSTFIX "-d") ++ set(CMAKE_DEBUG_POSTFIX "-d") + endif() + + # initialize CURL_LIBS + set(CURL_LIBS "") + +-if(ENABLE_ARES) +- set(USE_ARES 1) +- find_package(CARES REQUIRED) +- list(APPEND CURL_LIBS ${CARES_LIBRARY}) +- set(CURL_LIBS ${CURL_LIBS} ${CARES_LIBRARY}) +-endif() +- + include(CurlSymbolHiding) + +-option(HTTP_ONLY "disables all protocols except HTTP (This overrides all CURL_DISABLE_* options)" OFF) +-mark_as_advanced(HTTP_ONLY) +-option(CURL_DISABLE_FTP "disables FTP" OFF) +-mark_as_advanced(CURL_DISABLE_FTP) +-option(CURL_DISABLE_LDAP "disables LDAP" OFF) +-mark_as_advanced(CURL_DISABLE_LDAP) +-option(CURL_DISABLE_TELNET "disables Telnet" OFF) +-mark_as_advanced(CURL_DISABLE_TELNET) +-option(CURL_DISABLE_DICT "disables DICT" OFF) +-mark_as_advanced(CURL_DISABLE_DICT) +-option(CURL_DISABLE_FILE "disables FILE" OFF) +-mark_as_advanced(CURL_DISABLE_FILE) +-option(CURL_DISABLE_TFTP "disables TFTP" OFF) +-mark_as_advanced(CURL_DISABLE_TFTP) +-option(CURL_DISABLE_HTTP "disables HTTP" OFF) +-mark_as_advanced(CURL_DISABLE_HTTP) +- +-option(CURL_DISABLE_LDAPS "to disable LDAPS" OFF) +-mark_as_advanced(CURL_DISABLE_LDAPS) +- +-option(CURL_DISABLE_RTSP "to disable RTSP" OFF) +-mark_as_advanced(CURL_DISABLE_RTSP) +-option(CURL_DISABLE_PROXY "to disable proxy" OFF) +-mark_as_advanced(CURL_DISABLE_PROXY) +-option(CURL_DISABLE_POP3 "to disable POP3" OFF) +-mark_as_advanced(CURL_DISABLE_POP3) +-option(CURL_DISABLE_IMAP "to disable IMAP" OFF) +-mark_as_advanced(CURL_DISABLE_IMAP) +-option(CURL_DISABLE_SMTP "to disable SMTP" OFF) +-mark_as_advanced(CURL_DISABLE_SMTP) +-option(CURL_DISABLE_GOPHER "to disable Gopher" OFF) +-mark_as_advanced(CURL_DISABLE_GOPHER) +- +-if(HTTP_ONLY) +- set(CURL_DISABLE_FTP ON) +- set(CURL_DISABLE_LDAP ON) +- set(CURL_DISABLE_LDAPS ON) +- set(CURL_DISABLE_TELNET ON) +- set(CURL_DISABLE_DICT ON) +- set(CURL_DISABLE_FILE ON) +- set(CURL_DISABLE_TFTP ON) +- set(CURL_DISABLE_RTSP ON) +- set(CURL_DISABLE_POP3 ON) +- set(CURL_DISABLE_IMAP ON) +- set(CURL_DISABLE_SMTP ON) +- set(CURL_DISABLE_GOPHER ON) +-endif() ++# Http only ++set(CURL_DISABLE_FTP ON) ++set(CURL_DISABLE_LDAP ON) ++set(CURL_DISABLE_LDAPS ON) ++set(CURL_DISABLE_TELNET ON) ++set(CURL_DISABLE_DICT ON) ++set(CURL_DISABLE_FILE ON) ++set(CURL_DISABLE_TFTP ON) ++set(CURL_DISABLE_RTSP ON) ++set(CURL_DISABLE_POP3 ON) ++set(CURL_DISABLE_IMAP ON) ++set(CURL_DISABLE_SMTP ON) ++set(CURL_DISABLE_GOPHER ON) + + option(CURL_DISABLE_COOKIES "to disable cookies support" OFF) + mark_as_advanced(CURL_DISABLE_COOKIES) + + option(CURL_DISABLE_CRYPTO_AUTH "to disable cryptographic authentication" OFF) + mark_as_advanced(CURL_DISABLE_CRYPTO_AUTH) ++ + option(CURL_DISABLE_VERBOSE_STRINGS "to disable verbose strings" OFF) + mark_as_advanced(CURL_DISABLE_VERBOSE_STRINGS) ++ + option(ENABLE_IPV6 "Define if you want to enable IPv6 support" ON) + mark_as_advanced(ENABLE_IPV6) ++ + if(ENABLE_IPV6 AND NOT WIN32) +- include(CheckStructHasMember) +- check_struct_has_member("struct sockaddr_in6" sin6_addr "netinet/in.h" +- HAVE_SOCKADDR_IN6_SIN6_ADDR) +- check_struct_has_member("struct sockaddr_in6" sin6_scope_id "netinet/in.h" +- HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID) +- if(NOT HAVE_SOCKADDR_IN6_SIN6_ADDR) +- message(WARNING "struct sockaddr_in6 not available, disabling IPv6 support") +- # Force the feature off as this name is used as guard macro... +- set(ENABLE_IPV6 OFF +- CACHE BOOL "Define if you want to enable IPv6 support" FORCE) +- endif() ++ include(CheckStructHasMember) ++ check_struct_has_member("struct sockaddr_in6" sin6_addr "netinet/in.h" ++ HAVE_SOCKADDR_IN6_SIN6_ADDR) ++ check_struct_has_member("struct sockaddr_in6" sin6_scope_id "netinet/in.h" ++ HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID) ++ if(NOT HAVE_SOCKADDR_IN6_SIN6_ADDR) ++ message(WARNING "struct sockaddr_in6 not available, disabling IPv6 support") ++ # Force the feature off as this name is used as guard macro... ++ set(ENABLE_IPV6 OFF ++ CACHE BOOL "Define if you want to enable IPv6 support" FORCE) ++ endif() + endif() + + curl_nroff_check() +-find_package(Perl) +- +-cmake_dependent_option(ENABLE_MANUAL "to provide the built-in manual" +- ON "NROFF_USEFUL;PERL_FOUND" +- OFF) +- +-if(NOT PERL_FOUND) +- message(STATUS "Perl not found, testing disabled.") +- set(BUILD_TESTING OFF) +-endif() +-if(ENABLE_MANUAL) +- set(USE_MANUAL ON) +-endif() + + # We need ansi c-flags, especially on HP + set(CMAKE_C_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_C_FLAGS}") + set(CMAKE_REQUIRED_FLAGS ${CMAKE_ANSI_CFLAGS}) + +-if(CURL_STATIC_CRT) +- set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /MT") +- set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /MTd") +-endif() +- +-# Disable warnings on Borland to avoid changing 3rd party code. +-if(BORLAND) +- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w-") +-endif() +- +-# If we are on AIX, do the _ALL_SOURCE magic +-if(${CMAKE_SYSTEM_NAME} MATCHES AIX) +- set(_ALL_SOURCE 1) +-endif() +- + # Include all the necessary files for macros + include(CheckFunctionExists) + include(CheckIncludeFile) +@@ -242,21 +122,15 @@ + include(CheckTypeSize) + include(CheckCSourceCompiles) + +-# On windows preload settings +-if(WIN32) +- set(CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS} -D_WINSOCKAPI_=") +- include(${CMAKE_CURRENT_SOURCE_DIR}/CMake/Platforms/WindowsCache.cmake) +-endif() +- + if(ENABLE_THREADED_RESOLVER) +- find_package(Threads REQUIRED) +- if(WIN32) +- set(USE_THREADS_WIN32 ON) +- else() +- set(USE_THREADS_POSIX ${CMAKE_USE_PTHREADS_INIT}) +- set(HAVE_PTHREAD_H ${CMAKE_USE_PTHREADS_INIT}) +- endif() +- set(CURL_LIBS ${CURL_LIBS} ${CMAKE_THREAD_LIBS_INIT}) ++ find_package(Threads REQUIRED) ++ if(WIN32) ++ set(USE_THREADS_WIN32 ON) ++ else() ++ set(USE_THREADS_POSIX ${CMAKE_USE_PTHREADS_INIT}) ++ set(HAVE_PTHREAD_H ${CMAKE_USE_PTHREADS_INIT}) ++ endif() ++ set(CURL_LIBS ${CURL_LIBS} ${CMAKE_THREAD_LIBS_INIT}) + endif() + + # Check for all needed libraries +@@ -264,245 +138,24 @@ + check_library_exists_concat("socket" connect HAVE_LIBSOCKET) + check_library_exists("c" gethostbyname "" NOT_NEED_LIBNSL) + +-# Yellowtab Zeta needs different libraries than BeOS 5. +-if(BEOS) +- set(NOT_NEED_LIBNSL 1) +- check_library_exists_concat("bind" gethostbyname HAVE_LIBBIND) +- check_library_exists_concat("bnetapi" closesocket HAVE_LIBBNETAPI) +-endif() +- +-if(NOT NOT_NEED_LIBNSL) +- check_library_exists_concat("nsl" gethostbyname HAVE_LIBNSL) +-endif() +- + check_function_exists(gethostname HAVE_GETHOSTNAME) + +-if(WIN32) +- check_library_exists_concat("ws2_32" getch HAVE_LIBWS2_32) +- check_library_exists_concat("winmm" getch HAVE_LIBWINMM) +- list(APPEND CURL_LIBS "advapi32") +-endif() +- +-# check SSL libraries +-# TODO support GNUTLS, NSS, POLARSSL, CYASSL +- +-if(APPLE) +- option(CMAKE_USE_SECTRANSP "enable Apple OS native SSL/TLS" OFF) +-endif() +-if(WIN32) +- option(CMAKE_USE_WINSSL "enable Windows native SSL/TLS" OFF) +- cmake_dependent_option(CURL_WINDOWS_SSPI "Use windows libraries to allow NTLM authentication without openssl" ON +- CMAKE_USE_WINSSL OFF) +-endif() +-option(CMAKE_USE_MBEDTLS "Enable mbedTLS for SSL/TLS" OFF) +-option(CMAKE_USE_BEARSSL "Enable BearSSL for SSL/TLS" OFF) +- +-set(openssl_default ON) +-if(WIN32 OR CMAKE_USE_SECTRANSP OR CMAKE_USE_WINSSL OR CMAKE_USE_MBEDTLS) +- set(openssl_default OFF) +-endif() +-option(CMAKE_USE_OPENSSL "Use OpenSSL code. Experimental" ${openssl_default}) +- +-count_true(enabled_ssl_options_count +- CMAKE_USE_WINSSL +- CMAKE_USE_SECTRANSP +- CMAKE_USE_OPENSSL +- CMAKE_USE_MBEDTLS +- CMAKE_USE_BEARSSL +-) +-if(enabled_ssl_options_count GREATER "1") +- set(CURL_WITH_MULTI_SSL ON) +-endif() +- +-if(CMAKE_USE_WINSSL) +- set(SSL_ENABLED ON) +- set(USE_SCHANNEL ON) # Windows native SSL/TLS support +- set(USE_WINDOWS_SSPI ON) # CMAKE_USE_WINSSL implies CURL_WINDOWS_SSPI +- list(APPEND CURL_LIBS "crypt32") +-endif() +-if(CURL_WINDOWS_SSPI) +- set(USE_WINDOWS_SSPI ON) +- set(CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS} -DSECURITY_WIN32") +-endif() ++# From cmake/find/ssl.cmake ++if (OPENSSL_FOUND) ++ set(SSL_ENABLED ON) ++ set(USE_OPENSSL ON) + +-if(CMAKE_USE_DARWINSSL) +- message(FATAL_ERROR "The cmake option CMAKE_USE_DARWINSSL was renamed to CMAKE_USE_SECTRANSP.") +-endif() +- +-if(CMAKE_USE_SECTRANSP) +- find_library(COREFOUNDATION_FRAMEWORK "CoreFoundation") +- if(NOT COREFOUNDATION_FRAMEWORK) +- message(FATAL_ERROR "CoreFoundation framework not found") +- endif() +- +- find_library(SECURITY_FRAMEWORK "Security") +- if(NOT SECURITY_FRAMEWORK) +- message(FATAL_ERROR "Security framework not found") +- endif() +- +- set(SSL_ENABLED ON) +- set(USE_SECTRANSP ON) +- list(APPEND CURL_LIBS "${COREFOUNDATION_FRAMEWORK}" "${SECURITY_FRAMEWORK}") +-endif() +- +-if(CMAKE_USE_OPENSSL) +- find_package(OpenSSL REQUIRED) +- set(SSL_ENABLED ON) +- set(USE_OPENSSL ON) +- +- # Depend on OpenSSL via imported targets if supported by the running +- # version of CMake. This allows our dependents to get our dependencies +- # transitively. +- if(NOT CMAKE_VERSION VERSION_LESS 3.4) +- list(APPEND CURL_LIBS OpenSSL::SSL OpenSSL::Crypto) +- else() + list(APPEND CURL_LIBS ${OPENSSL_LIBRARIES}) +- include_directories(${OPENSSL_INCLUDE_DIR}) +- endif() +- +- set(CMAKE_REQUIRED_INCLUDES ${OPENSSL_INCLUDE_DIR}) +- check_include_file("openssl/crypto.h" HAVE_OPENSSL_CRYPTO_H) +- check_include_file("openssl/err.h" HAVE_OPENSSL_ERR_H) +- check_include_file("openssl/pem.h" HAVE_OPENSSL_PEM_H) +- check_include_file("openssl/rsa.h" HAVE_OPENSSL_RSA_H) +- check_include_file("openssl/ssl.h" HAVE_OPENSSL_SSL_H) +- check_include_file("openssl/x509.h" HAVE_OPENSSL_X509_H) +- check_include_file("openssl/rand.h" HAVE_OPENSSL_RAND_H) +- check_symbol_exists(RAND_status "${CURL_INCLUDES}" HAVE_RAND_STATUS) +- check_symbol_exists(RAND_screen "${CURL_INCLUDES}" HAVE_RAND_SCREEN) +- check_symbol_exists(RAND_egd "${CURL_INCLUDES}" HAVE_RAND_EGD) +-endif() +- +-if(CMAKE_USE_MBEDTLS) +- find_package(MbedTLS REQUIRED) +- set(SSL_ENABLED ON) +- set(USE_MBEDTLS ON) +- list(APPEND CURL_LIBS ${MBEDTLS_LIBRARIES}) +- include_directories(${MBEDTLS_INCLUDE_DIRS}) +-endif() +- +-if(CMAKE_USE_BEARSSL) +- find_package(BearSSL REQUIRED) +- set(SSL_ENABLED ON) +- set(USE_BEARSSL ON) +- list(APPEND CURL_LIBS ${BEARSSL_LIBRARY}) +- include_directories(${BEARSSL_INCLUDE_DIRS}) +-endif() +- +-option(USE_NGHTTP2 "Use Nghttp2 library" OFF) +-if(USE_NGHTTP2) +- find_package(NGHTTP2 REQUIRED) +- include_directories(${NGHTTP2_INCLUDE_DIRS}) +- list(APPEND CURL_LIBS ${NGHTTP2_LIBRARIES}) +-endif() +- +-if(NOT CURL_DISABLE_LDAP) +- if(WIN32) +- option(USE_WIN32_LDAP "Use Windows LDAP implementation" ON) +- if(USE_WIN32_LDAP) +- check_library_exists_concat("wldap32" cldap_open HAVE_WLDAP32) +- if(NOT HAVE_WLDAP32) +- set(USE_WIN32_LDAP OFF) +- endif() +- endif() +- endif() +- +- option(CMAKE_USE_OPENLDAP "Use OpenLDAP code." OFF) +- mark_as_advanced(CMAKE_USE_OPENLDAP) +- set(CMAKE_LDAP_LIB "ldap" CACHE STRING "Name or full path to ldap library") +- set(CMAKE_LBER_LIB "lber" CACHE STRING "Name or full path to lber library") +- +- if(CMAKE_USE_OPENLDAP AND USE_WIN32_LDAP) +- message(FATAL_ERROR "Cannot use USE_WIN32_LDAP and CMAKE_USE_OPENLDAP at the same time") +- endif() +- +- # Now that we know, we're not using windows LDAP... +- if(USE_WIN32_LDAP) +- check_include_file_concat("winldap.h" HAVE_WINLDAP_H) +- check_include_file_concat("winber.h" HAVE_WINBER_H) +- else() +- # Check for LDAP +- set(CMAKE_REQUIRED_LIBRARIES ${OPENSSL_LIBRARIES}) +- check_library_exists_concat(${CMAKE_LDAP_LIB} ldap_init HAVE_LIBLDAP) +- check_library_exists_concat(${CMAKE_LBER_LIB} ber_init HAVE_LIBLBER) +- +- set(CMAKE_REQUIRED_INCLUDES_BAK ${CMAKE_REQUIRED_INCLUDES}) +- set(CMAKE_LDAP_INCLUDE_DIR "" CACHE STRING "Path to LDAP include directory") +- if(CMAKE_LDAP_INCLUDE_DIR) +- list(APPEND CMAKE_REQUIRED_INCLUDES ${CMAKE_LDAP_INCLUDE_DIR}) +- endif() +- check_include_file_concat("ldap.h" HAVE_LDAP_H) +- check_include_file_concat("lber.h" HAVE_LBER_H) +- +- if(NOT HAVE_LDAP_H) +- message(STATUS "LDAP_H not found CURL_DISABLE_LDAP set ON") +- set(CURL_DISABLE_LDAP ON CACHE BOOL "" FORCE) +- set(CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES_BAK}) #LDAP includes won't be used +- elseif(NOT HAVE_LIBLDAP) +- message(STATUS "LDAP library '${CMAKE_LDAP_LIB}' not found CURL_DISABLE_LDAP set ON") +- set(CURL_DISABLE_LDAP ON CACHE BOOL "" FORCE) +- set(CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES_BAK}) #LDAP includes won't be used +- else() +- if(CMAKE_USE_OPENLDAP) +- set(USE_OPENLDAP ON) +- endif() +- if(CMAKE_LDAP_INCLUDE_DIR) +- include_directories(${CMAKE_LDAP_INCLUDE_DIR}) +- endif() +- set(NEED_LBER_H ON) +- set(_HEADER_LIST) +- if(HAVE_WINDOWS_H) +- list(APPEND _HEADER_LIST "windows.h") +- endif() +- if(HAVE_SYS_TYPES_H) +- list(APPEND _HEADER_LIST "sys/types.h") +- endif() +- list(APPEND _HEADER_LIST "ldap.h") +- +- set(_SRC_STRING "") +- foreach(_HEADER ${_HEADER_LIST}) +- set(_INCLUDE_STRING "${_INCLUDE_STRING}#include <${_HEADER}>\n") +- endforeach() +- +- set(_SRC_STRING +- " +- ${_INCLUDE_STRING} +- int main(int argc, char ** argv) +- { +- BerValue *bvp = NULL; +- BerElement *bep = ber_init(bvp); +- ber_free(bep, 1); +- return 0; +- }" +- ) +- set(CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS} -DLDAP_DEPRECATED=1") +- list(APPEND CMAKE_REQUIRED_LIBRARIES ${CMAKE_LDAP_LIB}) +- if(HAVE_LIBLBER) +- list(APPEND CMAKE_REQUIRED_LIBRARIES ${CMAKE_LBER_LIB}) +- endif() +- check_c_source_compiles("${_SRC_STRING}" NOT_NEED_LBER_H) +- unset(CMAKE_REQUIRED_LIBRARIES) +- +- if(NOT_NEED_LBER_H) +- set(NEED_LBER_H OFF) +- else() +- set(CURL_TEST_DEFINES "${CURL_TEST_DEFINES} -DNEED_LBER_H") +- endif() +- endif() +- endif() +-endif() +- +-# No ldap, no ldaps. +-if(CURL_DISABLE_LDAP) +- if(NOT CURL_DISABLE_LDAPS) +- message(STATUS "LDAP needs to be enabled to support LDAPS") +- set(CURL_DISABLE_LDAPS ON CACHE BOOL "" FORCE) +- endif() +-endif() +- +-if(NOT CURL_DISABLE_LDAPS) +- check_include_file_concat("ldap_ssl.h" HAVE_LDAP_SSL_H) +- check_include_file_concat("ldapssl.h" HAVE_LDAPSSL_H) ++ check_include_file("openssl/crypto.h" HAVE_OPENSSL_CRYPTO_H) ++ check_include_file("openssl/err.h" HAVE_OPENSSL_ERR_H) ++ check_include_file("openssl/pem.h" HAVE_OPENSSL_PEM_H) ++ check_include_file("openssl/rsa.h" HAVE_OPENSSL_RSA_H) ++ check_include_file("openssl/ssl.h" HAVE_OPENSSL_SSL_H) ++ check_include_file("openssl/x509.h" HAVE_OPENSSL_X509_H) ++ check_include_file("openssl/rand.h" HAVE_OPENSSL_RAND_H) ++ check_symbol_exists(RAND_status "${CURL_INCLUDES}" HAVE_RAND_STATUS) ++ check_symbol_exists(RAND_screen "${CURL_INCLUDES}" HAVE_RAND_SCREEN) ++ check_symbol_exists(RAND_egd "${CURL_INCLUDES}" HAVE_RAND_EGD) + endif() + + # Check for idn +@@ -511,224 +164,30 @@ + # Check for symbol dlopen (same as HAVE_LIBDL) + check_library_exists("${CURL_LIBS}" dlopen "" HAVE_DLOPEN) + +-option(CURL_ZLIB "Set to ON to enable building curl with zlib support." ON) +-set(HAVE_LIBZ OFF) +-set(HAVE_ZLIB_H OFF) +-set(USE_ZLIB OFF) +-if(CURL_ZLIB) +- find_package(ZLIB QUIET) +- if(ZLIB_FOUND) ++# From /cmake/find/zlib.cmake ++if (ZLIB_FOUND) + set(HAVE_ZLIB_H ON) + set(HAVE_LIBZ ON) + set(USE_ZLIB ON) + +- # Depend on ZLIB via imported targets if supported by the running +- # version of CMake. This allows our dependents to get our dependencies +- # transitively. +- if(NOT CMAKE_VERSION VERSION_LESS 3.4) +- list(APPEND CURL_LIBS ZLIB::ZLIB) +- else() +- list(APPEND CURL_LIBS ${ZLIB_LIBRARIES}) +- include_directories(${ZLIB_INCLUDE_DIRS}) +- endif() +- list(APPEND CMAKE_REQUIRED_INCLUDES ${ZLIB_INCLUDE_DIRS}) +- endif() +-endif() +- +-option(CURL_BROTLI "Set to ON to enable building curl with brotli support." OFF) +-set(HAVE_BROTLI OFF) +-if(CURL_BROTLI) +- find_package(Brotli QUIET) +- if(BROTLI_FOUND) +- set(HAVE_BROTLI ON) +- list(APPEND CURL_LIBS ${BROTLI_LIBRARIES}) +- include_directories(${BROTLI_INCLUDE_DIRS}) +- list(APPEND CMAKE_REQUIRED_INCLUDES ${BROTLI_INCLUDE_DIRS}) +- endif() +-endif() +- +-#libSSH2 +-option(CMAKE_USE_LIBSSH2 "Use libSSH2" ON) +-mark_as_advanced(CMAKE_USE_LIBSSH2) +-set(USE_LIBSSH2 OFF) +-set(HAVE_LIBSSH2 OFF) +-set(HAVE_LIBSSH2_H OFF) +- +-if(CMAKE_USE_LIBSSH2) +- find_package(LibSSH2) +- if(LIBSSH2_FOUND) +- list(APPEND CURL_LIBS ${LIBSSH2_LIBRARY}) +- set(CMAKE_REQUIRED_LIBRARIES ${LIBSSH2_LIBRARY}) +- list(APPEND CMAKE_REQUIRED_INCLUDES "${LIBSSH2_INCLUDE_DIR}") +- include_directories("${LIBSSH2_INCLUDE_DIR}") +- set(HAVE_LIBSSH2 ON) +- set(USE_LIBSSH2 ON) +- +- # find_package has already found the headers +- set(HAVE_LIBSSH2_H ON) +- set(CURL_INCLUDES ${CURL_INCLUDES} "${LIBSSH2_INCLUDE_DIR}/libssh2.h") +- set(CURL_TEST_DEFINES "${CURL_TEST_DEFINES} -DHAVE_LIBSSH2_H") +- +- # now check for specific libssh2 symbols as they were added in different versions +- set(CMAKE_EXTRA_INCLUDE_FILES "libssh2.h") +- check_function_exists(libssh2_version HAVE_LIBSSH2_VERSION) +- check_function_exists(libssh2_init HAVE_LIBSSH2_INIT) +- check_function_exists(libssh2_exit HAVE_LIBSSH2_EXIT) +- check_function_exists(libssh2_scp_send64 HAVE_LIBSSH2_SCP_SEND64) +- check_function_exists(libssh2_session_handshake HAVE_LIBSSH2_SESSION_HANDSHAKE) +- set(CMAKE_EXTRA_INCLUDE_FILES "") +- unset(CMAKE_REQUIRED_LIBRARIES) +- endif() +-endif() +- +-option(CMAKE_USE_GSSAPI "Use GSSAPI implementation (right now only Heimdal is supported with CMake build)" OFF) +-mark_as_advanced(CMAKE_USE_GSSAPI) +- +-if(CMAKE_USE_GSSAPI) +- find_package(GSS) +- +- set(HAVE_GSSAPI ${GSS_FOUND}) +- if(GSS_FOUND) +- +- message(STATUS "Found ${GSS_FLAVOUR} GSSAPI version: \"${GSS_VERSION}\"") +- +- list(APPEND CMAKE_REQUIRED_INCLUDES ${GSS_INCLUDE_DIR}) +- check_include_file_concat("gssapi/gssapi.h" HAVE_GSSAPI_GSSAPI_H) +- check_include_file_concat("gssapi/gssapi_generic.h" HAVE_GSSAPI_GSSAPI_GENERIC_H) +- check_include_file_concat("gssapi/gssapi_krb5.h" HAVE_GSSAPI_GSSAPI_KRB5_H) +- +- if(GSS_FLAVOUR STREQUAL "Heimdal") +- set(HAVE_GSSHEIMDAL ON) +- else() # MIT +- set(HAVE_GSSMIT ON) +- set(_INCLUDE_LIST "") +- if(HAVE_GSSAPI_GSSAPI_H) +- list(APPEND _INCLUDE_LIST "gssapi/gssapi.h") +- endif() +- if(HAVE_GSSAPI_GSSAPI_GENERIC_H) +- list(APPEND _INCLUDE_LIST "gssapi/gssapi_generic.h") +- endif() +- if(HAVE_GSSAPI_GSSAPI_KRB5_H) +- list(APPEND _INCLUDE_LIST "gssapi/gssapi_krb5.h") +- endif() +- +- string(REPLACE ";" " " _COMPILER_FLAGS_STR "${GSS_COMPILER_FLAGS}") +- string(REPLACE ";" " " _LINKER_FLAGS_STR "${GSS_LINKER_FLAGS}") +- +- foreach(_dir ${GSS_LINK_DIRECTORIES}) +- set(_LINKER_FLAGS_STR "${_LINKER_FLAGS_STR} -L\"${_dir}\"") +- endforeach() +- +- set(CMAKE_REQUIRED_FLAGS "${_COMPILER_FLAGS_STR} ${_LINKER_FLAGS_STR}") +- set(CMAKE_REQUIRED_LIBRARIES ${GSS_LIBRARIES}) +- check_symbol_exists("GSS_C_NT_HOSTBASED_SERVICE" ${_INCLUDE_LIST} HAVE_GSS_C_NT_HOSTBASED_SERVICE) +- if(NOT HAVE_GSS_C_NT_HOSTBASED_SERVICE) +- set(HAVE_OLD_GSSMIT ON) +- endif() +- unset(CMAKE_REQUIRED_LIBRARIES) +- +- endif() +- +- include_directories(${GSS_INCLUDE_DIR}) +- link_directories(${GSS_LINK_DIRECTORIES}) +- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${GSS_COMPILER_FLAGS}") +- set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${GSS_LINKER_FLAGS}") +- set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${GSS_LINKER_FLAGS}") +- list(APPEND CURL_LIBS ${GSS_LIBRARIES}) +- +- else() +- message(WARNING "GSSAPI support has been requested but no supporting libraries found. Skipping.") +- endif() ++ list(APPEND CURL_LIBS ${ZLIB_LIBRARIES}) + endif() + + option(ENABLE_UNIX_SOCKETS "Define if you want Unix domain sockets support" ON) + if(ENABLE_UNIX_SOCKETS) +- include(CheckStructHasMember) +- check_struct_has_member("struct sockaddr_un" sun_path "sys/un.h" USE_UNIX_SOCKETS) ++ include(CheckStructHasMember) ++ check_struct_has_member("struct sockaddr_un" sun_path "sys/un.h" USE_UNIX_SOCKETS) + else() +- unset(USE_UNIX_SOCKETS CACHE) ++ unset(USE_UNIX_SOCKETS CACHE) + endif() + +-# + # CA handling +-# +-set(CURL_CA_BUNDLE "auto" CACHE STRING +- "Path to the CA bundle. Set 'none' to disable or 'auto' for auto-detection. Defaults to 'auto'.") +-set(CURL_CA_FALLBACK OFF CACHE BOOL +- "Set ON to use built-in CA store of TLS backend. Defaults to OFF") +-set(CURL_CA_PATH "auto" CACHE STRING +- "Location of default CA path. Set 'none' to disable or 'auto' for auto-detection. Defaults to 'auto'.") +- +-if("${CURL_CA_BUNDLE}" STREQUAL "") +- message(FATAL_ERROR "Invalid value of CURL_CA_BUNDLE. Use 'none', 'auto' or file path.") +-elseif("${CURL_CA_BUNDLE}" STREQUAL "none") +- unset(CURL_CA_BUNDLE CACHE) +-elseif("${CURL_CA_BUNDLE}" STREQUAL "auto") +- unset(CURL_CA_BUNDLE CACHE) +- set(CURL_CA_BUNDLE_AUTODETECT TRUE) +-else() +- set(CURL_CA_BUNDLE_SET TRUE) +-endif() +- +-if("${CURL_CA_PATH}" STREQUAL "") +- message(FATAL_ERROR "Invalid value of CURL_CA_PATH. Use 'none', 'auto' or directory path.") +-elseif("${CURL_CA_PATH}" STREQUAL "none") +- unset(CURL_CA_PATH CACHE) +-elseif("${CURL_CA_PATH}" STREQUAL "auto") +- unset(CURL_CA_PATH CACHE) +- set(CURL_CA_PATH_AUTODETECT TRUE) +-else() +- set(CURL_CA_PATH_SET TRUE) +-endif() +- +-if(CURL_CA_BUNDLE_SET AND CURL_CA_PATH_AUTODETECT) +- # Skip autodetection of unset CA path because CA bundle is set explicitly +-elseif(CURL_CA_PATH_SET AND CURL_CA_BUNDLE_AUTODETECT) +- # Skip autodetection of unset CA bundle because CA path is set explicitly +-elseif(CURL_CA_PATH_AUTODETECT OR CURL_CA_BUNDLE_AUTODETECT) +- # first try autodetecting a CA bundle, then a CA path +- +- if(CURL_CA_BUNDLE_AUTODETECT) +- set(SEARCH_CA_BUNDLE_PATHS +- /etc/ssl/certs/ca-certificates.crt +- /etc/pki/tls/certs/ca-bundle.crt +- /usr/share/ssl/certs/ca-bundle.crt +- /usr/local/share/certs/ca-root-nss.crt +- /etc/ssl/cert.pem) +- +- foreach(SEARCH_CA_BUNDLE_PATH ${SEARCH_CA_BUNDLE_PATHS}) +- if(EXISTS "${SEARCH_CA_BUNDLE_PATH}") +- message(STATUS "Found CA bundle: ${SEARCH_CA_BUNDLE_PATH}") +- set(CURL_CA_BUNDLE "${SEARCH_CA_BUNDLE_PATH}") +- set(CURL_CA_BUNDLE_SET TRUE CACHE BOOL "Path to the CA bundle has been set") +- break() +- endif() +- endforeach() +- endif() +- +- if(CURL_CA_PATH_AUTODETECT AND (NOT CURL_CA_PATH_SET)) +- if(EXISTS "/etc/ssl/certs") +- set(CURL_CA_PATH "/etc/ssl/certs") +- set(CURL_CA_PATH_SET TRUE CACHE BOOL "Path to the CA bundle has been set") +- endif() +- endif() +-endif() +- +-if(CURL_CA_PATH_SET AND NOT USE_OPENSSL AND NOT USE_MBEDTLS) +- message(FATAL_ERROR +- "CA path only supported by OpenSSL, GnuTLS or mbed TLS. " +- "Set CURL_CA_PATH=none or enable one of those TLS backends.") +-endif() +- +-# Check for header files +-if(NOT UNIX) +- check_include_file_concat("windows.h" HAVE_WINDOWS_H) +- check_include_file_concat("winsock.h" HAVE_WINSOCK_H) +- check_include_file_concat("ws2tcpip.h" HAVE_WS2TCPIP_H) +- check_include_file_concat("winsock2.h" HAVE_WINSOCK2_H) +- if(NOT CURL_WINDOWS_SSPI AND USE_OPENSSL) +- set(CURL_LIBS ${CURL_LIBS} "crypt32") +- endif() ++# Explicitly set to most common case ++if (OPENSSL_FOUND) ++ set(CURL_CA_BUNDLE "/etc/ssl/certs/ca-certificates.crt") ++ set(CURL_CA_BUNDLE_SET TRUE CACHE BOOL "Path to the CA bundle has been set") ++ set(CURL_CA_PATH "/etc/ssl/certs") ++ set(CURL_CA_PATH_SET TRUE CACHE BOOL "Path to the CA bundle has been set") + endif() + + check_include_file_concat("stdio.h" HAVE_STDIO_H) +@@ -810,18 +269,18 @@ + check_type_size("long double" SIZEOF_LONG_DOUBLE) + check_type_size("time_t" SIZEOF_TIME_T) + if(NOT HAVE_SIZEOF_SSIZE_T) +- if(SIZEOF_LONG EQUAL SIZEOF_SIZE_T) +- set(ssize_t long) +- endif() +- if(NOT ssize_t AND SIZEOF___INT64 EQUAL SIZEOF_SIZE_T) +- set(ssize_t __int64) +- endif() ++ if(SIZEOF_LONG EQUAL SIZEOF_SIZE_T) ++ set(ssize_t long) ++ endif() ++ if(NOT ssize_t AND SIZEOF___INT64 EQUAL SIZEOF_SIZE_T) ++ set(ssize_t __int64) ++ endif() + endif() + # off_t is sized later, after the HAVE_FILE_OFFSET_BITS test + + if(HAVE_SIZEOF_LONG_LONG) +- set(HAVE_LONGLONG 1) +- set(HAVE_LL 1) ++ set(HAVE_LONGLONG 1) ++ set(HAVE_LL 1) + endif() + + find_file(RANDOM_FILE urandom /dev) +@@ -829,9 +288,9 @@ + + # Check for some functions that are used + if(HAVE_LIBWS2_32) +- set(CMAKE_REQUIRED_LIBRARIES ws2_32) ++ set(CMAKE_REQUIRED_LIBRARIES ws2_32) + elseif(HAVE_LIBSOCKET) +- set(CMAKE_REQUIRED_LIBRARIES socket) ++ set(CMAKE_REQUIRED_LIBRARIES socket) + endif() + + check_symbol_exists(basename "${CURL_INCLUDES}" HAVE_BASENAME) +@@ -849,7 +308,7 @@ + check_symbol_exists(strncmpi "${CURL_INCLUDES}" HAVE_STRNCMPI) + check_symbol_exists(alarm "${CURL_INCLUDES}" HAVE_ALARM) + if(NOT HAVE_STRNCMPI) +- set(HAVE_STRCMPI) ++ set(HAVE_STRCMPI) + endif() + check_symbol_exists(gethostbyaddr "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR) + check_symbol_exists(gethostbyaddr_r "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR_R) +@@ -879,7 +338,7 @@ + check_symbol_exists(signal "${CURL_INCLUDES}" HAVE_SIGNAL_FUNC) + check_symbol_exists(SIGALRM "${CURL_INCLUDES}" HAVE_SIGNAL_MACRO) + if(HAVE_SIGNAL_FUNC AND HAVE_SIGNAL_MACRO) +- set(HAVE_SIGNAL 1) ++ set(HAVE_SIGNAL 1) + endif() + check_symbol_exists(uname "${CURL_INCLUDES}" HAVE_UNAME) + check_symbol_exists(strtoll "${CURL_INCLUDES}" HAVE_STRTOLL) +@@ -906,89 +365,75 @@ + check_symbol_exists(setsockopt "${CURL_INCLUDES}" HAVE_SETSOCKOPT) + check_function_exists(mach_absolute_time HAVE_MACH_ABSOLUTE_TIME) + +-# symbol exists in win32, but function does not. +-if(WIN32) +- if(ENABLE_INET_PTON) +- check_function_exists(inet_pton HAVE_INET_PTON) +- # _WIN32_WINNT_VISTA (0x0600) +- add_definitions(-D_WIN32_WINNT=0x0600) +- else() +- # _WIN32_WINNT_WINXP (0x0501) +- add_definitions(-D_WIN32_WINNT=0x0501) +- endif() +-else() +- check_function_exists(inet_pton HAVE_INET_PTON) +-endif() +- + check_symbol_exists(fsetxattr "${CURL_INCLUDES}" HAVE_FSETXATTR) + if(HAVE_FSETXATTR) +- foreach(CURL_TEST HAVE_FSETXATTR_5 HAVE_FSETXATTR_6) +- curl_internal_test(${CURL_TEST}) +- endforeach() ++ foreach(CURL_TEST HAVE_FSETXATTR_5 HAVE_FSETXATTR_6) ++ curl_internal_test(${CURL_TEST}) ++ endforeach() + endif() + + # sigaction and sigsetjmp are special. Use special mechanism for + # detecting those, but only if previous attempt failed. + if(HAVE_SIGNAL_H) +- check_symbol_exists(sigaction "signal.h" HAVE_SIGACTION) ++ check_symbol_exists(sigaction "signal.h" HAVE_SIGACTION) + endif() + + if(NOT HAVE_SIGSETJMP) +- if(HAVE_SETJMP_H) +- check_symbol_exists(sigsetjmp "setjmp.h" HAVE_MACRO_SIGSETJMP) +- if(HAVE_MACRO_SIGSETJMP) +- set(HAVE_SIGSETJMP 1) ++ if(HAVE_SETJMP_H) ++ check_symbol_exists(sigsetjmp "setjmp.h" HAVE_MACRO_SIGSETJMP) ++ if(HAVE_MACRO_SIGSETJMP) ++ set(HAVE_SIGSETJMP 1) ++ endif() + endif() +- endif() + endif() + + # If there is no stricmp(), do not allow LDAP to parse URLs + if(NOT HAVE_STRICMP) +- set(HAVE_LDAP_URL_PARSE 1) ++ set(HAVE_LDAP_URL_PARSE 1) + endif() + + # Do curl specific tests + foreach(CURL_TEST +- HAVE_FCNTL_O_NONBLOCK +- HAVE_IOCTLSOCKET +- HAVE_IOCTLSOCKET_CAMEL +- HAVE_IOCTLSOCKET_CAMEL_FIONBIO +- HAVE_IOCTLSOCKET_FIONBIO +- HAVE_IOCTL_FIONBIO +- HAVE_IOCTL_SIOCGIFADDR +- HAVE_SETSOCKOPT_SO_NONBLOCK +- HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID +- TIME_WITH_SYS_TIME +- HAVE_O_NONBLOCK +- HAVE_GETHOSTBYADDR_R_5 +- HAVE_GETHOSTBYADDR_R_7 +- HAVE_GETHOSTBYADDR_R_8 +- HAVE_GETHOSTBYADDR_R_5_REENTRANT +- HAVE_GETHOSTBYADDR_R_7_REENTRANT +- HAVE_GETHOSTBYADDR_R_8_REENTRANT +- HAVE_GETHOSTBYNAME_R_3 +- HAVE_GETHOSTBYNAME_R_5 +- HAVE_GETHOSTBYNAME_R_6 +- HAVE_GETHOSTBYNAME_R_3_REENTRANT +- HAVE_GETHOSTBYNAME_R_5_REENTRANT +- HAVE_GETHOSTBYNAME_R_6_REENTRANT +- HAVE_IN_ADDR_T +- HAVE_BOOL_T +- STDC_HEADERS +- RETSIGTYPE_TEST +- HAVE_INET_NTOA_R_DECL +- HAVE_INET_NTOA_R_DECL_REENTRANT +- HAVE_GETADDRINFO +- HAVE_FILE_OFFSET_BITS +- HAVE_VARIADIC_MACROS_C99 +- HAVE_VARIADIC_MACROS_GCC +- ) +- curl_internal_test(${CURL_TEST}) ++ HAVE_FCNTL_O_NONBLOCK ++ HAVE_IOCTLSOCKET ++ HAVE_IOCTLSOCKET_CAMEL ++ HAVE_IOCTLSOCKET_CAMEL_FIONBIO ++ HAVE_IOCTLSOCKET_FIONBIO ++ HAVE_IOCTL_FIONBIO ++ HAVE_IOCTL_SIOCGIFADDR ++ HAVE_SETSOCKOPT_SO_NONBLOCK ++ HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID ++ TIME_WITH_SYS_TIME ++ HAVE_O_NONBLOCK ++ HAVE_GETHOSTBYADDR_R_5 ++ HAVE_GETHOSTBYADDR_R_7 ++ HAVE_GETHOSTBYADDR_R_8 ++ HAVE_GETHOSTBYADDR_R_5_REENTRANT ++ HAVE_GETHOSTBYADDR_R_7_REENTRANT ++ HAVE_GETHOSTBYADDR_R_8_REENTRANT ++ HAVE_GETHOSTBYNAME_R_3 ++ HAVE_GETHOSTBYNAME_R_5 ++ HAVE_GETHOSTBYNAME_R_6 ++ HAVE_GETHOSTBYNAME_R_3_REENTRANT ++ HAVE_GETHOSTBYNAME_R_5_REENTRANT ++ HAVE_GETHOSTBYNAME_R_6_REENTRANT ++ HAVE_IN_ADDR_T ++ HAVE_BOOL_T ++ STDC_HEADERS ++ RETSIGTYPE_TEST ++ HAVE_INET_NTOA_R_DECL ++ HAVE_INET_NTOA_R_DECL_REENTRANT ++ HAVE_GETADDRINFO ++ HAVE_FILE_OFFSET_BITS ++ HAVE_VARIADIC_MACROS_C99 ++ HAVE_VARIADIC_MACROS_GCC ++ ) ++ curl_internal_test(${CURL_TEST}) + endforeach() + + if(HAVE_FILE_OFFSET_BITS) +- set(_FILE_OFFSET_BITS 64) +- set(CMAKE_REQUIRED_FLAGS "-D_FILE_OFFSET_BITS=64") ++ set(_FILE_OFFSET_BITS 64) ++ set(CMAKE_REQUIRED_FLAGS "-D_FILE_OFFSET_BITS=64") + endif() + check_type_size("off_t" SIZEOF_OFF_T) + +@@ -998,49 +443,47 @@ + check_type_size("curl_off_t" SIZEOF_CURL_OFF_T) + set(CMAKE_EXTRA_INCLUDE_FILES "") + +-set(CMAKE_REQUIRED_FLAGS) +- + foreach(CURL_TEST +- HAVE_GLIBC_STRERROR_R +- HAVE_POSIX_STRERROR_R +- ) +- curl_internal_test(${CURL_TEST}) ++ HAVE_GLIBC_STRERROR_R ++ HAVE_POSIX_STRERROR_R ++ ) ++ curl_internal_test(${CURL_TEST}) + endforeach() + + # Check for reentrant + foreach(CURL_TEST +- HAVE_GETHOSTBYADDR_R_5 +- HAVE_GETHOSTBYADDR_R_7 +- HAVE_GETHOSTBYADDR_R_8 +- HAVE_GETHOSTBYNAME_R_3 +- HAVE_GETHOSTBYNAME_R_5 +- HAVE_GETHOSTBYNAME_R_6 +- HAVE_INET_NTOA_R_DECL_REENTRANT) +- if(NOT ${CURL_TEST}) +- if(${CURL_TEST}_REENTRANT) +- set(NEED_REENTRANT 1) ++ HAVE_GETHOSTBYADDR_R_5 ++ HAVE_GETHOSTBYADDR_R_7 ++ HAVE_GETHOSTBYADDR_R_8 ++ HAVE_GETHOSTBYNAME_R_3 ++ HAVE_GETHOSTBYNAME_R_5 ++ HAVE_GETHOSTBYNAME_R_6 ++ HAVE_INET_NTOA_R_DECL_REENTRANT) ++ if(NOT ${CURL_TEST}) ++ if(${CURL_TEST}_REENTRANT) ++ set(NEED_REENTRANT 1) ++ endif() + endif() +- endif() + endforeach() + + if(NEED_REENTRANT) +- foreach(CURL_TEST +- HAVE_GETHOSTBYADDR_R_5 +- HAVE_GETHOSTBYADDR_R_7 +- HAVE_GETHOSTBYADDR_R_8 +- HAVE_GETHOSTBYNAME_R_3 +- HAVE_GETHOSTBYNAME_R_5 +- HAVE_GETHOSTBYNAME_R_6) +- set(${CURL_TEST} 0) +- if(${CURL_TEST}_REENTRANT) +- set(${CURL_TEST} 1) +- endif() +- endforeach() ++ foreach(CURL_TEST ++ HAVE_GETHOSTBYADDR_R_5 ++ HAVE_GETHOSTBYADDR_R_7 ++ HAVE_GETHOSTBYADDR_R_8 ++ HAVE_GETHOSTBYNAME_R_3 ++ HAVE_GETHOSTBYNAME_R_5 ++ HAVE_GETHOSTBYNAME_R_6) ++ set(${CURL_TEST} 0) ++ if(${CURL_TEST}_REENTRANT) ++ set(${CURL_TEST} 1) ++ endif() ++ endforeach() + endif() + + if(HAVE_INET_NTOA_R_DECL_REENTRANT) +- set(HAVE_INET_NTOA_R_DECL 1) +- set(NEED_REENTRANT 1) ++ set(HAVE_INET_NTOA_R_DECL 1) ++ set(NEED_REENTRANT 1) + endif() + + # Check clock_gettime(CLOCK_MONOTONIC, x) support +@@ -1052,323 +495,132 @@ + # Some other minor tests + + if(NOT HAVE_IN_ADDR_T) +- set(in_addr_t "unsigned long") +-endif() +- +-# Fix libz / zlib.h +- +-if(NOT CURL_SPECIAL_LIBZ) +- if(NOT HAVE_LIBZ) +- set(HAVE_ZLIB_H 0) +- endif() +- +- if(NOT HAVE_ZLIB_H) +- set(HAVE_LIBZ 0) +- endif() ++ set(in_addr_t "unsigned long") + endif() + + # Check for nonblocking + set(HAVE_DISABLED_NONBLOCKING 1) + if(HAVE_FIONBIO OR +- HAVE_IOCTLSOCKET OR +- HAVE_IOCTLSOCKET_CASE OR +- HAVE_O_NONBLOCK) +- set(HAVE_DISABLED_NONBLOCKING) +-endif() +- +-if(RETSIGTYPE_TEST) +- set(RETSIGTYPE void) +-else() +- set(RETSIGTYPE int) +-endif() +- +-if(CMAKE_COMPILER_IS_GNUCC AND APPLE) +- include(CheckCCompilerFlag) +- check_c_compiler_flag(-Wno-long-double HAVE_C_FLAG_Wno_long_double) +- if(HAVE_C_FLAG_Wno_long_double) +- # The Mac version of GCC warns about use of long double. Disable it. +- get_source_file_property(MPRINTF_COMPILE_FLAGS mprintf.c COMPILE_FLAGS) +- if(MPRINTF_COMPILE_FLAGS) +- set(MPRINTF_COMPILE_FLAGS "${MPRINTF_COMPILE_FLAGS} -Wno-long-double") +- else() +- set(MPRINTF_COMPILE_FLAGS "-Wno-long-double") +- endif() +- set_source_files_properties(mprintf.c PROPERTIES +- COMPILE_FLAGS ${MPRINTF_COMPILE_FLAGS}) +- endif() ++ HAVE_IOCTLSOCKET OR ++ HAVE_IOCTLSOCKET_CASE OR ++ HAVE_O_NONBLOCK) ++ set(HAVE_DISABLED_NONBLOCKING) + endif() + +-# TODO test which of these headers are required +-if(WIN32) +- set(CURL_PULL_WS2TCPIP_H ${HAVE_WS2TCPIP_H}) +-else() +- set(CURL_PULL_SYS_TYPES_H ${HAVE_SYS_TYPES_H}) +- set(CURL_PULL_SYS_SOCKET_H ${HAVE_SYS_SOCKET_H}) +- set(CURL_PULL_SYS_POLL_H ${HAVE_SYS_POLL_H}) +-endif() ++set(CURL_PULL_SYS_TYPES_H ${HAVE_SYS_TYPES_H}) ++set(CURL_PULL_SYS_SOCKET_H ${HAVE_SYS_SOCKET_H}) ++set(CURL_PULL_SYS_POLL_H ${HAVE_SYS_POLL_H}) + set(CURL_PULL_STDINT_H ${HAVE_STDINT_H}) + set(CURL_PULL_INTTYPES_H ${HAVE_INTTYPES_H}) + + include(CMake/OtherTests.cmake) + +-add_definitions(-DHAVE_CONFIG_H) ++SET(LIB_VAUTH_CFILES ++ "${CURL_LIBRARY_DIR}/vauth/vauth.c" "${CURL_LIBRARY_DIR}/vauth/cleartext.c" "${CURL_LIBRARY_DIR}/vauth/cram.c" ++ "${CURL_LIBRARY_DIR}/vauth/digest.c" "${CURL_LIBRARY_DIR}/vauth/digest_sspi.c" "${CURL_LIBRARY_DIR}/vauth/krb5_gssapi.c" ++ "${CURL_LIBRARY_DIR}/vauth/krb5_sspi.c" "${CURL_LIBRARY_DIR}/vauth/ntlm.c" "${CURL_LIBRARY_DIR}/vauth/ntlm_sspi.c" "${CURL_LIBRARY_DIR}/vauth/oauth2.c" ++ "${CURL_LIBRARY_DIR}/vauth/spnego_gssapi.c" "${CURL_LIBRARY_DIR}/vauth/spnego_sspi.c") ++ ++SET(LIB_VAUTH_HFILES "${CURL_LIBRARY_DIR}/vauth/vauth.h" "${CURL_LIBRARY_DIR}/vauth/digest.h" "${CURL_LIBRARY_DIR}/vauth/ntlm.h") ++ ++SET(LIB_VTLS_CFILES "${CURL_LIBRARY_DIR}/vtls/openssl.c" "${CURL_LIBRARY_DIR}/vtls/gtls.c" "${CURL_LIBRARY_DIR}/vtls/vtls.c" "${CURL_LIBRARY_DIR}/vtls/nss.c" ++ "${CURL_LIBRARY_DIR}/vtls/polarssl.c" "${CURL_LIBRARY_DIR}/vtls/polarssl_threadlock.c" ++ "${CURL_LIBRARY_DIR}/vtls/wolfssl.c" "${CURL_LIBRARY_DIR}/vtls/schannel.c" "${CURL_LIBRARY_DIR}/vtls/schannel_verify.c" ++ "${CURL_LIBRARY_DIR}/vtls/sectransp.c" "${CURL_LIBRARY_DIR}/vtls/gskit.c" "${CURL_LIBRARY_DIR}/vtls/mbedtls.c" "${CURL_LIBRARY_DIR}/vtls/mesalink.c" ++ "${CURL_LIBRARY_DIR}/vtls/bearssl.c") ++ ++SET(LIB_VTLS_HFILES "${CURL_LIBRARY_DIR}/vtls/openssl.h" "${CURL_LIBRARY_DIR}/vtls/vtls.h" "${CURL_LIBRARY_DIR}/vtls/gtls.h" ++ "${CURL_LIBRARY_DIR}/vtls/nssg.h" "${CURL_LIBRARY_DIR}/vtls/polarssl.h" "${CURL_LIBRARY_DIR}/vtls/polarssl_threadlock.h" ++ "${CURL_LIBRARY_DIR}/vtls/wolfssl.h" "${CURL_LIBRARY_DIR}/vtls/schannel.h" "${CURL_LIBRARY_DIR}/vtls/sectransp.h" "${CURL_LIBRARY_DIR}/vtls/gskit.h" ++ "${CURL_LIBRARY_DIR}/vtls/mbedtls.h" "${CURL_LIBRARY_DIR}/vtls/mesalink.h" "${CURL_LIBRARY_DIR}/vtls/bearssl.h") ++ ++SET(LIB_VQUIC_CFILES "${CURL_LIBRARY_DIR}/vquic/ngtcp2.c" "${CURL_LIBRARY_DIR}/vquic/quiche.c") ++ ++SET(LIB_VQUIC_HFILES "${CURL_LIBRARY_DIR}/vquic/ngtcp2.h" "${CURL_LIBRARY_DIR}/vquic/quiche.h") ++ ++SET(LIB_VSSH_CFILES "${CURL_LIBRARY_DIR}/vssh/libssh2.c" "${CURL_LIBRARY_DIR}/vssh/libssh.c") ++ ++SET(LIB_VSSH_HFILES "${CURL_LIBRARY_DIR}/vssh/ssh.h") ++ ++SET(LIB_CFILES "${CURL_LIBRARY_DIR}/file.c" ++ "${CURL_LIBRARY_DIR}/timeval.c" "${CURL_LIBRARY_DIR}/base64.c" "${CURL_LIBRARY_DIR}/hostip.c" "${CURL_LIBRARY_DIR}/progress.c" "${CURL_LIBRARY_DIR}/formdata.c" ++ "${CURL_LIBRARY_DIR}/cookie.c" "${CURL_LIBRARY_DIR}/http.c" "${CURL_LIBRARY_DIR}/sendf.c" "${CURL_LIBRARY_DIR}/url.c" "${CURL_LIBRARY_DIR}/dict.c" "${CURL_LIBRARY_DIR}/if2ip.c" "${CURL_LIBRARY_DIR}/speedcheck.c" ++ "${CURL_LIBRARY_DIR}/ldap.c" "${CURL_LIBRARY_DIR}/version.c" "${CURL_LIBRARY_DIR}/getenv.c" "${CURL_LIBRARY_DIR}/escape.c" "${CURL_LIBRARY_DIR}/mprintf.c" "${CURL_LIBRARY_DIR}/telnet.c" "${CURL_LIBRARY_DIR}/netrc.c" ++ "${CURL_LIBRARY_DIR}/getinfo.c" "${CURL_LIBRARY_DIR}/transfer.c" "${CURL_LIBRARY_DIR}/strcase.c" "${CURL_LIBRARY_DIR}/easy.c" "${CURL_LIBRARY_DIR}/security.c" "${CURL_LIBRARY_DIR}/curl_fnmatch.c" ++ "${CURL_LIBRARY_DIR}/fileinfo.c" "${CURL_LIBRARY_DIR}/wildcard.c" "${CURL_LIBRARY_DIR}/krb5.c" "${CURL_LIBRARY_DIR}/memdebug.c" "${CURL_LIBRARY_DIR}/http_chunks.c" ++ "${CURL_LIBRARY_DIR}/strtok.c" "${CURL_LIBRARY_DIR}/connect.c" "${CURL_LIBRARY_DIR}/llist.c" "${CURL_LIBRARY_DIR}/hash.c" "${CURL_LIBRARY_DIR}/multi.c" "${CURL_LIBRARY_DIR}/content_encoding.c" "${CURL_LIBRARY_DIR}/share.c" ++ "${CURL_LIBRARY_DIR}/http_digest.c" "${CURL_LIBRARY_DIR}/md4.c" "${CURL_LIBRARY_DIR}/md5.c" "${CURL_LIBRARY_DIR}/http_negotiate.c" "${CURL_LIBRARY_DIR}/inet_pton.c" "${CURL_LIBRARY_DIR}/strtoofft.c" ++ "${CURL_LIBRARY_DIR}/strerror.c" "${CURL_LIBRARY_DIR}/amigaos.c" "${CURL_LIBRARY_DIR}/hostasyn.c" "${CURL_LIBRARY_DIR}/hostip4.c" "${CURL_LIBRARY_DIR}/hostip6.c" "${CURL_LIBRARY_DIR}/hostsyn.c" ++ "${CURL_LIBRARY_DIR}/inet_ntop.c" "${CURL_LIBRARY_DIR}/parsedate.c" "${CURL_LIBRARY_DIR}/select.c" "${CURL_LIBRARY_DIR}/splay.c" "${CURL_LIBRARY_DIR}/strdup.c" "${CURL_LIBRARY_DIR}/socks.c" ++ "${CURL_LIBRARY_DIR}/curl_addrinfo.c" "${CURL_LIBRARY_DIR}/socks_gssapi.c" "${CURL_LIBRARY_DIR}/socks_sspi.c" ++ "${CURL_LIBRARY_DIR}/curl_sspi.c" "${CURL_LIBRARY_DIR}/slist.c" "${CURL_LIBRARY_DIR}/nonblock.c" "${CURL_LIBRARY_DIR}/curl_memrchr.c" "${CURL_LIBRARY_DIR}/imap.c" "${CURL_LIBRARY_DIR}/pop3.c" "${CURL_LIBRARY_DIR}/smtp.c" ++ "${CURL_LIBRARY_DIR}/pingpong.c" "${CURL_LIBRARY_DIR}/rtsp.c" "${CURL_LIBRARY_DIR}/curl_threads.c" "${CURL_LIBRARY_DIR}/warnless.c" "${CURL_LIBRARY_DIR}/hmac.c" "${CURL_LIBRARY_DIR}/curl_rtmp.c" ++ "${CURL_LIBRARY_DIR}/openldap.c" "${CURL_LIBRARY_DIR}/curl_gethostname.c" "${CURL_LIBRARY_DIR}/gopher.c" "${CURL_LIBRARY_DIR}/idn_win32.c" ++ "${CURL_LIBRARY_DIR}/http_proxy.c" "${CURL_LIBRARY_DIR}/non-ascii.c" "${CURL_LIBRARY_DIR}/asyn-ares.c" "${CURL_LIBRARY_DIR}/asyn-thread.c" "${CURL_LIBRARY_DIR}/curl_gssapi.c" ++ "${CURL_LIBRARY_DIR}/http_ntlm.c" "${CURL_LIBRARY_DIR}/curl_ntlm_wb.c" "${CURL_LIBRARY_DIR}/curl_ntlm_core.c" "${CURL_LIBRARY_DIR}/curl_sasl.c" "${CURL_LIBRARY_DIR}/rand.c" ++ "${CURL_LIBRARY_DIR}/curl_multibyte.c" "${CURL_LIBRARY_DIR}/hostcheck.c" "${CURL_LIBRARY_DIR}/conncache.c" "${CURL_LIBRARY_DIR}/dotdot.c" ++ "${CURL_LIBRARY_DIR}/x509asn1.c" "${CURL_LIBRARY_DIR}/http2.c" "${CURL_LIBRARY_DIR}/smb.c" "${CURL_LIBRARY_DIR}/curl_endian.c" "${CURL_LIBRARY_DIR}/curl_des.c" "${CURL_LIBRARY_DIR}/system_win32.c" ++ "${CURL_LIBRARY_DIR}/mime.c" "${CURL_LIBRARY_DIR}/sha256.c" "${CURL_LIBRARY_DIR}/setopt.c" "${CURL_LIBRARY_DIR}/curl_path.c" "${CURL_LIBRARY_DIR}/curl_ctype.c" "${CURL_LIBRARY_DIR}/curl_range.c" "${CURL_LIBRARY_DIR}/psl.c" ++ "${CURL_LIBRARY_DIR}/doh.c" "${CURL_LIBRARY_DIR}/urlapi.c" "${CURL_LIBRARY_DIR}/curl_get_line.c" "${CURL_LIBRARY_DIR}/altsvc.c" "${CURL_LIBRARY_DIR}/socketpair.c") ++ ++SET(LIB_HFILES "${CURL_LIBRARY_DIR}/arpa_telnet.h" "${CURL_LIBRARY_DIR}/netrc.h" "${CURL_LIBRARY_DIR}/file.h" "${CURL_LIBRARY_DIR}/timeval.h" "${CURL_LIBRARY_DIR}/hostip.h" "${CURL_LIBRARY_DIR}/progress.h" ++ "${CURL_LIBRARY_DIR}/formdata.h" "${CURL_LIBRARY_DIR}/cookie.h" "${CURL_LIBRARY_DIR}/http.h" "${CURL_LIBRARY_DIR}/sendf.h" "${CURL_LIBRARY_DIR}/url.h" "${CURL_LIBRARY_DIR}/dict.h" "${CURL_LIBRARY_DIR}/if2ip.h" ++ "${CURL_LIBRARY_DIR}/speedcheck.h" "${CURL_LIBRARY_DIR}/urldata.h" "${CURL_LIBRARY_DIR}/curl_ldap.h" "${CURL_LIBRARY_DIR}/escape.h" "${CURL_LIBRARY_DIR}/telnet.h" "${CURL_LIBRARY_DIR}/getinfo.h" ++ "${CURL_LIBRARY_DIR}/strcase.h" "${CURL_LIBRARY_DIR}/curl_sec.h" "${CURL_LIBRARY_DIR}/memdebug.h" "${CURL_LIBRARY_DIR}/http_chunks.h" "${CURL_LIBRARY_DIR}/curl_fnmatch.h" ++ "${CURL_LIBRARY_DIR}/wildcard.h" "${CURL_LIBRARY_DIR}/fileinfo.h" "${CURL_LIBRARY_DIR}/strtok.h" "${CURL_LIBRARY_DIR}/connect.h" "${CURL_LIBRARY_DIR}/llist.h" ++ "${CURL_LIBRARY_DIR}/hash.h" "${CURL_LIBRARY_DIR}/content_encoding.h" "${CURL_LIBRARY_DIR}/share.h" "${CURL_LIBRARY_DIR}/curl_md4.h" "${CURL_LIBRARY_DIR}/curl_md5.h" "${CURL_LIBRARY_DIR}/http_digest.h" ++ "${CURL_LIBRARY_DIR}/http_negotiate.h" "${CURL_LIBRARY_DIR}/inet_pton.h" "${CURL_LIBRARY_DIR}/amigaos.h" "${CURL_LIBRARY_DIR}/strtoofft.h" "${CURL_LIBRARY_DIR}/strerror.h" ++ "${CURL_LIBRARY_DIR}/inet_ntop.h" "${CURL_LIBRARY_DIR}/curlx.h" "${CURL_LIBRARY_DIR}/curl_memory.h" "${CURL_LIBRARY_DIR}/curl_setup.h" "${CURL_LIBRARY_DIR}/transfer.h" "${CURL_LIBRARY_DIR}/select.h" ++ "${CURL_LIBRARY_DIR}/easyif.h" "${CURL_LIBRARY_DIR}/multiif.h" "${CURL_LIBRARY_DIR}/parsedate.h" "${CURL_LIBRARY_DIR}/sockaddr.h" "${CURL_LIBRARY_DIR}/splay.h" "${CURL_LIBRARY_DIR}/strdup.h" ++ "${CURL_LIBRARY_DIR}/socks.h" "${CURL_LIBRARY_DIR}/curl_base64.h" "${CURL_LIBRARY_DIR}/curl_addrinfo.h" "${CURL_LIBRARY_DIR}/curl_sspi.h" ++ "${CURL_LIBRARY_DIR}/slist.h" "${CURL_LIBRARY_DIR}/nonblock.h" "${CURL_LIBRARY_DIR}/curl_memrchr.h" "${CURL_LIBRARY_DIR}/imap.h" "${CURL_LIBRARY_DIR}/pop3.h" "${CURL_LIBRARY_DIR}/smtp.h" "${CURL_LIBRARY_DIR}/pingpong.h" ++ "${CURL_LIBRARY_DIR}/rtsp.h" "${CURL_LIBRARY_DIR}/curl_threads.h" "${CURL_LIBRARY_DIR}/warnless.h" "${CURL_LIBRARY_DIR}/curl_hmac.h" "${CURL_LIBRARY_DIR}/curl_rtmp.h" ++ "${CURL_LIBRARY_DIR}/curl_gethostname.h" "${CURL_LIBRARY_DIR}/gopher.h" "${CURL_LIBRARY_DIR}/http_proxy.h" "${CURL_LIBRARY_DIR}/non-ascii.h" "${CURL_LIBRARY_DIR}/asyn.h" ++ "${CURL_LIBRARY_DIR}/http_ntlm.h" "${CURL_LIBRARY_DIR}/curl_gssapi.h" "${CURL_LIBRARY_DIR}/curl_ntlm_wb.h" "${CURL_LIBRARY_DIR}/curl_ntlm_core.h" ++ "${CURL_LIBRARY_DIR}/curl_sasl.h" "${CURL_LIBRARY_DIR}/curl_multibyte.h" "${CURL_LIBRARY_DIR}/hostcheck.h" "${CURL_LIBRARY_DIR}/conncache.h" ++ "${CURL_LIBRARY_DIR}/multihandle.h" "${CURL_LIBRARY_DIR}/setup-vms.h" "${CURL_LIBRARY_DIR}/dotdot.h" ++ "${CURL_LIBRARY_DIR}/x509asn1.h" "${CURL_LIBRARY_DIR}/http2.h" "${CURL_LIBRARY_DIR}/sigpipe.h" "${CURL_LIBRARY_DIR}/smb.h" "${CURL_LIBRARY_DIR}/curl_endian.h" "${CURL_LIBRARY_DIR}/curl_des.h" ++ "${CURL_LIBRARY_DIR}/curl_printf.h" "${CURL_LIBRARY_DIR}/system_win32.h" "${CURL_LIBRARY_DIR}/rand.h" "${CURL_LIBRARY_DIR}/mime.h" "${CURL_LIBRARY_DIR}/curl_sha256.h" "${CURL_LIBRARY_DIR}/setopt.h" ++ "${CURL_LIBRARY_DIR}/curl_path.h" "${CURL_LIBRARY_DIR}/curl_ctype.h" "${CURL_LIBRARY_DIR}/curl_range.h" "${CURL_LIBRARY_DIR}/psl.h" "${CURL_LIBRARY_DIR}/doh.h" "${CURL_LIBRARY_DIR}/urlapi-int.h" ++ "${CURL_LIBRARY_DIR}/curl_get_line.h" "${CURL_LIBRARY_DIR}/altsvc.h" "${CURL_LIBRARY_DIR}/quic.h" "${CURL_LIBRARY_DIR}/socketpair.h") ++ ++SET(LIB_RCFILES "${CURL_LIBRARY_DIR}/libcurl.rc") ++ ++SET(CSOURCES ${LIB_CFILES} ${LIB_VAUTH_CFILES} ${LIB_VTLS_CFILES} ++ ${LIB_VQUIC_CFILES} ${LIB_VSSH_CFILES}) ++SET(HHEADERS ${LIB_HFILES} ${LIB_VAUTH_HFILES} ${LIB_VTLS_HFILES} ++ ${LIB_VQUIC_HFILES} ${LIB_VSSH_HFILES}) ++ ++configure_file(${CURL_SOURCE_DIR}/lib/curl_config.h.cmake ++ ${CMAKE_CURRENT_BINARY_DIR}/curl/curl_config.h) ++ ++list(APPEND HHEADERS ++ ${CMAKE_CURRENT_BINARY_DIR}/curl/curl_config.h ++ ) ++ ++add_library(libcurl ${HHEADERS} ${CSOURCES}) ++ ++if(NOT BUILD_SHARED_LIBS) ++ set_target_properties(libcurl PROPERTIES INTERFACE_COMPILE_DEFINITIONS CURL_STATICLIB) ++endif() ++ ++if(HIDES_CURL_PRIVATE_SYMBOLS) ++ set_property(TARGET libcurl APPEND PROPERTY COMPILE_DEFINITIONS "CURL_HIDDEN_SYMBOLS") ++ set_property(TARGET libcurl APPEND PROPERTY COMPILE_FLAGS ${CURL_CFLAG_SYMBOLS_HIDE}) ++endif() ++ ++if(OPENSSL_FOUND) ++ target_include_directories(libcurl PUBLIC ${OPENSSL_INCLUDE_DIR}) ++ message("-- Including openssl ${OPENSSL_INCLUDE_DIR} to curl") ++endif() ++ ++if(ZLIB_FOUND) ++ target_include_directories(libcurl PUBLIC ${ZLIB_INCLUDE_DIRS}}) ++ message("-- Including zlib ${ZLIB_INCLUDE_DIRS} to curl") ++endif() ++ ++target_compile_definitions(libcurl PUBLIC -DHAVE_CONFIG_H) ++target_compile_definitions(libcurl PUBLIC -DBUILDING_LIBCURL) ++target_include_directories(libcurl PUBLIC "${CURL_SOURCE_DIR}/include" "${CURL_LIBRARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/curl") + +-# For Windows, all compilers used by CMake should support large files +-if(WIN32) +- set(USE_WIN32_LARGE_FILES ON) +- +- # Use the manifest embedded in the Windows Resource +- set(CMAKE_RC_FLAGS "${CMAKE_RC_FLAGS} -DCURL_EMBED_MANIFEST") +-endif() +- +-if(MSVC) +- # Disable default manifest added by CMake +- set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO") +- +- add_definitions(-D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE) +- if(CMAKE_C_FLAGS MATCHES "/W[0-4]") +- string(REGEX REPLACE "/W[0-4]" "/W4" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") +- else() +- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W4") +- endif() +-endif() +- +-if(CURL_WERROR) +- if(MSVC_VERSION) +- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /WX") +- else() +- # this assumes clang or gcc style options +- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror") +- endif() +-endif() +- +-# Ugly (but functional) way to include "Makefile.inc" by transforming it (= regenerate it). +-function(transform_makefile_inc INPUT_FILE OUTPUT_FILE) +- file(READ ${INPUT_FILE} MAKEFILE_INC_TEXT) +- string(REPLACE "$(top_srcdir)" "\${CURL_SOURCE_DIR}" MAKEFILE_INC_TEXT ${MAKEFILE_INC_TEXT}) +- string(REPLACE "$(top_builddir)" "\${CURL_BINARY_DIR}" MAKEFILE_INC_TEXT ${MAKEFILE_INC_TEXT}) +- +- string(REGEX REPLACE "\\\\\n" "!π!α!" MAKEFILE_INC_TEXT ${MAKEFILE_INC_TEXT}) +- string(REGEX REPLACE "([a-zA-Z_][a-zA-Z0-9_]*)[\t ]*=[\t ]*([^\n]*)" "SET(\\1 \\2)" MAKEFILE_INC_TEXT ${MAKEFILE_INC_TEXT}) +- string(REPLACE "!π!α!" "\n" MAKEFILE_INC_TEXT ${MAKEFILE_INC_TEXT}) +- +- string(REGEX REPLACE "\\$\\(([a-zA-Z_][a-zA-Z0-9_]*)\\)" "\${\\1}" MAKEFILE_INC_TEXT ${MAKEFILE_INC_TEXT}) # Replace $() with ${} +- string(REGEX REPLACE "@([a-zA-Z_][a-zA-Z0-9_]*)@" "\${\\1}" MAKEFILE_INC_TEXT ${MAKEFILE_INC_TEXT}) # Replace @@ with ${}, even if that may not be read by CMake scripts. +- file(WRITE ${OUTPUT_FILE} ${MAKEFILE_INC_TEXT}) +- +-endfunction() +- +-include(GNUInstallDirs) +- +-set(CURL_INSTALL_CMAKE_DIR ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}) +-set(TARGETS_EXPORT_NAME "${PROJECT_NAME}Targets") +-set(generated_dir "${CMAKE_CURRENT_BINARY_DIR}/generated") +-set(project_config "${generated_dir}/${PROJECT_NAME}Config.cmake") +-set(version_config "${generated_dir}/${PROJECT_NAME}ConfigVersion.cmake") +- +-if(USE_MANUAL) +- add_subdirectory(docs) +-endif() +- +-add_subdirectory(lib) +- +-if(BUILD_CURL_EXE) +- add_subdirectory(src) +-endif() +- +-include(CTest) +-if(BUILD_TESTING) +- add_subdirectory(tests) +-endif() +- +-# Helper to populate a list (_items) with a label when conditions (the remaining +-# args) are satisfied +-function(_add_if label) +- # TODO need to disable policy CMP0054 (CMake 3.1) to allow this indirection +- if(${ARGN}) +- set(_items ${_items} "${label}" PARENT_SCOPE) +- endif() +-endfunction() +- +-# Clear list and try to detect available features +-set(_items) +-_add_if("SSL" SSL_ENABLED) +-_add_if("IPv6" ENABLE_IPV6) +-_add_if("unix-sockets" USE_UNIX_SOCKETS) +-_add_if("libz" HAVE_LIBZ) +-_add_if("AsynchDNS" USE_ARES OR USE_THREADS_POSIX OR USE_THREADS_WIN32) +-_add_if("IDN" HAVE_LIBIDN2) +-_add_if("Largefile" (CURL_SIZEOF_CURL_OFF_T GREATER 4) AND +- ((SIZEOF_OFF_T GREATER 4) OR USE_WIN32_LARGE_FILES)) +-# TODO SSP1 (WinSSL) check is missing +-_add_if("SSPI" USE_WINDOWS_SSPI) +-_add_if("GSS-API" HAVE_GSSAPI) +-# TODO SSP1 missing for SPNEGO +-_add_if("SPNEGO" NOT CURL_DISABLE_CRYPTO_AUTH AND +- (HAVE_GSSAPI OR USE_WINDOWS_SSPI)) +-_add_if("Kerberos" NOT CURL_DISABLE_CRYPTO_AUTH AND +- (HAVE_GSSAPI OR USE_WINDOWS_SSPI)) +-# NTLM support requires crypto function adaptions from various SSL libs +-# TODO alternative SSL libs tests for SSP1, GNUTLS, NSS +-if(NOT CURL_DISABLE_CRYPTO_AUTH AND (USE_OPENSSL OR USE_WINDOWS_SSPI OR USE_SECTRANSP OR USE_MBEDTLS)) +- _add_if("NTLM" 1) +- # TODO missing option (autoconf: --enable-ntlm-wb) +- _add_if("NTLM_WB" NOT CURL_DISABLE_HTTP AND NTLM_WB_ENABLED) +-endif() +-# TODO missing option (--enable-tls-srp), depends on GNUTLS_SRP/OPENSSL_SRP +-_add_if("TLS-SRP" USE_TLS_SRP) +-# TODO option --with-nghttp2 tests for nghttp2 lib and nghttp2/nghttp2.h header +-_add_if("HTTP2" USE_NGHTTP2) +-string(REPLACE ";" " " SUPPORT_FEATURES "${_items}") +-message(STATUS "Enabled features: ${SUPPORT_FEATURES}") +- +-# Clear list and try to detect available protocols +-set(_items) +-_add_if("HTTP" NOT CURL_DISABLE_HTTP) +-_add_if("HTTPS" NOT CURL_DISABLE_HTTP AND SSL_ENABLED) +-_add_if("FTP" NOT CURL_DISABLE_FTP) +-_add_if("FTPS" NOT CURL_DISABLE_FTP AND SSL_ENABLED) +-_add_if("FILE" NOT CURL_DISABLE_FILE) +-_add_if("TELNET" NOT CURL_DISABLE_TELNET) +-_add_if("LDAP" NOT CURL_DISABLE_LDAP) +-# CURL_DISABLE_LDAP implies CURL_DISABLE_LDAPS +-# TODO check HAVE_LDAP_SSL (in autoconf this is enabled with --enable-ldaps) +-_add_if("LDAPS" NOT CURL_DISABLE_LDAPS AND +- ((USE_OPENLDAP AND SSL_ENABLED) OR +- (NOT USE_OPENLDAP AND HAVE_LDAP_SSL))) +-_add_if("DICT" NOT CURL_DISABLE_DICT) +-_add_if("TFTP" NOT CURL_DISABLE_TFTP) +-_add_if("GOPHER" NOT CURL_DISABLE_GOPHER) +-_add_if("POP3" NOT CURL_DISABLE_POP3) +-_add_if("POP3S" NOT CURL_DISABLE_POP3 AND SSL_ENABLED) +-_add_if("IMAP" NOT CURL_DISABLE_IMAP) +-_add_if("IMAPS" NOT CURL_DISABLE_IMAP AND SSL_ENABLED) +-_add_if("SMTP" NOT CURL_DISABLE_SMTP) +-_add_if("SMTPS" NOT CURL_DISABLE_SMTP AND SSL_ENABLED) +-_add_if("SCP" USE_LIBSSH2) +-_add_if("SFTP" USE_LIBSSH2) +-_add_if("RTSP" NOT CURL_DISABLE_RTSP) +-_add_if("RTMP" USE_LIBRTMP) +-if(_items) +- list(SORT _items) +-endif() +-string(REPLACE ";" " " SUPPORT_PROTOCOLS "${_items}") +-message(STATUS "Enabled protocols: ${SUPPORT_PROTOCOLS}") +- +-# Clear list and collect SSL backends +-set(_items) +-_add_if("WinSSL" SSL_ENABLED AND USE_WINDOWS_SSPI) +-_add_if("OpenSSL" SSL_ENABLED AND USE_OPENSSL) +-_add_if("Secure Transport" SSL_ENABLED AND USE_SECTRANSP) +-_add_if("mbedTLS" SSL_ENABLED AND USE_MBEDTLS) +-_add_if("BearSSL" SSL_ENABLED AND USE_BEARSSL) +-if(_items) +- list(SORT _items) +-endif() +-string(REPLACE ";" " " SSL_BACKENDS "${_items}") +-message(STATUS "Enabled SSL backends: ${SSL_BACKENDS}") +- +-# curl-config needs the following options to be set. +-set(CC "${CMAKE_C_COMPILER}") +-# TODO probably put a -D... options here? +-set(CONFIGURE_OPTIONS "") +-# TODO when to set "-DCURL_STATICLIB" for CPPFLAG_CURL_STATICLIB? +-set(CPPFLAG_CURL_STATICLIB "") +-set(CURLVERSION "${CURL_VERSION}") +-if(BUILD_SHARED_LIBS) +- set(ENABLE_SHARED "yes") +- set(ENABLE_STATIC "no") +-else() +- set(ENABLE_SHARED "no") +- set(ENABLE_STATIC "yes") +-endif() +-set(exec_prefix "\${prefix}") +-set(includedir "\${prefix}/include") +-set(LDFLAGS "${CMAKE_SHARED_LINKER_FLAGS}") +-set(LIBCURL_LIBS "") +-set(libdir "${CMAKE_INSTALL_PREFIX}/lib") +-foreach(_lib ${CMAKE_C_IMPLICIT_LINK_LIBRARIES} ${CURL_LIBS}) +- if(_lib MATCHES ".*/.*" OR _lib MATCHES "^-") +- set(LIBCURL_LIBS "${LIBCURL_LIBS} ${_lib}") +- else() +- set(LIBCURL_LIBS "${LIBCURL_LIBS} -l${_lib}") +- endif() +-endforeach() +-# "a" (Linux) or "lib" (Windows) +-string(REPLACE "." "" libext "${CMAKE_STATIC_LIBRARY_SUFFIX}") +-set(prefix "${CMAKE_INSTALL_PREFIX}") +-# Set this to "yes" to append all libraries on which -lcurl is dependent +-set(REQUIRE_LIB_DEPS "no") +-# SUPPORT_FEATURES +-# SUPPORT_PROTOCOLS +-set(VERSIONNUM "${CURL_VERSION_NUM}") +- +-# Finally generate a "curl-config" matching this config +-# Use: +-# * ENABLE_SHARED +-# * ENABLE_STATIC +-configure_file("${CURL_SOURCE_DIR}/curl-config.in" +- "${CURL_BINARY_DIR}/curl-config" @ONLY) +-install(FILES "${CURL_BINARY_DIR}/curl-config" +- DESTINATION ${CMAKE_INSTALL_BINDIR} +- PERMISSIONS +- OWNER_READ OWNER_WRITE OWNER_EXECUTE +- GROUP_READ GROUP_EXECUTE +- WORLD_READ WORLD_EXECUTE) +- +-# Finally generate a pkg-config file matching this config +-configure_file("${CURL_SOURCE_DIR}/libcurl.pc.in" +- "${CURL_BINARY_DIR}/libcurl.pc" @ONLY) +-install(FILES "${CURL_BINARY_DIR}/libcurl.pc" +- DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) +- +-# install headers +-install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/include/curl" +- DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +- FILES_MATCHING PATTERN "*.h") +- +-include(CMakePackageConfigHelpers) +-write_basic_package_version_file( +- "${version_config}" +- VERSION ${CURL_VERSION} +- COMPATIBILITY SameMajorVersion +-) +- +-# Use: +-# * TARGETS_EXPORT_NAME +-# * PROJECT_NAME +-configure_package_config_file(CMake/curl-config.cmake.in +- "${project_config}" +- INSTALL_DESTINATION ${CURL_INSTALL_CMAKE_DIR} +-) +- +-install( +- EXPORT "${TARGETS_EXPORT_NAME}" +- NAMESPACE "${PROJECT_NAME}::" +- DESTINATION ${CURL_INSTALL_CMAKE_DIR} +-) +- +-install( +- FILES ${version_config} ${project_config} +- DESTINATION ${CURL_INSTALL_CMAKE_DIR} +-) +- +-# Workaround for MSVS10 to avoid the Dialog Hell +-# FIXME: This could be removed with future version of CMake. +-if(MSVC_VERSION EQUAL 1600) +- set(CURL_SLN_FILENAME "${CMAKE_CURRENT_BINARY_DIR}/CURL.sln") +- if(EXISTS "${CURL_SLN_FILENAME}") +- file(APPEND "${CURL_SLN_FILENAME}" "\n# This should be regenerated!\n") +- endif() +-endif() +- +-if(NOT TARGET uninstall) +- configure_file( +- ${CMAKE_CURRENT_SOURCE_DIR}/CMake/cmake_uninstall.cmake.in +- ${CMAKE_CURRENT_BINARY_DIR}/CMake/cmake_uninstall.cmake +- IMMEDIATE @ONLY) +- +- add_custom_target(uninstall +- COMMAND ${CMAKE_COMMAND} -P +- ${CMAKE_CURRENT_BINARY_DIR}/CMake/cmake_uninstall.cmake) +-endif() ++target_link_libraries(libcurl ${CURL_LIBS}) diff --git a/contrib/curl-cmake/CMakeLists.txt b/contrib/curl-cmake/CMakeLists.txt new file mode 100644 index 00000000000..17aeef6e165 --- /dev/null +++ b/contrib/curl-cmake/CMakeLists.txt @@ -0,0 +1,634 @@ +#*************************************************************************** +# _ _ ____ _ +# Project ___| | | | _ \| | +# / __| | | | |_) | | +# | (__| |_| | _ <| |___ +# \___|\___/|_| \_\_____| +# +# Copyright (C) 1998 - 2019, Daniel Stenberg, , et al. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at https://curl.haxx.se/docs/copyright.html. +# +# You may opt to use, copy, modify, merge, publish, distribute and/or sell +# copies of the Software, and permit persons to whom the Software is +# furnished to do so, under the terms of the COPYING file. +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY +# KIND, either express or implied. +# +########################################################################### + +# NOTE: +# This file is shrinked and reworked version of original curl CMakeLists.txt +# Original file link https://github.com/curl/curl/blob/3b8bbbbd1609c638a3d3d0acb148a33dedb67be3/CMakeLists.txt +# If you need to update curl building you can find patch file in this directory +# and apply it to fresh original CMakeLists.txt file. +cmake_minimum_required(VERSION 3.0 FATAL_ERROR) + +SET(CURL_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl) +SET(CURL_LIBRARY_DIR ${CURL_SOURCE_DIR}/lib) +set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake;${CMAKE_MODULE_PATH}") +# Disable status messages when perform checks +set(CMAKE_REQUIRED_QUIET TRUE) + +include(Macros) +include(CMakeDependentOption) +include(CheckCCompilerFlag) + +file(READ ${CURL_SOURCE_DIR}/include/curl/curlver.h CURL_VERSION_H_CONTENTS) +string(REGEX MATCH "#define LIBCURL_VERSION \"[^\"]*" + CURL_VERSION ${CURL_VERSION_H_CONTENTS}) +string(REGEX REPLACE "[^\"]+\"" "" CURL_VERSION ${CURL_VERSION}) +string(REGEX MATCH "#define LIBCURL_VERSION_NUM 0x[0-9a-fA-F]+" + CURL_VERSION_NUM ${CURL_VERSION_H_CONTENTS}) +string(REGEX REPLACE "[^0]+0x" "" CURL_VERSION_NUM ${CURL_VERSION_NUM}) + +message(STATUS "Use curl version=[${CURL_VERSION}]") +set(OPERATING_SYSTEM "${CMAKE_SYSTEM_NAME}") +set(OS "\"${CMAKE_SYSTEM_NAME}\"") + +option(PICKY_COMPILER "Enable picky compiler options" ON) +option(ENABLE_THREADED_RESOLVER "Set to ON to enable threaded DNS lookup" ON) + +if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG) + if(PICKY_COMPILER) + foreach(_CCOPT -pedantic -Wall -W -Wpointer-arith -Wwrite-strings -Wunused -Wshadow -Winline -Wnested-externs -Wmissing-declarations -Wmissing-prototypes -Wno-long-long -Wfloat-equal -Wno-multichar -Wsign-compare -Wundef -Wno-format-nonliteral -Wendif-labels -Wstrict-prototypes -Wdeclaration-after-statement -Wstrict-aliasing=3 -Wcast-align -Wtype-limits -Wold-style-declaration -Wmissing-parameter-type -Wempty-body -Wclobbered -Wignored-qualifiers -Wconversion -Wno-sign-conversion -Wvla -Wdouble-promotion -Wno-system-headers -Wno-pedantic-ms-format) + # surprisingly, CHECK_C_COMPILER_FLAG needs a new variable to store each new + # test result in. + check_c_compiler_flag(${_CCOPT} OPT${_CCOPT}) + if(OPT${_CCOPT}) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_CCOPT}") + endif() + endforeach() + endif() +endif() + +# For debug libs and exes, add "-d" postfix +if(NOT DEFINED CMAKE_DEBUG_POSTFIX) + set(CMAKE_DEBUG_POSTFIX "-d") +endif() + +# initialize CURL_LIBS +set(CURL_LIBS "") + +include(CurlSymbolHiding) + +# Http only +set(CURL_DISABLE_FTP ON) +set(CURL_DISABLE_LDAP ON) +set(CURL_DISABLE_LDAPS ON) +set(CURL_DISABLE_TELNET ON) +set(CURL_DISABLE_DICT ON) +set(CURL_DISABLE_FILE ON) +set(CURL_DISABLE_TFTP ON) +set(CURL_DISABLE_RTSP ON) +set(CURL_DISABLE_POP3 ON) +set(CURL_DISABLE_IMAP ON) +set(CURL_DISABLE_SMTP ON) +set(CURL_DISABLE_GOPHER ON) + +option(CURL_DISABLE_COOKIES "to disable cookies support" OFF) +mark_as_advanced(CURL_DISABLE_COOKIES) + +option(CURL_DISABLE_CRYPTO_AUTH "to disable cryptographic authentication" OFF) +mark_as_advanced(CURL_DISABLE_CRYPTO_AUTH) + +option(CURL_DISABLE_VERBOSE_STRINGS "to disable verbose strings" OFF) +mark_as_advanced(CURL_DISABLE_VERBOSE_STRINGS) + +option(ENABLE_IPV6 "Define if you want to enable IPv6 support" ON) +mark_as_advanced(ENABLE_IPV6) + +if(ENABLE_IPV6 AND NOT WIN32) + include(CheckStructHasMember) + check_struct_has_member("struct sockaddr_in6" sin6_addr "netinet/in.h" + HAVE_SOCKADDR_IN6_SIN6_ADDR) + check_struct_has_member("struct sockaddr_in6" sin6_scope_id "netinet/in.h" + HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID) + if(NOT HAVE_SOCKADDR_IN6_SIN6_ADDR) + message(WARNING "struct sockaddr_in6 not available, disabling IPv6 support") + # Force the feature off as this name is used as guard macro... + set(ENABLE_IPV6 OFF + CACHE BOOL "Define if you want to enable IPv6 support" FORCE) + endif() +endif() + +curl_nroff_check() + +# We need ansi c-flags, especially on HP +set(CMAKE_C_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_C_FLAGS}") +set(CMAKE_REQUIRED_FLAGS ${CMAKE_ANSI_CFLAGS}) + +# Include all the necessary files for macros +include(CheckFunctionExists) +include(CheckIncludeFile) +include(CheckIncludeFiles) +include(CheckLibraryExists) +include(CheckSymbolExists) +include(CheckTypeSize) +include(CheckCSourceCompiles) + +if(ENABLE_THREADED_RESOLVER) + find_package(Threads REQUIRED) + if(WIN32) + set(USE_THREADS_WIN32 ON) + else() + set(USE_THREADS_POSIX ${CMAKE_USE_PTHREADS_INIT}) + set(HAVE_PTHREAD_H ${CMAKE_USE_PTHREADS_INIT}) + endif() + set(CURL_LIBS ${CURL_LIBS} ${CMAKE_THREAD_LIBS_INIT}) +endif() + +# Check for all needed libraries +check_library_exists_concat("${CMAKE_DL_LIBS}" dlopen HAVE_LIBDL) +check_library_exists_concat("socket" connect HAVE_LIBSOCKET) +check_library_exists("c" gethostbyname "" NOT_NEED_LIBNSL) + +check_function_exists(gethostname HAVE_GETHOSTNAME) + +# From cmake/find/ssl.cmake +if (OPENSSL_FOUND) + set(SSL_ENABLED ON) + set(USE_OPENSSL ON) + + list(APPEND CURL_LIBS ${OPENSSL_LIBRARIES}) + check_include_file("openssl/crypto.h" HAVE_OPENSSL_CRYPTO_H) + check_include_file("openssl/err.h" HAVE_OPENSSL_ERR_H) + check_include_file("openssl/pem.h" HAVE_OPENSSL_PEM_H) + check_include_file("openssl/rsa.h" HAVE_OPENSSL_RSA_H) + check_include_file("openssl/ssl.h" HAVE_OPENSSL_SSL_H) + check_include_file("openssl/x509.h" HAVE_OPENSSL_X509_H) + check_include_file("openssl/rand.h" HAVE_OPENSSL_RAND_H) + check_symbol_exists(RAND_status "${CURL_INCLUDES}" HAVE_RAND_STATUS) + check_symbol_exists(RAND_screen "${CURL_INCLUDES}" HAVE_RAND_SCREEN) + check_symbol_exists(RAND_egd "${CURL_INCLUDES}" HAVE_RAND_EGD) +endif() + +# Check for idn +check_library_exists_concat("idn2" idn2_lookup_ul HAVE_LIBIDN2) + +# Check for symbol dlopen (same as HAVE_LIBDL) +check_library_exists("${CURL_LIBS}" dlopen "" HAVE_DLOPEN) + +# From /cmake/find/zlib.cmake +if (ZLIB_FOUND) + set(HAVE_ZLIB_H ON) + set(HAVE_LIBZ ON) + set(USE_ZLIB ON) + + list(APPEND CURL_LIBS ${ZLIB_LIBRARIES}) +endif() + +option(ENABLE_UNIX_SOCKETS "Define if you want Unix domain sockets support" ON) +if(ENABLE_UNIX_SOCKETS) + include(CheckStructHasMember) + check_struct_has_member("struct sockaddr_un" sun_path "sys/un.h" USE_UNIX_SOCKETS) +else() + unset(USE_UNIX_SOCKETS CACHE) +endif() + +# CA handling +# Explicitly set to most common case +if (OPENSSL_FOUND) + set(CURL_CA_BUNDLE "/etc/ssl/certs/ca-certificates.crt") + set(CURL_CA_BUNDLE_SET TRUE CACHE BOOL "Path to the CA bundle has been set") + set(CURL_CA_PATH "/etc/ssl/certs") + set(CURL_CA_PATH_SET TRUE CACHE BOOL "Path to the CA bundle has been set") +endif() + +check_include_file_concat("stdio.h" HAVE_STDIO_H) +check_include_file_concat("inttypes.h" HAVE_INTTYPES_H) +check_include_file_concat("sys/filio.h" HAVE_SYS_FILIO_H) +check_include_file_concat("sys/ioctl.h" HAVE_SYS_IOCTL_H) +check_include_file_concat("sys/param.h" HAVE_SYS_PARAM_H) +check_include_file_concat("sys/poll.h" HAVE_SYS_POLL_H) +check_include_file_concat("sys/resource.h" HAVE_SYS_RESOURCE_H) +check_include_file_concat("sys/select.h" HAVE_SYS_SELECT_H) +check_include_file_concat("sys/socket.h" HAVE_SYS_SOCKET_H) +check_include_file_concat("sys/sockio.h" HAVE_SYS_SOCKIO_H) +check_include_file_concat("sys/stat.h" HAVE_SYS_STAT_H) +check_include_file_concat("sys/time.h" HAVE_SYS_TIME_H) +check_include_file_concat("sys/types.h" HAVE_SYS_TYPES_H) +check_include_file_concat("sys/uio.h" HAVE_SYS_UIO_H) +check_include_file_concat("sys/un.h" HAVE_SYS_UN_H) +check_include_file_concat("sys/utime.h" HAVE_SYS_UTIME_H) +check_include_file_concat("sys/xattr.h" HAVE_SYS_XATTR_H) +check_include_file_concat("alloca.h" HAVE_ALLOCA_H) +check_include_file_concat("arpa/inet.h" HAVE_ARPA_INET_H) +check_include_file_concat("arpa/tftp.h" HAVE_ARPA_TFTP_H) +check_include_file_concat("assert.h" HAVE_ASSERT_H) +check_include_file_concat("crypto.h" HAVE_CRYPTO_H) +check_include_file_concat("des.h" HAVE_DES_H) +check_include_file_concat("err.h" HAVE_ERR_H) +check_include_file_concat("errno.h" HAVE_ERRNO_H) +check_include_file_concat("fcntl.h" HAVE_FCNTL_H) +check_include_file_concat("idn2.h" HAVE_IDN2_H) +check_include_file_concat("ifaddrs.h" HAVE_IFADDRS_H) +check_include_file_concat("io.h" HAVE_IO_H) +check_include_file_concat("krb.h" HAVE_KRB_H) +check_include_file_concat("libgen.h" HAVE_LIBGEN_H) +check_include_file_concat("locale.h" HAVE_LOCALE_H) +check_include_file_concat("net/if.h" HAVE_NET_IF_H) +check_include_file_concat("netdb.h" HAVE_NETDB_H) +check_include_file_concat("netinet/in.h" HAVE_NETINET_IN_H) +check_include_file_concat("netinet/tcp.h" HAVE_NETINET_TCP_H) + +check_include_file_concat("pem.h" HAVE_PEM_H) +check_include_file_concat("poll.h" HAVE_POLL_H) +check_include_file_concat("pwd.h" HAVE_PWD_H) +check_include_file_concat("rsa.h" HAVE_RSA_H) +check_include_file_concat("setjmp.h" HAVE_SETJMP_H) +check_include_file_concat("sgtty.h" HAVE_SGTTY_H) +check_include_file_concat("signal.h" HAVE_SIGNAL_H) +check_include_file_concat("ssl.h" HAVE_SSL_H) +check_include_file_concat("stdbool.h" HAVE_STDBOOL_H) +check_include_file_concat("stdint.h" HAVE_STDINT_H) +check_include_file_concat("stdio.h" HAVE_STDIO_H) +check_include_file_concat("stdlib.h" HAVE_STDLIB_H) +check_include_file_concat("string.h" HAVE_STRING_H) +check_include_file_concat("strings.h" HAVE_STRINGS_H) +check_include_file_concat("stropts.h" HAVE_STROPTS_H) +check_include_file_concat("termio.h" HAVE_TERMIO_H) +check_include_file_concat("termios.h" HAVE_TERMIOS_H) +check_include_file_concat("time.h" HAVE_TIME_H) +check_include_file_concat("unistd.h" HAVE_UNISTD_H) +check_include_file_concat("utime.h" HAVE_UTIME_H) +check_include_file_concat("x509.h" HAVE_X509_H) + +check_include_file_concat("process.h" HAVE_PROCESS_H) +check_include_file_concat("stddef.h" HAVE_STDDEF_H) +check_include_file_concat("dlfcn.h" HAVE_DLFCN_H) +check_include_file_concat("malloc.h" HAVE_MALLOC_H) +check_include_file_concat("memory.h" HAVE_MEMORY_H) +check_include_file_concat("netinet/if_ether.h" HAVE_NETINET_IF_ETHER_H) +check_include_file_concat("stdint.h" HAVE_STDINT_H) +check_include_file_concat("sockio.h" HAVE_SOCKIO_H) +check_include_file_concat("sys/utsname.h" HAVE_SYS_UTSNAME_H) + +check_type_size(size_t SIZEOF_SIZE_T) +check_type_size(ssize_t SIZEOF_SSIZE_T) +check_type_size("long long" SIZEOF_LONG_LONG) +check_type_size("long" SIZEOF_LONG) +check_type_size("short" SIZEOF_SHORT) +check_type_size("int" SIZEOF_INT) +check_type_size("__int64" SIZEOF___INT64) +check_type_size("long double" SIZEOF_LONG_DOUBLE) +check_type_size("time_t" SIZEOF_TIME_T) +if(NOT HAVE_SIZEOF_SSIZE_T) + if(SIZEOF_LONG EQUAL SIZEOF_SIZE_T) + set(ssize_t long) + endif() + if(NOT ssize_t AND SIZEOF___INT64 EQUAL SIZEOF_SIZE_T) + set(ssize_t __int64) + endif() +endif() +# off_t is sized later, after the HAVE_FILE_OFFSET_BITS test + +if(HAVE_SIZEOF_LONG_LONG) + set(HAVE_LONGLONG 1) + set(HAVE_LL 1) +endif() + +find_file(RANDOM_FILE urandom /dev) +mark_as_advanced(RANDOM_FILE) + +# Check for some functions that are used +if(HAVE_LIBWS2_32) + set(CMAKE_REQUIRED_LIBRARIES ws2_32) +elseif(HAVE_LIBSOCKET) + set(CMAKE_REQUIRED_LIBRARIES socket) +endif() + +check_symbol_exists(basename "${CURL_INCLUDES}" HAVE_BASENAME) +check_symbol_exists(socket "${CURL_INCLUDES}" HAVE_SOCKET) +check_symbol_exists(select "${CURL_INCLUDES}" HAVE_SELECT) +check_symbol_exists(poll "${CURL_INCLUDES}" HAVE_POLL) +check_symbol_exists(strdup "${CURL_INCLUDES}" HAVE_STRDUP) +check_symbol_exists(strstr "${CURL_INCLUDES}" HAVE_STRSTR) +check_symbol_exists(strtok_r "${CURL_INCLUDES}" HAVE_STRTOK_R) +check_symbol_exists(strftime "${CURL_INCLUDES}" HAVE_STRFTIME) +check_symbol_exists(uname "${CURL_INCLUDES}" HAVE_UNAME) +check_symbol_exists(strcasecmp "${CURL_INCLUDES}" HAVE_STRCASECMP) +check_symbol_exists(stricmp "${CURL_INCLUDES}" HAVE_STRICMP) +check_symbol_exists(strcmpi "${CURL_INCLUDES}" HAVE_STRCMPI) +check_symbol_exists(strncmpi "${CURL_INCLUDES}" HAVE_STRNCMPI) +check_symbol_exists(alarm "${CURL_INCLUDES}" HAVE_ALARM) +if(NOT HAVE_STRNCMPI) + set(HAVE_STRCMPI) +endif() +check_symbol_exists(gethostbyaddr "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR) +check_symbol_exists(gethostbyaddr_r "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR_R) +check_symbol_exists(gettimeofday "${CURL_INCLUDES}" HAVE_GETTIMEOFDAY) +check_symbol_exists(inet_addr "${CURL_INCLUDES}" HAVE_INET_ADDR) +check_symbol_exists(inet_ntoa "${CURL_INCLUDES}" HAVE_INET_NTOA) +check_symbol_exists(inet_ntoa_r "${CURL_INCLUDES}" HAVE_INET_NTOA_R) +check_symbol_exists(tcsetattr "${CURL_INCLUDES}" HAVE_TCSETATTR) +check_symbol_exists(tcgetattr "${CURL_INCLUDES}" HAVE_TCGETATTR) +check_symbol_exists(perror "${CURL_INCLUDES}" HAVE_PERROR) +check_symbol_exists(closesocket "${CURL_INCLUDES}" HAVE_CLOSESOCKET) +check_symbol_exists(setvbuf "${CURL_INCLUDES}" HAVE_SETVBUF) +check_symbol_exists(sigsetjmp "${CURL_INCLUDES}" HAVE_SIGSETJMP) +check_symbol_exists(getpass_r "${CURL_INCLUDES}" HAVE_GETPASS_R) +check_symbol_exists(strlcat "${CURL_INCLUDES}" HAVE_STRLCAT) +check_symbol_exists(getpwuid "${CURL_INCLUDES}" HAVE_GETPWUID) +check_symbol_exists(getpwuid_r "${CURL_INCLUDES}" HAVE_GETPWUID_R) +check_symbol_exists(geteuid "${CURL_INCLUDES}" HAVE_GETEUID) +check_symbol_exists(usleep "${CURL_INCLUDES}" HAVE_USLEEP) +check_symbol_exists(utime "${CURL_INCLUDES}" HAVE_UTIME) +check_symbol_exists(gmtime_r "${CURL_INCLUDES}" HAVE_GMTIME_R) +check_symbol_exists(localtime_r "${CURL_INCLUDES}" HAVE_LOCALTIME_R) + +check_symbol_exists(gethostbyname "${CURL_INCLUDES}" HAVE_GETHOSTBYNAME) +check_symbol_exists(gethostbyname_r "${CURL_INCLUDES}" HAVE_GETHOSTBYNAME_R) + +check_symbol_exists(signal "${CURL_INCLUDES}" HAVE_SIGNAL_FUNC) +check_symbol_exists(SIGALRM "${CURL_INCLUDES}" HAVE_SIGNAL_MACRO) +if(HAVE_SIGNAL_FUNC AND HAVE_SIGNAL_MACRO) + set(HAVE_SIGNAL 1) +endif() +check_symbol_exists(uname "${CURL_INCLUDES}" HAVE_UNAME) +check_symbol_exists(strtoll "${CURL_INCLUDES}" HAVE_STRTOLL) +check_symbol_exists(_strtoi64 "${CURL_INCLUDES}" HAVE__STRTOI64) +check_symbol_exists(strerror_r "${CURL_INCLUDES}" HAVE_STRERROR_R) +check_symbol_exists(siginterrupt "${CURL_INCLUDES}" HAVE_SIGINTERRUPT) +check_symbol_exists(perror "${CURL_INCLUDES}" HAVE_PERROR) +check_symbol_exists(fork "${CURL_INCLUDES}" HAVE_FORK) +check_symbol_exists(getaddrinfo "${CURL_INCLUDES}" HAVE_GETADDRINFO) +check_symbol_exists(freeaddrinfo "${CURL_INCLUDES}" HAVE_FREEADDRINFO) +check_symbol_exists(freeifaddrs "${CURL_INCLUDES}" HAVE_FREEIFADDRS) +check_symbol_exists(pipe "${CURL_INCLUDES}" HAVE_PIPE) +check_symbol_exists(ftruncate "${CURL_INCLUDES}" HAVE_FTRUNCATE) +check_symbol_exists(getprotobyname "${CURL_INCLUDES}" HAVE_GETPROTOBYNAME) +check_symbol_exists(getpeername "${CURL_INCLUDES}" HAVE_GETPEERNAME) +check_symbol_exists(getsockname "${CURL_INCLUDES}" HAVE_GETSOCKNAME) +check_symbol_exists(if_nametoindex "${CURL_INCLUDES}" HAVE_IF_NAMETOINDEX) +check_symbol_exists(getrlimit "${CURL_INCLUDES}" HAVE_GETRLIMIT) +check_symbol_exists(setlocale "${CURL_INCLUDES}" HAVE_SETLOCALE) +check_symbol_exists(setmode "${CURL_INCLUDES}" HAVE_SETMODE) +check_symbol_exists(setrlimit "${CURL_INCLUDES}" HAVE_SETRLIMIT) +check_symbol_exists(fcntl "${CURL_INCLUDES}" HAVE_FCNTL) +check_symbol_exists(ioctl "${CURL_INCLUDES}" HAVE_IOCTL) +check_symbol_exists(setsockopt "${CURL_INCLUDES}" HAVE_SETSOCKOPT) +check_function_exists(mach_absolute_time HAVE_MACH_ABSOLUTE_TIME) + +check_symbol_exists(fsetxattr "${CURL_INCLUDES}" HAVE_FSETXATTR) +if(HAVE_FSETXATTR) + foreach(CURL_TEST HAVE_FSETXATTR_5 HAVE_FSETXATTR_6) + curl_internal_test(${CURL_TEST}) + endforeach() +endif() + +# sigaction and sigsetjmp are special. Use special mechanism for +# detecting those, but only if previous attempt failed. +if(HAVE_SIGNAL_H) + check_symbol_exists(sigaction "signal.h" HAVE_SIGACTION) +endif() + +if(NOT HAVE_SIGSETJMP) + if(HAVE_SETJMP_H) + check_symbol_exists(sigsetjmp "setjmp.h" HAVE_MACRO_SIGSETJMP) + if(HAVE_MACRO_SIGSETJMP) + set(HAVE_SIGSETJMP 1) + endif() + endif() +endif() + +# If there is no stricmp(), do not allow LDAP to parse URLs +if(NOT HAVE_STRICMP) + set(HAVE_LDAP_URL_PARSE 1) +endif() + +# Do curl specific tests +foreach(CURL_TEST + HAVE_FCNTL_O_NONBLOCK + HAVE_IOCTLSOCKET + HAVE_IOCTLSOCKET_CAMEL + HAVE_IOCTLSOCKET_CAMEL_FIONBIO + HAVE_IOCTLSOCKET_FIONBIO + HAVE_IOCTL_FIONBIO + HAVE_IOCTL_SIOCGIFADDR + HAVE_SETSOCKOPT_SO_NONBLOCK + HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID + TIME_WITH_SYS_TIME + HAVE_O_NONBLOCK + HAVE_GETHOSTBYADDR_R_5 + HAVE_GETHOSTBYADDR_R_7 + HAVE_GETHOSTBYADDR_R_8 + HAVE_GETHOSTBYADDR_R_5_REENTRANT + HAVE_GETHOSTBYADDR_R_7_REENTRANT + HAVE_GETHOSTBYADDR_R_8_REENTRANT + HAVE_GETHOSTBYNAME_R_3 + HAVE_GETHOSTBYNAME_R_5 + HAVE_GETHOSTBYNAME_R_6 + HAVE_GETHOSTBYNAME_R_3_REENTRANT + HAVE_GETHOSTBYNAME_R_5_REENTRANT + HAVE_GETHOSTBYNAME_R_6_REENTRANT + HAVE_IN_ADDR_T + HAVE_BOOL_T + STDC_HEADERS + RETSIGTYPE_TEST + HAVE_INET_NTOA_R_DECL + HAVE_INET_NTOA_R_DECL_REENTRANT + HAVE_GETADDRINFO + HAVE_FILE_OFFSET_BITS + HAVE_VARIADIC_MACROS_C99 + HAVE_VARIADIC_MACROS_GCC + ) + curl_internal_test(${CURL_TEST}) +endforeach() + +if(HAVE_FILE_OFFSET_BITS) + set(_FILE_OFFSET_BITS 64) + set(CMAKE_REQUIRED_FLAGS "-D_FILE_OFFSET_BITS=64") +endif() +check_type_size("off_t" SIZEOF_OFF_T) + +# include this header to get the type +set(CMAKE_REQUIRED_INCLUDES "${CURL_SOURCE_DIR}/include") +set(CMAKE_EXTRA_INCLUDE_FILES "curl/system.h") +check_type_size("curl_off_t" SIZEOF_CURL_OFF_T) +set(CMAKE_EXTRA_INCLUDE_FILES "") + +foreach(CURL_TEST + HAVE_GLIBC_STRERROR_R + HAVE_POSIX_STRERROR_R + ) + curl_internal_test(${CURL_TEST}) +endforeach() + +# Check for reentrant +foreach(CURL_TEST + HAVE_GETHOSTBYADDR_R_5 + HAVE_GETHOSTBYADDR_R_7 + HAVE_GETHOSTBYADDR_R_8 + HAVE_GETHOSTBYNAME_R_3 + HAVE_GETHOSTBYNAME_R_5 + HAVE_GETHOSTBYNAME_R_6 + HAVE_INET_NTOA_R_DECL_REENTRANT) + if(NOT ${CURL_TEST}) + if(${CURL_TEST}_REENTRANT) + set(NEED_REENTRANT 1) + endif() + endif() +endforeach() + +if(NEED_REENTRANT) + foreach(CURL_TEST + HAVE_GETHOSTBYADDR_R_5 + HAVE_GETHOSTBYADDR_R_7 + HAVE_GETHOSTBYADDR_R_8 + HAVE_GETHOSTBYNAME_R_3 + HAVE_GETHOSTBYNAME_R_5 + HAVE_GETHOSTBYNAME_R_6) + set(${CURL_TEST} 0) + if(${CURL_TEST}_REENTRANT) + set(${CURL_TEST} 1) + endif() + endforeach() +endif() + +if(HAVE_INET_NTOA_R_DECL_REENTRANT) + set(HAVE_INET_NTOA_R_DECL 1) + set(NEED_REENTRANT 1) +endif() + +# Check clock_gettime(CLOCK_MONOTONIC, x) support +curl_internal_test(HAVE_CLOCK_GETTIME_MONOTONIC) + +# Check compiler support of __builtin_available() +curl_internal_test(HAVE_BUILTIN_AVAILABLE) + +# Some other minor tests + +if(NOT HAVE_IN_ADDR_T) + set(in_addr_t "unsigned long") +endif() + +# Check for nonblocking +set(HAVE_DISABLED_NONBLOCKING 1) +if(HAVE_FIONBIO OR + HAVE_IOCTLSOCKET OR + HAVE_IOCTLSOCKET_CASE OR + HAVE_O_NONBLOCK) + set(HAVE_DISABLED_NONBLOCKING) +endif() + +set(CURL_PULL_SYS_TYPES_H ${HAVE_SYS_TYPES_H}) +set(CURL_PULL_SYS_SOCKET_H ${HAVE_SYS_SOCKET_H}) +set(CURL_PULL_SYS_POLL_H ${HAVE_SYS_POLL_H}) +set(CURL_PULL_STDINT_H ${HAVE_STDINT_H}) +set(CURL_PULL_INTTYPES_H ${HAVE_INTTYPES_H}) + +include(CMake/OtherTests.cmake) + +SET(LIB_VAUTH_CFILES + "${CURL_LIBRARY_DIR}/vauth/vauth.c" "${CURL_LIBRARY_DIR}/vauth/cleartext.c" "${CURL_LIBRARY_DIR}/vauth/cram.c" + "${CURL_LIBRARY_DIR}/vauth/digest.c" "${CURL_LIBRARY_DIR}/vauth/digest_sspi.c" "${CURL_LIBRARY_DIR}/vauth/krb5_gssapi.c" + "${CURL_LIBRARY_DIR}/vauth/krb5_sspi.c" "${CURL_LIBRARY_DIR}/vauth/ntlm.c" "${CURL_LIBRARY_DIR}/vauth/ntlm_sspi.c" "${CURL_LIBRARY_DIR}/vauth/oauth2.c" + "${CURL_LIBRARY_DIR}/vauth/spnego_gssapi.c" "${CURL_LIBRARY_DIR}/vauth/spnego_sspi.c") + +SET(LIB_VAUTH_HFILES "${CURL_LIBRARY_DIR}/vauth/vauth.h" "${CURL_LIBRARY_DIR}/vauth/digest.h" "${CURL_LIBRARY_DIR}/vauth/ntlm.h") + +SET(LIB_VTLS_CFILES "${CURL_LIBRARY_DIR}/vtls/openssl.c" "${CURL_LIBRARY_DIR}/vtls/gtls.c" "${CURL_LIBRARY_DIR}/vtls/vtls.c" "${CURL_LIBRARY_DIR}/vtls/nss.c" + "${CURL_LIBRARY_DIR}/vtls/polarssl.c" "${CURL_LIBRARY_DIR}/vtls/polarssl_threadlock.c" + "${CURL_LIBRARY_DIR}/vtls/wolfssl.c" "${CURL_LIBRARY_DIR}/vtls/schannel.c" "${CURL_LIBRARY_DIR}/vtls/schannel_verify.c" + "${CURL_LIBRARY_DIR}/vtls/sectransp.c" "${CURL_LIBRARY_DIR}/vtls/gskit.c" "${CURL_LIBRARY_DIR}/vtls/mbedtls.c" "${CURL_LIBRARY_DIR}/vtls/mesalink.c" + "${CURL_LIBRARY_DIR}/vtls/bearssl.c") + +SET(LIB_VTLS_HFILES "${CURL_LIBRARY_DIR}/vtls/openssl.h" "${CURL_LIBRARY_DIR}/vtls/vtls.h" "${CURL_LIBRARY_DIR}/vtls/gtls.h" + "${CURL_LIBRARY_DIR}/vtls/nssg.h" "${CURL_LIBRARY_DIR}/vtls/polarssl.h" "${CURL_LIBRARY_DIR}/vtls/polarssl_threadlock.h" + "${CURL_LIBRARY_DIR}/vtls/wolfssl.h" "${CURL_LIBRARY_DIR}/vtls/schannel.h" "${CURL_LIBRARY_DIR}/vtls/sectransp.h" "${CURL_LIBRARY_DIR}/vtls/gskit.h" + "${CURL_LIBRARY_DIR}/vtls/mbedtls.h" "${CURL_LIBRARY_DIR}/vtls/mesalink.h" "${CURL_LIBRARY_DIR}/vtls/bearssl.h") + +SET(LIB_VQUIC_CFILES "${CURL_LIBRARY_DIR}/vquic/ngtcp2.c" "${CURL_LIBRARY_DIR}/vquic/quiche.c") + +SET(LIB_VQUIC_HFILES "${CURL_LIBRARY_DIR}/vquic/ngtcp2.h" "${CURL_LIBRARY_DIR}/vquic/quiche.h") + +SET(LIB_VSSH_CFILES "${CURL_LIBRARY_DIR}/vssh/libssh2.c" "${CURL_LIBRARY_DIR}/vssh/libssh.c") + +SET(LIB_VSSH_HFILES "${CURL_LIBRARY_DIR}/vssh/ssh.h") + +SET(LIB_CFILES "${CURL_LIBRARY_DIR}/file.c" + "${CURL_LIBRARY_DIR}/timeval.c" "${CURL_LIBRARY_DIR}/base64.c" "${CURL_LIBRARY_DIR}/hostip.c" "${CURL_LIBRARY_DIR}/progress.c" "${CURL_LIBRARY_DIR}/formdata.c" + "${CURL_LIBRARY_DIR}/cookie.c" "${CURL_LIBRARY_DIR}/http.c" "${CURL_LIBRARY_DIR}/sendf.c" "${CURL_LIBRARY_DIR}/url.c" "${CURL_LIBRARY_DIR}/dict.c" "${CURL_LIBRARY_DIR}/if2ip.c" "${CURL_LIBRARY_DIR}/speedcheck.c" + "${CURL_LIBRARY_DIR}/ldap.c" "${CURL_LIBRARY_DIR}/version.c" "${CURL_LIBRARY_DIR}/getenv.c" "${CURL_LIBRARY_DIR}/escape.c" "${CURL_LIBRARY_DIR}/mprintf.c" "${CURL_LIBRARY_DIR}/telnet.c" "${CURL_LIBRARY_DIR}/netrc.c" + "${CURL_LIBRARY_DIR}/getinfo.c" "${CURL_LIBRARY_DIR}/transfer.c" "${CURL_LIBRARY_DIR}/strcase.c" "${CURL_LIBRARY_DIR}/easy.c" "${CURL_LIBRARY_DIR}/security.c" "${CURL_LIBRARY_DIR}/curl_fnmatch.c" + "${CURL_LIBRARY_DIR}/fileinfo.c" "${CURL_LIBRARY_DIR}/wildcard.c" "${CURL_LIBRARY_DIR}/krb5.c" "${CURL_LIBRARY_DIR}/memdebug.c" "${CURL_LIBRARY_DIR}/http_chunks.c" + "${CURL_LIBRARY_DIR}/strtok.c" "${CURL_LIBRARY_DIR}/connect.c" "${CURL_LIBRARY_DIR}/llist.c" "${CURL_LIBRARY_DIR}/hash.c" "${CURL_LIBRARY_DIR}/multi.c" "${CURL_LIBRARY_DIR}/content_encoding.c" "${CURL_LIBRARY_DIR}/share.c" + "${CURL_LIBRARY_DIR}/http_digest.c" "${CURL_LIBRARY_DIR}/md4.c" "${CURL_LIBRARY_DIR}/md5.c" "${CURL_LIBRARY_DIR}/http_negotiate.c" "${CURL_LIBRARY_DIR}/inet_pton.c" "${CURL_LIBRARY_DIR}/strtoofft.c" + "${CURL_LIBRARY_DIR}/strerror.c" "${CURL_LIBRARY_DIR}/amigaos.c" "${CURL_LIBRARY_DIR}/hostasyn.c" "${CURL_LIBRARY_DIR}/hostip4.c" "${CURL_LIBRARY_DIR}/hostip6.c" "${CURL_LIBRARY_DIR}/hostsyn.c" + "${CURL_LIBRARY_DIR}/inet_ntop.c" "${CURL_LIBRARY_DIR}/parsedate.c" "${CURL_LIBRARY_DIR}/select.c" "${CURL_LIBRARY_DIR}/splay.c" "${CURL_LIBRARY_DIR}/strdup.c" "${CURL_LIBRARY_DIR}/socks.c" + "${CURL_LIBRARY_DIR}/curl_addrinfo.c" "${CURL_LIBRARY_DIR}/socks_gssapi.c" "${CURL_LIBRARY_DIR}/socks_sspi.c" + "${CURL_LIBRARY_DIR}/curl_sspi.c" "${CURL_LIBRARY_DIR}/slist.c" "${CURL_LIBRARY_DIR}/nonblock.c" "${CURL_LIBRARY_DIR}/curl_memrchr.c" "${CURL_LIBRARY_DIR}/imap.c" "${CURL_LIBRARY_DIR}/pop3.c" "${CURL_LIBRARY_DIR}/smtp.c" + "${CURL_LIBRARY_DIR}/pingpong.c" "${CURL_LIBRARY_DIR}/rtsp.c" "${CURL_LIBRARY_DIR}/curl_threads.c" "${CURL_LIBRARY_DIR}/warnless.c" "${CURL_LIBRARY_DIR}/hmac.c" "${CURL_LIBRARY_DIR}/curl_rtmp.c" + "${CURL_LIBRARY_DIR}/openldap.c" "${CURL_LIBRARY_DIR}/curl_gethostname.c" "${CURL_LIBRARY_DIR}/gopher.c" "${CURL_LIBRARY_DIR}/idn_win32.c" + "${CURL_LIBRARY_DIR}/http_proxy.c" "${CURL_LIBRARY_DIR}/non-ascii.c" "${CURL_LIBRARY_DIR}/asyn-ares.c" "${CURL_LIBRARY_DIR}/asyn-thread.c" "${CURL_LIBRARY_DIR}/curl_gssapi.c" + "${CURL_LIBRARY_DIR}/http_ntlm.c" "${CURL_LIBRARY_DIR}/curl_ntlm_wb.c" "${CURL_LIBRARY_DIR}/curl_ntlm_core.c" "${CURL_LIBRARY_DIR}/curl_sasl.c" "${CURL_LIBRARY_DIR}/rand.c" + "${CURL_LIBRARY_DIR}/curl_multibyte.c" "${CURL_LIBRARY_DIR}/hostcheck.c" "${CURL_LIBRARY_DIR}/conncache.c" "${CURL_LIBRARY_DIR}/dotdot.c" + "${CURL_LIBRARY_DIR}/x509asn1.c" "${CURL_LIBRARY_DIR}/http2.c" "${CURL_LIBRARY_DIR}/smb.c" "${CURL_LIBRARY_DIR}/curl_endian.c" "${CURL_LIBRARY_DIR}/curl_des.c" "${CURL_LIBRARY_DIR}/system_win32.c" + "${CURL_LIBRARY_DIR}/mime.c" "${CURL_LIBRARY_DIR}/sha256.c" "${CURL_LIBRARY_DIR}/setopt.c" "${CURL_LIBRARY_DIR}/curl_path.c" "${CURL_LIBRARY_DIR}/curl_ctype.c" "${CURL_LIBRARY_DIR}/curl_range.c" "${CURL_LIBRARY_DIR}/psl.c" + "${CURL_LIBRARY_DIR}/doh.c" "${CURL_LIBRARY_DIR}/urlapi.c" "${CURL_LIBRARY_DIR}/curl_get_line.c" "${CURL_LIBRARY_DIR}/altsvc.c" "${CURL_LIBRARY_DIR}/socketpair.c") + +SET(LIB_HFILES "${CURL_LIBRARY_DIR}/arpa_telnet.h" "${CURL_LIBRARY_DIR}/netrc.h" "${CURL_LIBRARY_DIR}/file.h" "${CURL_LIBRARY_DIR}/timeval.h" "${CURL_LIBRARY_DIR}/hostip.h" "${CURL_LIBRARY_DIR}/progress.h" + "${CURL_LIBRARY_DIR}/formdata.h" "${CURL_LIBRARY_DIR}/cookie.h" "${CURL_LIBRARY_DIR}/http.h" "${CURL_LIBRARY_DIR}/sendf.h" "${CURL_LIBRARY_DIR}/url.h" "${CURL_LIBRARY_DIR}/dict.h" "${CURL_LIBRARY_DIR}/if2ip.h" + "${CURL_LIBRARY_DIR}/speedcheck.h" "${CURL_LIBRARY_DIR}/urldata.h" "${CURL_LIBRARY_DIR}/curl_ldap.h" "${CURL_LIBRARY_DIR}/escape.h" "${CURL_LIBRARY_DIR}/telnet.h" "${CURL_LIBRARY_DIR}/getinfo.h" + "${CURL_LIBRARY_DIR}/strcase.h" "${CURL_LIBRARY_DIR}/curl_sec.h" "${CURL_LIBRARY_DIR}/memdebug.h" "${CURL_LIBRARY_DIR}/http_chunks.h" "${CURL_LIBRARY_DIR}/curl_fnmatch.h" + "${CURL_LIBRARY_DIR}/wildcard.h" "${CURL_LIBRARY_DIR}/fileinfo.h" "${CURL_LIBRARY_DIR}/strtok.h" "${CURL_LIBRARY_DIR}/connect.h" "${CURL_LIBRARY_DIR}/llist.h" + "${CURL_LIBRARY_DIR}/hash.h" "${CURL_LIBRARY_DIR}/content_encoding.h" "${CURL_LIBRARY_DIR}/share.h" "${CURL_LIBRARY_DIR}/curl_md4.h" "${CURL_LIBRARY_DIR}/curl_md5.h" "${CURL_LIBRARY_DIR}/http_digest.h" + "${CURL_LIBRARY_DIR}/http_negotiate.h" "${CURL_LIBRARY_DIR}/inet_pton.h" "${CURL_LIBRARY_DIR}/amigaos.h" "${CURL_LIBRARY_DIR}/strtoofft.h" "${CURL_LIBRARY_DIR}/strerror.h" + "${CURL_LIBRARY_DIR}/inet_ntop.h" "${CURL_LIBRARY_DIR}/curlx.h" "${CURL_LIBRARY_DIR}/curl_memory.h" "${CURL_LIBRARY_DIR}/curl_setup.h" "${CURL_LIBRARY_DIR}/transfer.h" "${CURL_LIBRARY_DIR}/select.h" + "${CURL_LIBRARY_DIR}/easyif.h" "${CURL_LIBRARY_DIR}/multiif.h" "${CURL_LIBRARY_DIR}/parsedate.h" "${CURL_LIBRARY_DIR}/sockaddr.h" "${CURL_LIBRARY_DIR}/splay.h" "${CURL_LIBRARY_DIR}/strdup.h" + "${CURL_LIBRARY_DIR}/socks.h" "${CURL_LIBRARY_DIR}/curl_base64.h" "${CURL_LIBRARY_DIR}/curl_addrinfo.h" "${CURL_LIBRARY_DIR}/curl_sspi.h" + "${CURL_LIBRARY_DIR}/slist.h" "${CURL_LIBRARY_DIR}/nonblock.h" "${CURL_LIBRARY_DIR}/curl_memrchr.h" "${CURL_LIBRARY_DIR}/imap.h" "${CURL_LIBRARY_DIR}/pop3.h" "${CURL_LIBRARY_DIR}/smtp.h" "${CURL_LIBRARY_DIR}/pingpong.h" + "${CURL_LIBRARY_DIR}/rtsp.h" "${CURL_LIBRARY_DIR}/curl_threads.h" "${CURL_LIBRARY_DIR}/warnless.h" "${CURL_LIBRARY_DIR}/curl_hmac.h" "${CURL_LIBRARY_DIR}/curl_rtmp.h" + "${CURL_LIBRARY_DIR}/curl_gethostname.h" "${CURL_LIBRARY_DIR}/gopher.h" "${CURL_LIBRARY_DIR}/http_proxy.h" "${CURL_LIBRARY_DIR}/non-ascii.h" "${CURL_LIBRARY_DIR}/asyn.h" + "${CURL_LIBRARY_DIR}/http_ntlm.h" "${CURL_LIBRARY_DIR}/curl_gssapi.h" "${CURL_LIBRARY_DIR}/curl_ntlm_wb.h" "${CURL_LIBRARY_DIR}/curl_ntlm_core.h" + "${CURL_LIBRARY_DIR}/curl_sasl.h" "${CURL_LIBRARY_DIR}/curl_multibyte.h" "${CURL_LIBRARY_DIR}/hostcheck.h" "${CURL_LIBRARY_DIR}/conncache.h" + "${CURL_LIBRARY_DIR}/multihandle.h" "${CURL_LIBRARY_DIR}/setup-vms.h" "${CURL_LIBRARY_DIR}/dotdot.h" + "${CURL_LIBRARY_DIR}/x509asn1.h" "${CURL_LIBRARY_DIR}/http2.h" "${CURL_LIBRARY_DIR}/sigpipe.h" "${CURL_LIBRARY_DIR}/smb.h" "${CURL_LIBRARY_DIR}/curl_endian.h" "${CURL_LIBRARY_DIR}/curl_des.h" + "${CURL_LIBRARY_DIR}/curl_printf.h" "${CURL_LIBRARY_DIR}/system_win32.h" "${CURL_LIBRARY_DIR}/rand.h" "${CURL_LIBRARY_DIR}/mime.h" "${CURL_LIBRARY_DIR}/curl_sha256.h" "${CURL_LIBRARY_DIR}/setopt.h" + "${CURL_LIBRARY_DIR}/curl_path.h" "${CURL_LIBRARY_DIR}/curl_ctype.h" "${CURL_LIBRARY_DIR}/curl_range.h" "${CURL_LIBRARY_DIR}/psl.h" "${CURL_LIBRARY_DIR}/doh.h" "${CURL_LIBRARY_DIR}/urlapi-int.h" + "${CURL_LIBRARY_DIR}/curl_get_line.h" "${CURL_LIBRARY_DIR}/altsvc.h" "${CURL_LIBRARY_DIR}/quic.h" "${CURL_LIBRARY_DIR}/socketpair.h") + +SET(LIB_RCFILES "${CURL_LIBRARY_DIR}/libcurl.rc") + +SET(CSOURCES ${LIB_CFILES} ${LIB_VAUTH_CFILES} ${LIB_VTLS_CFILES} + ${LIB_VQUIC_CFILES} ${LIB_VSSH_CFILES}) +SET(HHEADERS ${LIB_HFILES} ${LIB_VAUTH_HFILES} ${LIB_VTLS_HFILES} + ${LIB_VQUIC_HFILES} ${LIB_VSSH_HFILES}) + +configure_file(${CURL_SOURCE_DIR}/lib/curl_config.h.cmake + ${CMAKE_CURRENT_BINARY_DIR}/curl/curl_config.h) + +list(APPEND HHEADERS + ${CMAKE_CURRENT_BINARY_DIR}/curl/curl_config.h + ) + +add_library(libcurl ${HHEADERS} ${CSOURCES}) + +if(NOT BUILD_SHARED_LIBS) + set_target_properties(libcurl PROPERTIES INTERFACE_COMPILE_DEFINITIONS CURL_STATICLIB) +endif() + +if(HIDES_CURL_PRIVATE_SYMBOLS) + set_property(TARGET libcurl APPEND PROPERTY COMPILE_DEFINITIONS "CURL_HIDDEN_SYMBOLS") + set_property(TARGET libcurl APPEND PROPERTY COMPILE_FLAGS ${CURL_CFLAG_SYMBOLS_HIDE}) +endif() + +if(OPENSSL_FOUND) + target_include_directories(libcurl PUBLIC ${OPENSSL_INCLUDE_DIR}) + message("-- Including openssl ${OPENSSL_INCLUDE_DIR} to curl") +endif() + +if(ZLIB_FOUND) + target_include_directories(libcurl PUBLIC ${ZLIB_INCLUDE_DIRS}}) + message("-- Including zlib ${ZLIB_INCLUDE_DIRS} to curl") +endif() + +target_compile_definitions(libcurl PUBLIC -DHAVE_CONFIG_H) +target_compile_definitions(libcurl PUBLIC -DBUILDING_LIBCURL) +target_include_directories(libcurl PUBLIC "${CURL_SOURCE_DIR}/include" "${CURL_LIBRARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/curl") + +target_link_libraries(libcurl ${CURL_LIBS}) diff --git a/contrib/icu b/contrib/icu new file mode 160000 index 00000000000..faa2f9f9e1f --- /dev/null +++ b/contrib/icu @@ -0,0 +1 @@ +Subproject commit faa2f9f9e1fe74c5ed00eba371d2830134cdbea1 diff --git a/contrib/icu-cmake/CMakeLists.txt b/contrib/icu-cmake/CMakeLists.txt new file mode 100644 index 00000000000..64e82366076 --- /dev/null +++ b/contrib/icu-cmake/CMakeLists.txt @@ -0,0 +1,459 @@ +set(ICU_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/icu/icu4c/source) +set(ICUDATA_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/icudata/) + +# These lists of sources were generated from build log of the original ICU build system (configure + make). + +set(ICUUC_SOURCES +${ICU_SOURCE_DIR}/common/errorcode.cpp +${ICU_SOURCE_DIR}/common/putil.cpp +${ICU_SOURCE_DIR}/common/umath.cpp +${ICU_SOURCE_DIR}/common/utypes.cpp +${ICU_SOURCE_DIR}/common/uinvchar.cpp +${ICU_SOURCE_DIR}/common/umutex.cpp +${ICU_SOURCE_DIR}/common/ucln_cmn.cpp +${ICU_SOURCE_DIR}/common/uinit.cpp +${ICU_SOURCE_DIR}/common/uobject.cpp +${ICU_SOURCE_DIR}/common/cmemory.cpp +${ICU_SOURCE_DIR}/common/charstr.cpp +${ICU_SOURCE_DIR}/common/cstr.cpp +${ICU_SOURCE_DIR}/common/udata.cpp +${ICU_SOURCE_DIR}/common/ucmndata.cpp +${ICU_SOURCE_DIR}/common/udatamem.cpp +${ICU_SOURCE_DIR}/common/umapfile.cpp +${ICU_SOURCE_DIR}/common/udataswp.cpp +${ICU_SOURCE_DIR}/common/utrie_swap.cpp +${ICU_SOURCE_DIR}/common/ucol_swp.cpp +${ICU_SOURCE_DIR}/common/utrace.cpp +${ICU_SOURCE_DIR}/common/uhash.cpp +${ICU_SOURCE_DIR}/common/uhash_us.cpp +${ICU_SOURCE_DIR}/common/uenum.cpp +${ICU_SOURCE_DIR}/common/ustrenum.cpp +${ICU_SOURCE_DIR}/common/uvector.cpp +${ICU_SOURCE_DIR}/common/ustack.cpp +${ICU_SOURCE_DIR}/common/uvectr32.cpp +${ICU_SOURCE_DIR}/common/uvectr64.cpp +${ICU_SOURCE_DIR}/common/ucnv.cpp +${ICU_SOURCE_DIR}/common/ucnv_bld.cpp +${ICU_SOURCE_DIR}/common/ucnv_cnv.cpp +${ICU_SOURCE_DIR}/common/ucnv_io.cpp +${ICU_SOURCE_DIR}/common/ucnv_cb.cpp +${ICU_SOURCE_DIR}/common/ucnv_err.cpp +${ICU_SOURCE_DIR}/common/ucnvlat1.cpp +${ICU_SOURCE_DIR}/common/ucnv_u7.cpp +${ICU_SOURCE_DIR}/common/ucnv_u8.cpp +${ICU_SOURCE_DIR}/common/ucnv_u16.cpp +${ICU_SOURCE_DIR}/common/ucnv_u32.cpp +${ICU_SOURCE_DIR}/common/ucnvscsu.cpp +${ICU_SOURCE_DIR}/common/ucnvbocu.cpp +${ICU_SOURCE_DIR}/common/ucnv_ext.cpp +${ICU_SOURCE_DIR}/common/ucnvmbcs.cpp +${ICU_SOURCE_DIR}/common/ucnv2022.cpp +${ICU_SOURCE_DIR}/common/ucnvhz.cpp +${ICU_SOURCE_DIR}/common/ucnv_lmb.cpp +${ICU_SOURCE_DIR}/common/ucnvisci.cpp +${ICU_SOURCE_DIR}/common/ucnvdisp.cpp +${ICU_SOURCE_DIR}/common/ucnv_set.cpp +${ICU_SOURCE_DIR}/common/ucnv_ct.cpp +${ICU_SOURCE_DIR}/common/resource.cpp +${ICU_SOURCE_DIR}/common/uresbund.cpp +${ICU_SOURCE_DIR}/common/ures_cnv.cpp +${ICU_SOURCE_DIR}/common/uresdata.cpp +${ICU_SOURCE_DIR}/common/resbund.cpp +${ICU_SOURCE_DIR}/common/resbund_cnv.cpp +${ICU_SOURCE_DIR}/common/ucurr.cpp +${ICU_SOURCE_DIR}/common/localebuilder.cpp +${ICU_SOURCE_DIR}/common/localeprioritylist.cpp +${ICU_SOURCE_DIR}/common/messagepattern.cpp +${ICU_SOURCE_DIR}/common/ucat.cpp +${ICU_SOURCE_DIR}/common/locmap.cpp +${ICU_SOURCE_DIR}/common/uloc.cpp +${ICU_SOURCE_DIR}/common/locid.cpp +${ICU_SOURCE_DIR}/common/locutil.cpp +${ICU_SOURCE_DIR}/common/locavailable.cpp +${ICU_SOURCE_DIR}/common/locdispnames.cpp +${ICU_SOURCE_DIR}/common/locdspnm.cpp +${ICU_SOURCE_DIR}/common/loclikely.cpp +${ICU_SOURCE_DIR}/common/locresdata.cpp +${ICU_SOURCE_DIR}/common/lsr.cpp +${ICU_SOURCE_DIR}/common/loclikelysubtags.cpp +${ICU_SOURCE_DIR}/common/locdistance.cpp +${ICU_SOURCE_DIR}/common/localematcher.cpp +${ICU_SOURCE_DIR}/common/bytestream.cpp +${ICU_SOURCE_DIR}/common/stringpiece.cpp +${ICU_SOURCE_DIR}/common/bytesinkutil.cpp +${ICU_SOURCE_DIR}/common/stringtriebuilder.cpp +${ICU_SOURCE_DIR}/common/bytestriebuilder.cpp +${ICU_SOURCE_DIR}/common/bytestrie.cpp +${ICU_SOURCE_DIR}/common/bytestrieiterator.cpp +${ICU_SOURCE_DIR}/common/ucharstrie.cpp +${ICU_SOURCE_DIR}/common/ucharstriebuilder.cpp +${ICU_SOURCE_DIR}/common/ucharstrieiterator.cpp +${ICU_SOURCE_DIR}/common/dictionarydata.cpp +${ICU_SOURCE_DIR}/common/edits.cpp +${ICU_SOURCE_DIR}/common/appendable.cpp +${ICU_SOURCE_DIR}/common/ustr_cnv.cpp +${ICU_SOURCE_DIR}/common/unistr_cnv.cpp +${ICU_SOURCE_DIR}/common/unistr.cpp +${ICU_SOURCE_DIR}/common/unistr_case.cpp +${ICU_SOURCE_DIR}/common/unistr_props.cpp +${ICU_SOURCE_DIR}/common/utf_impl.cpp +${ICU_SOURCE_DIR}/common/ustring.cpp +${ICU_SOURCE_DIR}/common/ustrcase.cpp +${ICU_SOURCE_DIR}/common/ucasemap.cpp +${ICU_SOURCE_DIR}/common/ucasemap_titlecase_brkiter.cpp +${ICU_SOURCE_DIR}/common/cstring.cpp +${ICU_SOURCE_DIR}/common/ustrfmt.cpp +${ICU_SOURCE_DIR}/common/ustrtrns.cpp +${ICU_SOURCE_DIR}/common/ustr_wcs.cpp +${ICU_SOURCE_DIR}/common/utext.cpp +${ICU_SOURCE_DIR}/common/unistr_case_locale.cpp +${ICU_SOURCE_DIR}/common/ustrcase_locale.cpp +${ICU_SOURCE_DIR}/common/unistr_titlecase_brkiter.cpp +${ICU_SOURCE_DIR}/common/ustr_titlecase_brkiter.cpp +${ICU_SOURCE_DIR}/common/normalizer2impl.cpp +${ICU_SOURCE_DIR}/common/normalizer2.cpp +${ICU_SOURCE_DIR}/common/filterednormalizer2.cpp +${ICU_SOURCE_DIR}/common/normlzr.cpp +${ICU_SOURCE_DIR}/common/unorm.cpp +${ICU_SOURCE_DIR}/common/unormcmp.cpp +${ICU_SOURCE_DIR}/common/loadednormalizer2impl.cpp +${ICU_SOURCE_DIR}/common/chariter.cpp +${ICU_SOURCE_DIR}/common/schriter.cpp +${ICU_SOURCE_DIR}/common/uchriter.cpp +${ICU_SOURCE_DIR}/common/uiter.cpp +${ICU_SOURCE_DIR}/common/patternprops.cpp +${ICU_SOURCE_DIR}/common/uchar.cpp +${ICU_SOURCE_DIR}/common/uprops.cpp +${ICU_SOURCE_DIR}/common/ucase.cpp +${ICU_SOURCE_DIR}/common/propname.cpp +${ICU_SOURCE_DIR}/common/ubidi_props.cpp +${ICU_SOURCE_DIR}/common/characterproperties.cpp +${ICU_SOURCE_DIR}/common/ubidi.cpp +${ICU_SOURCE_DIR}/common/ubidiwrt.cpp +${ICU_SOURCE_DIR}/common/ubidiln.cpp +${ICU_SOURCE_DIR}/common/ushape.cpp +${ICU_SOURCE_DIR}/common/uscript.cpp +${ICU_SOURCE_DIR}/common/uscript_props.cpp +${ICU_SOURCE_DIR}/common/usc_impl.cpp +${ICU_SOURCE_DIR}/common/unames.cpp +${ICU_SOURCE_DIR}/common/utrie.cpp +${ICU_SOURCE_DIR}/common/utrie2.cpp +${ICU_SOURCE_DIR}/common/utrie2_builder.cpp +${ICU_SOURCE_DIR}/common/ucptrie.cpp +${ICU_SOURCE_DIR}/common/umutablecptrie.cpp +${ICU_SOURCE_DIR}/common/bmpset.cpp +${ICU_SOURCE_DIR}/common/unisetspan.cpp +${ICU_SOURCE_DIR}/common/uset_props.cpp +${ICU_SOURCE_DIR}/common/uniset_props.cpp +${ICU_SOURCE_DIR}/common/uniset_closure.cpp +${ICU_SOURCE_DIR}/common/uset.cpp +${ICU_SOURCE_DIR}/common/uniset.cpp +${ICU_SOURCE_DIR}/common/usetiter.cpp +${ICU_SOURCE_DIR}/common/ruleiter.cpp +${ICU_SOURCE_DIR}/common/caniter.cpp +${ICU_SOURCE_DIR}/common/unifilt.cpp +${ICU_SOURCE_DIR}/common/unifunct.cpp +${ICU_SOURCE_DIR}/common/uarrsort.cpp +${ICU_SOURCE_DIR}/common/brkiter.cpp +${ICU_SOURCE_DIR}/common/ubrk.cpp +${ICU_SOURCE_DIR}/common/brkeng.cpp +${ICU_SOURCE_DIR}/common/dictbe.cpp +${ICU_SOURCE_DIR}/common/filteredbrk.cpp +${ICU_SOURCE_DIR}/common/rbbi.cpp +${ICU_SOURCE_DIR}/common/rbbidata.cpp +${ICU_SOURCE_DIR}/common/rbbinode.cpp +${ICU_SOURCE_DIR}/common/rbbirb.cpp +${ICU_SOURCE_DIR}/common/rbbiscan.cpp +${ICU_SOURCE_DIR}/common/rbbisetb.cpp +${ICU_SOURCE_DIR}/common/rbbistbl.cpp +${ICU_SOURCE_DIR}/common/rbbitblb.cpp +${ICU_SOURCE_DIR}/common/rbbi_cache.cpp +${ICU_SOURCE_DIR}/common/serv.cpp +${ICU_SOURCE_DIR}/common/servnotf.cpp +${ICU_SOURCE_DIR}/common/servls.cpp +${ICU_SOURCE_DIR}/common/servlk.cpp +${ICU_SOURCE_DIR}/common/servlkf.cpp +${ICU_SOURCE_DIR}/common/servrbf.cpp +${ICU_SOURCE_DIR}/common/servslkf.cpp +${ICU_SOURCE_DIR}/common/uidna.cpp +${ICU_SOURCE_DIR}/common/usprep.cpp +${ICU_SOURCE_DIR}/common/uts46.cpp +${ICU_SOURCE_DIR}/common/punycode.cpp +${ICU_SOURCE_DIR}/common/util.cpp +${ICU_SOURCE_DIR}/common/util_props.cpp +${ICU_SOURCE_DIR}/common/parsepos.cpp +${ICU_SOURCE_DIR}/common/locbased.cpp +${ICU_SOURCE_DIR}/common/cwchar.cpp +${ICU_SOURCE_DIR}/common/wintz.cpp +${ICU_SOURCE_DIR}/common/dtintrv.cpp +${ICU_SOURCE_DIR}/common/ucnvsel.cpp +${ICU_SOURCE_DIR}/common/propsvec.cpp +${ICU_SOURCE_DIR}/common/ulist.cpp +${ICU_SOURCE_DIR}/common/uloc_tag.cpp +${ICU_SOURCE_DIR}/common/icudataver.cpp +${ICU_SOURCE_DIR}/common/icuplug.cpp +${ICU_SOURCE_DIR}/common/sharedobject.cpp +${ICU_SOURCE_DIR}/common/simpleformatter.cpp +${ICU_SOURCE_DIR}/common/unifiedcache.cpp +${ICU_SOURCE_DIR}/common/uloc_keytype.cpp +${ICU_SOURCE_DIR}/common/ubiditransform.cpp +${ICU_SOURCE_DIR}/common/pluralmap.cpp +${ICU_SOURCE_DIR}/common/static_unicode_sets.cpp +${ICU_SOURCE_DIR}/common/restrace.cpp) + +set(ICUI18N_SOURCES +${ICU_SOURCE_DIR}/i18n/ucln_in.cpp +${ICU_SOURCE_DIR}/i18n/fmtable.cpp +${ICU_SOURCE_DIR}/i18n/format.cpp +${ICU_SOURCE_DIR}/i18n/msgfmt.cpp +${ICU_SOURCE_DIR}/i18n/umsg.cpp +${ICU_SOURCE_DIR}/i18n/numfmt.cpp +${ICU_SOURCE_DIR}/i18n/unum.cpp +${ICU_SOURCE_DIR}/i18n/decimfmt.cpp +${ICU_SOURCE_DIR}/i18n/dcfmtsym.cpp +${ICU_SOURCE_DIR}/i18n/fmtable_cnv.cpp +${ICU_SOURCE_DIR}/i18n/choicfmt.cpp +${ICU_SOURCE_DIR}/i18n/datefmt.cpp +${ICU_SOURCE_DIR}/i18n/smpdtfmt.cpp +${ICU_SOURCE_DIR}/i18n/reldtfmt.cpp +${ICU_SOURCE_DIR}/i18n/dtfmtsym.cpp +${ICU_SOURCE_DIR}/i18n/udat.cpp +${ICU_SOURCE_DIR}/i18n/dtptngen.cpp +${ICU_SOURCE_DIR}/i18n/udatpg.cpp +${ICU_SOURCE_DIR}/i18n/nfrs.cpp +${ICU_SOURCE_DIR}/i18n/nfrule.cpp +${ICU_SOURCE_DIR}/i18n/nfsubs.cpp +${ICU_SOURCE_DIR}/i18n/rbnf.cpp +${ICU_SOURCE_DIR}/i18n/numsys.cpp +${ICU_SOURCE_DIR}/i18n/unumsys.cpp +${ICU_SOURCE_DIR}/i18n/ucsdet.cpp +${ICU_SOURCE_DIR}/i18n/ucal.cpp +${ICU_SOURCE_DIR}/i18n/calendar.cpp +${ICU_SOURCE_DIR}/i18n/gregocal.cpp +${ICU_SOURCE_DIR}/i18n/timezone.cpp +${ICU_SOURCE_DIR}/i18n/simpletz.cpp +${ICU_SOURCE_DIR}/i18n/olsontz.cpp +${ICU_SOURCE_DIR}/i18n/astro.cpp +${ICU_SOURCE_DIR}/i18n/taiwncal.cpp +${ICU_SOURCE_DIR}/i18n/buddhcal.cpp +${ICU_SOURCE_DIR}/i18n/persncal.cpp +${ICU_SOURCE_DIR}/i18n/islamcal.cpp +${ICU_SOURCE_DIR}/i18n/japancal.cpp +${ICU_SOURCE_DIR}/i18n/gregoimp.cpp +${ICU_SOURCE_DIR}/i18n/hebrwcal.cpp +${ICU_SOURCE_DIR}/i18n/indiancal.cpp +${ICU_SOURCE_DIR}/i18n/chnsecal.cpp +${ICU_SOURCE_DIR}/i18n/cecal.cpp +${ICU_SOURCE_DIR}/i18n/coptccal.cpp +${ICU_SOURCE_DIR}/i18n/dangical.cpp +${ICU_SOURCE_DIR}/i18n/ethpccal.cpp +${ICU_SOURCE_DIR}/i18n/coleitr.cpp +${ICU_SOURCE_DIR}/i18n/coll.cpp +${ICU_SOURCE_DIR}/i18n/sortkey.cpp +${ICU_SOURCE_DIR}/i18n/bocsu.cpp +${ICU_SOURCE_DIR}/i18n/ucoleitr.cpp +${ICU_SOURCE_DIR}/i18n/ucol.cpp +${ICU_SOURCE_DIR}/i18n/ucol_res.cpp +${ICU_SOURCE_DIR}/i18n/ucol_sit.cpp +${ICU_SOURCE_DIR}/i18n/collation.cpp +${ICU_SOURCE_DIR}/i18n/collationsettings.cpp +${ICU_SOURCE_DIR}/i18n/collationdata.cpp +${ICU_SOURCE_DIR}/i18n/collationtailoring.cpp +${ICU_SOURCE_DIR}/i18n/collationdatareader.cpp +${ICU_SOURCE_DIR}/i18n/collationdatawriter.cpp +${ICU_SOURCE_DIR}/i18n/collationfcd.cpp +${ICU_SOURCE_DIR}/i18n/collationiterator.cpp +${ICU_SOURCE_DIR}/i18n/utf16collationiterator.cpp +${ICU_SOURCE_DIR}/i18n/utf8collationiterator.cpp +${ICU_SOURCE_DIR}/i18n/uitercollationiterator.cpp +${ICU_SOURCE_DIR}/i18n/collationsets.cpp +${ICU_SOURCE_DIR}/i18n/collationcompare.cpp +${ICU_SOURCE_DIR}/i18n/collationfastlatin.cpp +${ICU_SOURCE_DIR}/i18n/collationkeys.cpp +${ICU_SOURCE_DIR}/i18n/rulebasedcollator.cpp +${ICU_SOURCE_DIR}/i18n/collationroot.cpp +${ICU_SOURCE_DIR}/i18n/collationrootelements.cpp +${ICU_SOURCE_DIR}/i18n/collationdatabuilder.cpp +${ICU_SOURCE_DIR}/i18n/collationweights.cpp +${ICU_SOURCE_DIR}/i18n/collationruleparser.cpp +${ICU_SOURCE_DIR}/i18n/collationbuilder.cpp +${ICU_SOURCE_DIR}/i18n/collationfastlatinbuilder.cpp +${ICU_SOURCE_DIR}/i18n/listformatter.cpp +${ICU_SOURCE_DIR}/i18n/ulistformatter.cpp +${ICU_SOURCE_DIR}/i18n/strmatch.cpp +${ICU_SOURCE_DIR}/i18n/usearch.cpp +${ICU_SOURCE_DIR}/i18n/search.cpp +${ICU_SOURCE_DIR}/i18n/stsearch.cpp +${ICU_SOURCE_DIR}/i18n/translit.cpp +${ICU_SOURCE_DIR}/i18n/utrans.cpp +${ICU_SOURCE_DIR}/i18n/esctrn.cpp +${ICU_SOURCE_DIR}/i18n/unesctrn.cpp +${ICU_SOURCE_DIR}/i18n/funcrepl.cpp +${ICU_SOURCE_DIR}/i18n/strrepl.cpp +${ICU_SOURCE_DIR}/i18n/tridpars.cpp +${ICU_SOURCE_DIR}/i18n/cpdtrans.cpp +${ICU_SOURCE_DIR}/i18n/rbt.cpp +${ICU_SOURCE_DIR}/i18n/rbt_data.cpp +${ICU_SOURCE_DIR}/i18n/rbt_pars.cpp +${ICU_SOURCE_DIR}/i18n/rbt_rule.cpp +${ICU_SOURCE_DIR}/i18n/rbt_set.cpp +${ICU_SOURCE_DIR}/i18n/nultrans.cpp +${ICU_SOURCE_DIR}/i18n/remtrans.cpp +${ICU_SOURCE_DIR}/i18n/casetrn.cpp +${ICU_SOURCE_DIR}/i18n/titletrn.cpp +${ICU_SOURCE_DIR}/i18n/tolowtrn.cpp +${ICU_SOURCE_DIR}/i18n/toupptrn.cpp +${ICU_SOURCE_DIR}/i18n/anytrans.cpp +${ICU_SOURCE_DIR}/i18n/name2uni.cpp +${ICU_SOURCE_DIR}/i18n/uni2name.cpp +${ICU_SOURCE_DIR}/i18n/nortrans.cpp +${ICU_SOURCE_DIR}/i18n/quant.cpp +${ICU_SOURCE_DIR}/i18n/transreg.cpp +${ICU_SOURCE_DIR}/i18n/brktrans.cpp +${ICU_SOURCE_DIR}/i18n/regexcmp.cpp +${ICU_SOURCE_DIR}/i18n/rematch.cpp +${ICU_SOURCE_DIR}/i18n/repattrn.cpp +${ICU_SOURCE_DIR}/i18n/regexst.cpp +${ICU_SOURCE_DIR}/i18n/regextxt.cpp +${ICU_SOURCE_DIR}/i18n/regeximp.cpp +${ICU_SOURCE_DIR}/i18n/uregex.cpp +${ICU_SOURCE_DIR}/i18n/uregexc.cpp +${ICU_SOURCE_DIR}/i18n/ulocdata.cpp +${ICU_SOURCE_DIR}/i18n/measfmt.cpp +${ICU_SOURCE_DIR}/i18n/currfmt.cpp +${ICU_SOURCE_DIR}/i18n/curramt.cpp +${ICU_SOURCE_DIR}/i18n/currunit.cpp +${ICU_SOURCE_DIR}/i18n/measure.cpp +${ICU_SOURCE_DIR}/i18n/utmscale.cpp +${ICU_SOURCE_DIR}/i18n/csdetect.cpp +${ICU_SOURCE_DIR}/i18n/csmatch.cpp +${ICU_SOURCE_DIR}/i18n/csr2022.cpp +${ICU_SOURCE_DIR}/i18n/csrecog.cpp +${ICU_SOURCE_DIR}/i18n/csrmbcs.cpp +${ICU_SOURCE_DIR}/i18n/csrsbcs.cpp +${ICU_SOURCE_DIR}/i18n/csrucode.cpp +${ICU_SOURCE_DIR}/i18n/csrutf8.cpp +${ICU_SOURCE_DIR}/i18n/inputext.cpp +${ICU_SOURCE_DIR}/i18n/wintzimpl.cpp +${ICU_SOURCE_DIR}/i18n/windtfmt.cpp +${ICU_SOURCE_DIR}/i18n/winnmfmt.cpp +${ICU_SOURCE_DIR}/i18n/basictz.cpp +${ICU_SOURCE_DIR}/i18n/dtrule.cpp +${ICU_SOURCE_DIR}/i18n/rbtz.cpp +${ICU_SOURCE_DIR}/i18n/tzrule.cpp +${ICU_SOURCE_DIR}/i18n/tztrans.cpp +${ICU_SOURCE_DIR}/i18n/vtzone.cpp +${ICU_SOURCE_DIR}/i18n/zonemeta.cpp +${ICU_SOURCE_DIR}/i18n/standardplural.cpp +${ICU_SOURCE_DIR}/i18n/upluralrules.cpp +${ICU_SOURCE_DIR}/i18n/plurrule.cpp +${ICU_SOURCE_DIR}/i18n/plurfmt.cpp +${ICU_SOURCE_DIR}/i18n/selfmt.cpp +${ICU_SOURCE_DIR}/i18n/dtitvfmt.cpp +${ICU_SOURCE_DIR}/i18n/dtitvinf.cpp +${ICU_SOURCE_DIR}/i18n/udateintervalformat.cpp +${ICU_SOURCE_DIR}/i18n/tmunit.cpp +${ICU_SOURCE_DIR}/i18n/tmutamt.cpp +${ICU_SOURCE_DIR}/i18n/tmutfmt.cpp +${ICU_SOURCE_DIR}/i18n/currpinf.cpp +${ICU_SOURCE_DIR}/i18n/uspoof.cpp +${ICU_SOURCE_DIR}/i18n/uspoof_impl.cpp +${ICU_SOURCE_DIR}/i18n/uspoof_build.cpp +${ICU_SOURCE_DIR}/i18n/uspoof_conf.cpp +${ICU_SOURCE_DIR}/i18n/smpdtfst.cpp +${ICU_SOURCE_DIR}/i18n/ztrans.cpp +${ICU_SOURCE_DIR}/i18n/zrule.cpp +${ICU_SOURCE_DIR}/i18n/vzone.cpp +${ICU_SOURCE_DIR}/i18n/fphdlimp.cpp +${ICU_SOURCE_DIR}/i18n/fpositer.cpp +${ICU_SOURCE_DIR}/i18n/ufieldpositer.cpp +${ICU_SOURCE_DIR}/i18n/decNumber.cpp +${ICU_SOURCE_DIR}/i18n/decContext.cpp +${ICU_SOURCE_DIR}/i18n/alphaindex.cpp +${ICU_SOURCE_DIR}/i18n/tznames.cpp +${ICU_SOURCE_DIR}/i18n/tznames_impl.cpp +${ICU_SOURCE_DIR}/i18n/tzgnames.cpp +${ICU_SOURCE_DIR}/i18n/tzfmt.cpp +${ICU_SOURCE_DIR}/i18n/compactdecimalformat.cpp +${ICU_SOURCE_DIR}/i18n/gender.cpp +${ICU_SOURCE_DIR}/i18n/region.cpp +${ICU_SOURCE_DIR}/i18n/scriptset.cpp +${ICU_SOURCE_DIR}/i18n/uregion.cpp +${ICU_SOURCE_DIR}/i18n/reldatefmt.cpp +${ICU_SOURCE_DIR}/i18n/quantityformatter.cpp +${ICU_SOURCE_DIR}/i18n/measunit.cpp +${ICU_SOURCE_DIR}/i18n/sharedbreakiterator.cpp +${ICU_SOURCE_DIR}/i18n/scientificnumberformatter.cpp +${ICU_SOURCE_DIR}/i18n/dayperiodrules.cpp +${ICU_SOURCE_DIR}/i18n/nounit.cpp +${ICU_SOURCE_DIR}/i18n/number_affixutils.cpp +${ICU_SOURCE_DIR}/i18n/number_compact.cpp +${ICU_SOURCE_DIR}/i18n/number_decimalquantity.cpp +${ICU_SOURCE_DIR}/i18n/number_decimfmtprops.cpp +${ICU_SOURCE_DIR}/i18n/number_fluent.cpp +${ICU_SOURCE_DIR}/i18n/number_formatimpl.cpp +${ICU_SOURCE_DIR}/i18n/number_grouping.cpp +${ICU_SOURCE_DIR}/i18n/number_integerwidth.cpp +${ICU_SOURCE_DIR}/i18n/number_longnames.cpp +${ICU_SOURCE_DIR}/i18n/number_modifiers.cpp +${ICU_SOURCE_DIR}/i18n/number_notation.cpp +${ICU_SOURCE_DIR}/i18n/number_output.cpp +${ICU_SOURCE_DIR}/i18n/number_padding.cpp +${ICU_SOURCE_DIR}/i18n/number_patternmodifier.cpp +${ICU_SOURCE_DIR}/i18n/number_patternstring.cpp +${ICU_SOURCE_DIR}/i18n/number_rounding.cpp +${ICU_SOURCE_DIR}/i18n/number_scientific.cpp +${ICU_SOURCE_DIR}/i18n/number_utils.cpp +${ICU_SOURCE_DIR}/i18n/number_asformat.cpp +${ICU_SOURCE_DIR}/i18n/number_mapper.cpp +${ICU_SOURCE_DIR}/i18n/number_multiplier.cpp +${ICU_SOURCE_DIR}/i18n/number_currencysymbols.cpp +${ICU_SOURCE_DIR}/i18n/number_skeletons.cpp +${ICU_SOURCE_DIR}/i18n/number_capi.cpp +${ICU_SOURCE_DIR}/i18n/double-conversion-string-to-double.cpp +${ICU_SOURCE_DIR}/i18n/double-conversion-double-to-string.cpp +${ICU_SOURCE_DIR}/i18n/double-conversion-bignum-dtoa.cpp +${ICU_SOURCE_DIR}/i18n/double-conversion-bignum.cpp +${ICU_SOURCE_DIR}/i18n/double-conversion-cached-powers.cpp +${ICU_SOURCE_DIR}/i18n/double-conversion-fast-dtoa.cpp +${ICU_SOURCE_DIR}/i18n/double-conversion-strtod.cpp +${ICU_SOURCE_DIR}/i18n/string_segment.cpp +${ICU_SOURCE_DIR}/i18n/numparse_parsednumber.cpp +${ICU_SOURCE_DIR}/i18n/numparse_impl.cpp +${ICU_SOURCE_DIR}/i18n/numparse_symbols.cpp +${ICU_SOURCE_DIR}/i18n/numparse_decimal.cpp +${ICU_SOURCE_DIR}/i18n/numparse_scientific.cpp +${ICU_SOURCE_DIR}/i18n/numparse_currency.cpp +${ICU_SOURCE_DIR}/i18n/numparse_affixes.cpp +${ICU_SOURCE_DIR}/i18n/numparse_compositions.cpp +${ICU_SOURCE_DIR}/i18n/numparse_validators.cpp +${ICU_SOURCE_DIR}/i18n/numrange_fluent.cpp +${ICU_SOURCE_DIR}/i18n/numrange_impl.cpp +${ICU_SOURCE_DIR}/i18n/erarules.cpp +${ICU_SOURCE_DIR}/i18n/formattedvalue.cpp +${ICU_SOURCE_DIR}/i18n/formattedval_iterimpl.cpp +${ICU_SOURCE_DIR}/i18n/formattedval_sbimpl.cpp +${ICU_SOURCE_DIR}/i18n/formatted_string_builder.cpp) + +enable_language(ASM) +set(ICUDATA_SOURCES ${ICUDATA_SOURCE_DIR}/icudt66l_dat.S) + +# Note that we don't like any kind of binary plugins (because of runtime dependencies, vulnerabilities, ABI incompatibilities). +add_definitions(-D_REENTRANT -DU_HAVE_ELF_H=1 -DU_HAVE_STRTOD_L=1 -DU_HAVE_XLOCALE_H=0 -DDEFAULT_ICU_PLUGINS="/dev/null") + +add_library(icuuc ${ICUUC_SOURCES}) +add_library(icui18n ${ICUI18N_SOURCES}) +add_library(icudata ${ICUDATA_SOURCES}) + +target_link_libraries(icuuc icudata) +target_link_libraries(icui18n icuuc) + +target_include_directories(icuuc SYSTEM PUBLIC ${ICU_SOURCE_DIR}/common/) +target_include_directories(icui18n SYSTEM PUBLIC ${ICU_SOURCE_DIR}/i18n/) + +target_compile_definitions(icuuc PRIVATE -DU_COMMON_IMPLEMENTATION) +target_compile_definitions(icui18n PRIVATE -DU_I18N_IMPLEMENTATION) + +if (COMPILER_CLANG) + target_compile_options(icudata PRIVATE -Wno-unused-command-line-argument) +endif () diff --git a/contrib/icudata b/contrib/icudata new file mode 160000 index 00000000000..f020820388e --- /dev/null +++ b/contrib/icudata @@ -0,0 +1 @@ +Subproject commit f020820388e3faafb44cc643574a2d563dfde572 diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h index 0e1cf49ad97..9c46a3a9320 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h +++ b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h @@ -297,7 +297,7 @@ * MADV_FREE, though typically with higher * system overhead. */ -#define JEMALLOC_PURGE_MADVISE_FREE +// #define JEMALLOC_PURGE_MADVISE_FREE #define JEMALLOC_PURGE_MADVISE_DONTNEED #define JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS diff --git a/contrib/libbtrie/CMakeLists.txt b/contrib/libbtrie/CMakeLists.txt index f590520c416..2b0c8e3fd75 100644 --- a/contrib/libbtrie/CMakeLists.txt +++ b/contrib/libbtrie/CMakeLists.txt @@ -3,9 +3,4 @@ add_library(btrie include/btrie.h ) -target_include_directories (btrie PUBLIC include) - -if (ENABLE_TESTS) - add_executable(test_btrie test/test_btrie.c) - target_link_libraries(test_btrie btrie) -endif () +target_include_directories (btrie SYSTEM PUBLIC include) diff --git a/contrib/libc-headers b/contrib/libc-headers new file mode 160000 index 00000000000..cd82fd9d8ee --- /dev/null +++ b/contrib/libc-headers @@ -0,0 +1 @@ +Subproject commit cd82fd9d8eefe50a47a0adf7c617c3ea7d558d11 diff --git a/contrib/libcxx b/contrib/libcxx index 9807685d51d..f7c63235238 160000 --- a/contrib/libcxx +++ b/contrib/libcxx @@ -1 +1 @@ -Subproject commit 9807685d51db467e097ad5eb8d5c2c16922794b2 +Subproject commit f7c63235238a71b7e0563fab8c7c5ec1b54831f6 diff --git a/contrib/libcxx-cmake/CMakeLists.txt b/contrib/libcxx-cmake/CMakeLists.txt index 82d11e3e32d..ee5fe625079 100644 --- a/contrib/libcxx-cmake/CMakeLists.txt +++ b/contrib/libcxx-cmake/CMakeLists.txt @@ -1,41 +1,45 @@ +include(CheckCXXCompilerFlag) + set(LIBCXX_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxx) set(SRCS -${LIBCXX_SOURCE_DIR}/src/optional.cpp -${LIBCXX_SOURCE_DIR}/src/variant.cpp -${LIBCXX_SOURCE_DIR}/src/chrono.cpp -${LIBCXX_SOURCE_DIR}/src/thread.cpp -${LIBCXX_SOURCE_DIR}/src/experimental/memory_resource.cpp -${LIBCXX_SOURCE_DIR}/src/iostream.cpp -${LIBCXX_SOURCE_DIR}/src/strstream.cpp -${LIBCXX_SOURCE_DIR}/src/ios.cpp -${LIBCXX_SOURCE_DIR}/src/future.cpp -${LIBCXX_SOURCE_DIR}/src/shared_mutex.cpp -${LIBCXX_SOURCE_DIR}/src/condition_variable.cpp -${LIBCXX_SOURCE_DIR}/src/hash.cpp -${LIBCXX_SOURCE_DIR}/src/string.cpp -${LIBCXX_SOURCE_DIR}/src/debug.cpp -${LIBCXX_SOURCE_DIR}/src/stdexcept.cpp -${LIBCXX_SOURCE_DIR}/src/utility.cpp -${LIBCXX_SOURCE_DIR}/src/any.cpp -${LIBCXX_SOURCE_DIR}/src/exception.cpp -${LIBCXX_SOURCE_DIR}/src/memory.cpp -${LIBCXX_SOURCE_DIR}/src/new.cpp -${LIBCXX_SOURCE_DIR}/src/valarray.cpp -${LIBCXX_SOURCE_DIR}/src/vector.cpp ${LIBCXX_SOURCE_DIR}/src/algorithm.cpp -${LIBCXX_SOURCE_DIR}/src/functional.cpp -${LIBCXX_SOURCE_DIR}/src/regex.cpp +${LIBCXX_SOURCE_DIR}/src/any.cpp ${LIBCXX_SOURCE_DIR}/src/bind.cpp -${LIBCXX_SOURCE_DIR}/src/mutex.cpp ${LIBCXX_SOURCE_DIR}/src/charconv.cpp -${LIBCXX_SOURCE_DIR}/src/typeinfo.cpp -${LIBCXX_SOURCE_DIR}/src/locale.cpp -${LIBCXX_SOURCE_DIR}/src/filesystem/operations.cpp -${LIBCXX_SOURCE_DIR}/src/filesystem/int128_builtins.cpp +${LIBCXX_SOURCE_DIR}/src/chrono.cpp +${LIBCXX_SOURCE_DIR}/src/condition_variable.cpp +${LIBCXX_SOURCE_DIR}/src/condition_variable_destructor.cpp +${LIBCXX_SOURCE_DIR}/src/debug.cpp +${LIBCXX_SOURCE_DIR}/src/exception.cpp +${LIBCXX_SOURCE_DIR}/src/experimental/memory_resource.cpp ${LIBCXX_SOURCE_DIR}/src/filesystem/directory_iterator.cpp -${LIBCXX_SOURCE_DIR}/src/system_error.cpp +${LIBCXX_SOURCE_DIR}/src/filesystem/int128_builtins.cpp +${LIBCXX_SOURCE_DIR}/src/filesystem/operations.cpp +${LIBCXX_SOURCE_DIR}/src/functional.cpp +${LIBCXX_SOURCE_DIR}/src/future.cpp +${LIBCXX_SOURCE_DIR}/src/hash.cpp +${LIBCXX_SOURCE_DIR}/src/ios.cpp +${LIBCXX_SOURCE_DIR}/src/iostream.cpp +${LIBCXX_SOURCE_DIR}/src/locale.cpp +${LIBCXX_SOURCE_DIR}/src/memory.cpp +${LIBCXX_SOURCE_DIR}/src/mutex.cpp +${LIBCXX_SOURCE_DIR}/src/mutex_destructor.cpp +${LIBCXX_SOURCE_DIR}/src/new.cpp +${LIBCXX_SOURCE_DIR}/src/optional.cpp ${LIBCXX_SOURCE_DIR}/src/random.cpp +${LIBCXX_SOURCE_DIR}/src/regex.cpp +${LIBCXX_SOURCE_DIR}/src/shared_mutex.cpp +${LIBCXX_SOURCE_DIR}/src/stdexcept.cpp +${LIBCXX_SOURCE_DIR}/src/string.cpp +${LIBCXX_SOURCE_DIR}/src/strstream.cpp +${LIBCXX_SOURCE_DIR}/src/system_error.cpp +${LIBCXX_SOURCE_DIR}/src/thread.cpp +${LIBCXX_SOURCE_DIR}/src/typeinfo.cpp +${LIBCXX_SOURCE_DIR}/src/utility.cpp +${LIBCXX_SOURCE_DIR}/src/valarray.cpp +${LIBCXX_SOURCE_DIR}/src/variant.cpp +${LIBCXX_SOURCE_DIR}/src/vector.cpp ) add_library(cxx ${SRCS}) @@ -43,8 +47,15 @@ add_library(cxx ${SRCS}) target_include_directories(cxx SYSTEM BEFORE PUBLIC $) target_compile_definitions(cxx PRIVATE -D_LIBCPP_BUILDING_LIBRARY -DLIBCXX_BUILDING_LIBCXXABI) -target_compile_options(cxx PUBLIC -nostdinc++ -Wno-reserved-id-macro) -if (OS_DARWIN AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9) +target_compile_options(cxx PUBLIC $<$:-nostdinc++>) + +check_cxx_compiler_flag(-Wreserved-id-macro HAVE_WARNING_RESERVED_ID_MACRO) +if (HAVE_WARNING_RESERVED_ID_MACRO) + target_compile_options(cxx PUBLIC -Wno-reserved-id-macro) +endif () + +check_cxx_compiler_flag(-Wctad-maybe-unsupported HAVE_WARNING_CTAD_MAYBE_UNSUPPORTED) +if (HAVE_WARNING_CTAD_MAYBE_UNSUPPORTED) target_compile_options(cxx PUBLIC -Wno-ctad-maybe-unsupported) endif () diff --git a/contrib/libcxxabi b/contrib/libcxxabi index d56efcc7a52..c26cf36f838 160000 --- a/contrib/libcxxabi +++ b/contrib/libcxxabi @@ -1 +1 @@ -Subproject commit d56efcc7a52739518dbe7df9e743073e00951fa1 +Subproject commit c26cf36f8387c5edf2cabb4a630f0975c35aa9fb diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index 28f32e948b2..e68f0bacf46 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -182,6 +182,9 @@ set(SRCS ${HDFS3_SOURCE_DIR}/common/FileWrapper.h ) +# old kernels (< 3.17) doens't have SYS_getrandom. Always use POSIX implementation to have better compatibility +set_source_files_properties(${HDFS3_SOURCE_DIR}/rpc/RpcClient.cpp PROPERTIES COMPILE_FLAGS "-DBOOST_UUID_RANDOM_PROVIDER_FORCE_POSIX=1") + # target add_library(hdfs3 ${SRCS} ${PROTO_SOURCES} ${PROTO_HEADERS}) diff --git a/contrib/libtcmalloc/AUTHORS b/contrib/libtcmalloc/AUTHORS deleted file mode 100644 index 3995ed4cf57..00000000000 --- a/contrib/libtcmalloc/AUTHORS +++ /dev/null @@ -1,2 +0,0 @@ -google-perftools@googlegroups.com - diff --git a/contrib/libtcmalloc/CMakeLists.txt b/contrib/libtcmalloc/CMakeLists.txt deleted file mode 100644 index d7f52e1d384..00000000000 --- a/contrib/libtcmalloc/CMakeLists.txt +++ /dev/null @@ -1,80 +0,0 @@ -message (STATUS "Building: tcmalloc_minimal_internal") - -add_library (tcmalloc_minimal_internal -./src/malloc_hook.cc -./src/base/spinlock_internal.cc -./src/base/spinlock.cc -./src/base/dynamic_annotations.c -./src/base/linuxthreads.cc -./src/base/elf_mem_image.cc -./src/base/vdso_support.cc -./src/base/sysinfo.cc -./src/base/low_level_alloc.cc -./src/base/thread_lister.c -./src/base/logging.cc -./src/base/atomicops-internals-x86.cc -./src/memfs_malloc.cc -./src/tcmalloc.cc -./src/malloc_extension.cc -./src/thread_cache.cc -./src/symbolize.cc -./src/page_heap.cc -./src/maybe_threads.cc -./src/central_freelist.cc -./src/static_vars.cc -./src/sampler.cc -./src/internal_logging.cc -./src/system-alloc.cc -./src/span.cc -./src/common.cc -./src/stacktrace.cc -./src/stack_trace_table.cc -./src/heap-checker.cc -./src/heap-checker-bcad.cc -./src/heap-profile-table.cc -./src/raw_printer.cc -./src/memory_region_map.cc -) - - -target_compile_options (tcmalloc_minimal_internal - PRIVATE - -DNO_TCMALLOC_SAMPLES - -DNDEBUG - -DNO_FRAME_POINTER - -Wwrite-strings - -Wno-sign-compare - -Wno-unused-result - -Wno-deprecated-declarations - -Wno-unused-function - -Wno-unused-private-field - - PUBLIC - -fno-builtin-malloc - -fno-builtin-free - -fno-builtin-realloc - -fno-builtin-calloc - -fno-builtin-cfree - -fno-builtin-memalign - -fno-builtin-posix_memalign - -fno-builtin-valloc - -fno-builtin-pvalloc -) - -if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 3.9.1) - target_compile_options(tcmalloc_minimal_internal PUBLIC -Wno-dynamic-exception-spec ) -endif () - -if (CMAKE_SYSTEM MATCHES "FreeBSD" AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - target_compile_options(tcmalloc_minimal_internal PUBLIC -Wno-unused-but-set-variable) -endif () - -if (CMAKE_SYSTEM MATCHES "FreeBSD") - target_compile_definitions(tcmalloc_minimal_internal PUBLIC _GNU_SOURCE) -endif () - -target_include_directories (tcmalloc_minimal_internal PUBLIC include) -target_include_directories (tcmalloc_minimal_internal PRIVATE src) - -find_package (Threads) -target_link_libraries (tcmalloc_minimal_internal ${CMAKE_THREAD_LIBS_INIT}) diff --git a/contrib/libtcmalloc/COPYING b/contrib/libtcmalloc/COPYING deleted file mode 100644 index e4956cfd9fd..00000000000 --- a/contrib/libtcmalloc/COPYING +++ /dev/null @@ -1,28 +0,0 @@ -Copyright (c) 2005, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/contrib/libtcmalloc/README b/contrib/libtcmalloc/README deleted file mode 100644 index 8f7377c0b3b..00000000000 --- a/contrib/libtcmalloc/README +++ /dev/null @@ -1,8 +0,0 @@ -https://github.com/gperftools/gperftools/commit/dde32f8bbc95312379f9f5a651799815bb6327c5 - -Several modifications: -1. Disabled TCMALLOC_AGGRESSIVE_DECOMMIT by default. It is important. -2. Using only files for tcmalloc_minimal build (./configure --enable-minimal). -3. Using some compiler flags from project. -4. Removed warning about unused variable when build with NDEBUG (by default). -5. Including config.h with relative path. diff --git a/contrib/libtcmalloc/include/gperftools/heap-checker.h b/contrib/libtcmalloc/include/gperftools/heap-checker.h deleted file mode 100644 index 5a87d8da7f7..00000000000 --- a/contrib/libtcmalloc/include/gperftools/heap-checker.h +++ /dev/null @@ -1,422 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Maxim Lifantsev (with design ideas by Sanjay Ghemawat) -// -// -// Module for detecing heap (memory) leaks. -// -// For full(er) information, see doc/heap_checker.html -// -// This module can be linked into programs with -// no slowdown caused by this unless you activate the leak-checker: -// -// 1. Set the environment variable HEAPCHEK to _type_ before -// running the program. -// -// _type_ is usually "normal" but can also be "minimal", "strict", or -// "draconian". (See the html file for other options, like 'local'.) -// -// After that, just run your binary. If the heap-checker detects -// a memory leak at program-exit, it will print instructions on how -// to track down the leak. - -#ifndef BASE_HEAP_CHECKER_H_ -#define BASE_HEAP_CHECKER_H_ - -#include // for size_t -// I can't #include config.h in this public API file, but I should -// really use configure (and make malloc_extension.h a .in file) to -// figure out if the system has stdint.h or not. But I'm lazy, so -// for now I'm assuming it's a problem only with MSVC. -#ifndef _MSC_VER -#include // for uintptr_t -#endif -#include // for va_list -#include - -// Annoying stuff for windows -- makes sure clients can import these functions -#ifndef PERFTOOLS_DLL_DECL -# ifdef _WIN32 -# define PERFTOOLS_DLL_DECL __declspec(dllimport) -# else -# define PERFTOOLS_DLL_DECL -# endif -#endif - - -// The class is thread-safe with respect to all the provided static methods, -// as well as HeapLeakChecker objects: they can be accessed by multiple threads. -class PERFTOOLS_DLL_DECL HeapLeakChecker { - public: - - // ----------------------------------------------------------------------- // - // Static functions for working with (whole-program) leak checking. - - // If heap leak checking is currently active in some mode - // e.g. if leak checking was started (and is still active now) - // due to HEAPCHECK=... defined in the environment. - // The return value reflects iff HeapLeakChecker objects manually - // constructed right now will be doing leak checking or nothing. - // Note that we can go from active to inactive state during InitGoogle() - // if FLAGS_heap_check gets set to "" by some code before/during InitGoogle(). - static bool IsActive(); - - // Return pointer to the whole-program checker if it has been created - // and NULL otherwise. - // Once GlobalChecker() returns non-NULL that object will not disappear and - // will be returned by all later GlobalChecker calls. - // This is mainly to access BytesLeaked() and ObjectsLeaked() (see below) - // for the whole-program checker after one calls NoGlobalLeaks() - // or similar and gets false. - static HeapLeakChecker* GlobalChecker(); - - // Do whole-program leak check now (if it was activated for this binary); - // return false only if it was activated and has failed. - // The mode of the check is controlled by the command-line flags. - // This method can be called repeatedly. - // Things like GlobalChecker()->SameHeap() can also be called explicitly - // to do the desired flavor of the check. - static bool NoGlobalLeaks(); - - // If whole-program checker if active, - // cancel its automatic execution after main() exits. - // This requires that some leak check (e.g. NoGlobalLeaks()) - // has been called at least once on the whole-program checker. - static void CancelGlobalCheck(); - - // ----------------------------------------------------------------------- // - // Non-static functions for starting and doing leak checking. - - // Start checking and name the leak check performed. - // The name is used in naming dumped profiles - // and needs to be unique only within your binary. - // It must also be a string that can be a part of a file name, - // in particular not contain path expressions. - explicit HeapLeakChecker(const char *name); - - // Destructor (verifies that some *NoLeaks or *SameHeap method - // has been called at least once). - ~HeapLeakChecker(); - - // These used to be different but are all the same now: they return - // true iff all memory allocated since this HeapLeakChecker object - // was constructor is still reachable from global state. - // - // Because we fork to convert addresses to symbol-names, and forking - // is not thread-safe, and we may be called in a threaded context, - // we do not try to symbolize addresses when called manually. - bool NoLeaks() { return DoNoLeaks(DO_NOT_SYMBOLIZE); } - - // These forms are obsolete; use NoLeaks() instead. - // TODO(csilvers): mark as DEPRECATED. - bool QuickNoLeaks() { return NoLeaks(); } - bool BriefNoLeaks() { return NoLeaks(); } - bool SameHeap() { return NoLeaks(); } - bool QuickSameHeap() { return NoLeaks(); } - bool BriefSameHeap() { return NoLeaks(); } - - // Detailed information about the number of leaked bytes and objects - // (both of these can be negative as well). - // These are available only after a *SameHeap or *NoLeaks - // method has been called. - // Note that it's possible for both of these to be zero - // while SameHeap() or NoLeaks() returned false in case - // of a heap state change that is significant - // but preserves the byte and object counts. - ssize_t BytesLeaked() const; - ssize_t ObjectsLeaked() const; - - // ----------------------------------------------------------------------- // - // Static helpers to make us ignore certain leaks. - - // Scoped helper class. Should be allocated on the stack inside a - // block of code. Any heap allocations done in the code block - // covered by the scoped object (including in nested function calls - // done by the code block) will not be reported as leaks. This is - // the recommended replacement for the GetDisableChecksStart() and - // DisableChecksToHereFrom() routines below. - // - // Example: - // void Foo() { - // HeapLeakChecker::Disabler disabler; - // ... code that allocates objects whose leaks should be ignored ... - // } - // - // REQUIRES: Destructor runs in same thread as constructor - class Disabler { - public: - Disabler(); - ~Disabler(); - private: - Disabler(const Disabler&); // disallow copy - void operator=(const Disabler&); // and assign - }; - - // Ignore an object located at 'ptr' (can go at the start or into the object) - // as well as all heap objects (transitively) referenced from it for the - // purposes of heap leak checking. Returns 'ptr' so that one can write - // static T* obj = IgnoreObject(new T(...)); - // - // If 'ptr' does not point to an active allocated object at the time of this - // call, it is ignored; but if it does, the object must not get deleted from - // the heap later on. - // - // See also HiddenPointer, below, if you need to prevent a pointer from - // being traversed by the heap checker but do not wish to transitively - // whitelist objects referenced through it. - template - static T* IgnoreObject(T* ptr) { - DoIgnoreObject(static_cast(const_cast(ptr))); - return ptr; - } - - // Undo what an earlier IgnoreObject() call promised and asked to do. - // At the time of this call 'ptr' must point at or inside of an active - // allocated object which was previously registered with IgnoreObject(). - static void UnIgnoreObject(const void* ptr); - - // ----------------------------------------------------------------------- // - // Internal types defined in .cc - - class Allocator; - struct RangeValue; - - private: - - // ----------------------------------------------------------------------- // - // Various helpers - - // Create the name of the heap profile file. - // Should be deleted via Allocator::Free(). - char* MakeProfileNameLocked(); - - // Helper for constructors - void Create(const char *name, bool make_start_snapshot); - - enum ShouldSymbolize { SYMBOLIZE, DO_NOT_SYMBOLIZE }; - - // Helper for *NoLeaks and *SameHeap - bool DoNoLeaks(ShouldSymbolize should_symbolize); - - // Helper for NoGlobalLeaks, also called by the global destructor. - static bool NoGlobalLeaksMaybeSymbolize(ShouldSymbolize should_symbolize); - - // These used to be public, but they are now deprecated. - // Will remove entirely when all internal uses are fixed. - // In the meantime, use friendship so the unittest can still test them. - static void* GetDisableChecksStart(); - static void DisableChecksToHereFrom(const void* start_address); - static void DisableChecksIn(const char* pattern); - friend void RangeDisabledLeaks(); - friend void NamedTwoDisabledLeaks(); - friend void* RunNamedDisabledLeaks(void*); - friend void TestHeapLeakCheckerNamedDisabling(); - - // Actually implements IgnoreObject(). - static void DoIgnoreObject(const void* ptr); - - // Disable checks based on stack trace entry at a depth <= - // max_depth. Used to hide allocations done inside some special - // libraries. - static void DisableChecksFromToLocked(const void* start_address, - const void* end_address, - int max_depth); - - // Helper for DoNoLeaks to ignore all objects reachable from all live data - static void IgnoreAllLiveObjectsLocked(const void* self_stack_top); - - // Callback we pass to TCMalloc_ListAllProcessThreads (see thread_lister.h) - // that is invoked when all threads of our process are found and stopped. - // The call back does the things needed to ignore live data reachable from - // thread stacks and registers for all our threads - // as well as do other global-live-data ignoring - // (via IgnoreNonThreadLiveObjectsLocked) - // during the quiet state of all threads being stopped. - // For the argument meaning see the comment by TCMalloc_ListAllProcessThreads. - // Here we only use num_threads and thread_pids, that TCMalloc_ListAllProcessThreads - // fills for us with the number and pids of all the threads of our process - // it found and attached to. - static int IgnoreLiveThreadsLocked(void* parameter, - int num_threads, - pid_t* thread_pids, - va_list ap); - - // Helper for IgnoreAllLiveObjectsLocked and IgnoreLiveThreadsLocked - // that we prefer to execute from IgnoreLiveThreadsLocked - // while all threads are stopped. - // This helper does live object discovery and ignoring - // for all objects that are reachable from everything - // not related to thread stacks and registers. - static void IgnoreNonThreadLiveObjectsLocked(); - - // Helper for IgnoreNonThreadLiveObjectsLocked and IgnoreLiveThreadsLocked - // to discover and ignore all heap objects - // reachable from currently considered live objects - // (live_objects static global variable in out .cc file). - // "name", "name2" are two strings that we print one after another - // in a debug message to describe what kind of live object sources - // are being used. - static void IgnoreLiveObjectsLocked(const char* name, const char* name2); - - // Do the overall whole-program heap leak check if needed; - // returns true when did the leak check. - static bool DoMainHeapCheck(); - - // Type of task for UseProcMapsLocked - enum ProcMapsTask { - RECORD_GLOBAL_DATA, - DISABLE_LIBRARY_ALLOCS - }; - - // Success/Error Return codes for UseProcMapsLocked. - enum ProcMapsResult { - PROC_MAPS_USED, - CANT_OPEN_PROC_MAPS, - NO_SHARED_LIBS_IN_PROC_MAPS - }; - - // Read /proc/self/maps, parse it, and do the 'proc_maps_task' for each line. - static ProcMapsResult UseProcMapsLocked(ProcMapsTask proc_maps_task); - - // A ProcMapsTask to disable allocations from 'library' - // that is mapped to [start_address..end_address) - // (only if library is a certain system library). - static void DisableLibraryAllocsLocked(const char* library, - uintptr_t start_address, - uintptr_t end_address); - - // Return true iff "*ptr" points to a heap object - // ("*ptr" can point at the start or inside of a heap object - // so that this works e.g. for pointers to C++ arrays, C++ strings, - // multiple-inherited objects, or pointers to members). - // We also fill *object_size for this object then - // and we move "*ptr" to point to the very start of the heap object. - static inline bool HaveOnHeapLocked(const void** ptr, size_t* object_size); - - // Helper to shutdown heap leak checker when it's not needed - // or can't function properly. - static void TurnItselfOffLocked(); - - // Internally-used c-tor to start whole-executable checking. - HeapLeakChecker(); - - // ----------------------------------------------------------------------- // - // Friends and externally accessed helpers. - - // Helper for VerifyHeapProfileTableStackGet in the unittest - // to get the recorded allocation caller for ptr, - // which must be a heap object. - static const void* GetAllocCaller(void* ptr); - friend void VerifyHeapProfileTableStackGet(); - - // This gets to execute before constructors for all global objects - static void BeforeConstructorsLocked(); - friend void HeapLeakChecker_BeforeConstructors(); - - // This gets to execute after destructors for all global objects - friend void HeapLeakChecker_AfterDestructors(); - - // Full starting of recommended whole-program checking. - friend void HeapLeakChecker_InternalInitStart(); - - // Runs REGISTER_HEAPCHECK_CLEANUP cleanups and potentially - // calls DoMainHeapCheck - friend void HeapLeakChecker_RunHeapCleanups(); - - // ----------------------------------------------------------------------- // - // Member data. - - class SpinLock* lock_; // to make HeapLeakChecker objects thread-safe - const char* name_; // our remembered name (we own it) - // NULL means this leak checker is a noop - - // Snapshot taken when the checker was created. May be NULL - // for the global heap checker object. We use void* instead of - // HeapProfileTable::Snapshot* to avoid including heap-profile-table.h. - void* start_snapshot_; - - bool has_checked_; // if we have done the leak check, so these are ready: - ssize_t inuse_bytes_increase_; // bytes-in-use increase for this checker - ssize_t inuse_allocs_increase_; // allocations-in-use increase - // for this checker - bool keep_profiles_; // iff we should keep the heap profiles we've made - - // ----------------------------------------------------------------------- // - - // Disallow "evil" constructors. - HeapLeakChecker(const HeapLeakChecker&); - void operator=(const HeapLeakChecker&); -}; - - -// Holds a pointer that will not be traversed by the heap checker. -// Contrast with HeapLeakChecker::IgnoreObject(o), in which o and -// all objects reachable from o are ignored by the heap checker. -template -class HiddenPointer { - public: - explicit HiddenPointer(T* t) - : masked_t_(reinterpret_cast(t) ^ kHideMask) { - } - // Returns unhidden pointer. Be careful where you save the result. - T* get() const { return reinterpret_cast(masked_t_ ^ kHideMask); } - - private: - // Arbitrary value, but not such that xor'ing with it is likely - // to map one valid pointer to another valid pointer: - static const uintptr_t kHideMask = - static_cast(0xF03A5F7BF03A5F7Bll); - uintptr_t masked_t_; -}; - -// A class that exists solely to run its destructor. This class should not be -// used directly, but instead by the REGISTER_HEAPCHECK_CLEANUP macro below. -class PERFTOOLS_DLL_DECL HeapCleaner { - public: - typedef void (*void_function)(void); - HeapCleaner(void_function f); - static void RunHeapCleanups(); - private: - static std::vector* heap_cleanups_; -}; - -// A macro to declare module heap check cleanup tasks -// (they run only if we are doing heap leak checking.) -// 'body' should be the cleanup code to run. 'name' doesn't matter, -// but must be unique amongst all REGISTER_HEAPCHECK_CLEANUP calls. -#define REGISTER_HEAPCHECK_CLEANUP(name, body) \ - namespace { \ - void heapcheck_cleanup_##name() { body; } \ - static HeapCleaner heapcheck_cleaner_##name(&heapcheck_cleanup_##name); \ - } - -#endif // BASE_HEAP_CHECKER_H_ diff --git a/contrib/libtcmalloc/include/gperftools/heap-profiler.h b/contrib/libtcmalloc/include/gperftools/heap-profiler.h deleted file mode 100644 index 9b673645747..00000000000 --- a/contrib/libtcmalloc/include/gperftools/heap-profiler.h +++ /dev/null @@ -1,105 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2005, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Sanjay Ghemawat - * - * Module for heap-profiling. - * - * For full(er) information, see doc/heapprofile.html - * - * This module can be linked into your program with - * no slowdown caused by this unless you activate the profiler - * using one of the following methods: - * - * 1. Before starting the program, set the environment variable - * "HEAPPROFILE" to be the name of the file to which the profile - * data should be written. - * - * 2. Programmatically, start and stop the profiler using the - * routines "HeapProfilerStart(filename)" and "HeapProfilerStop()". - * - */ - -#ifndef BASE_HEAP_PROFILER_H_ -#define BASE_HEAP_PROFILER_H_ - -#include - -/* Annoying stuff for windows; makes sure clients can import these functions */ -#ifndef PERFTOOLS_DLL_DECL -# ifdef _WIN32 -# define PERFTOOLS_DLL_DECL __declspec(dllimport) -# else -# define PERFTOOLS_DLL_DECL -# endif -#endif - -/* All this code should be usable from within C apps. */ -#ifdef __cplusplus -extern "C" { -#endif - -/* Start profiling and arrange to write profile data to file names - * of the form: "prefix.0000", "prefix.0001", ... - */ -PERFTOOLS_DLL_DECL void HeapProfilerStart(const char* prefix); - -/* Returns non-zero if we are currently profiling the heap. (Returns - * an int rather than a bool so it's usable from C.) This is true - * between calls to HeapProfilerStart() and HeapProfilerStop(), and - * also if the program has been run with HEAPPROFILER, or some other - * way to turn on whole-program profiling. - */ -int IsHeapProfilerRunning(); - -/* Stop heap profiling. Can be restarted again with HeapProfilerStart(), - * but the currently accumulated profiling information will be cleared. - */ -PERFTOOLS_DLL_DECL void HeapProfilerStop(); - -/* Dump a profile now - can be used for dumping at a hopefully - * quiescent state in your program, in order to more easily track down - * memory leaks. Will include the reason in the logged message - */ -PERFTOOLS_DLL_DECL void HeapProfilerDump(const char *reason); - -/* Generate current heap profiling information. - * Returns an empty string when heap profiling is not active. - * The returned pointer is a '\0'-terminated string allocated using malloc() - * and should be free()-ed as soon as the caller does not need it anymore. - */ -PERFTOOLS_DLL_DECL char* GetHeapProfile(); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif /* BASE_HEAP_PROFILER_H_ */ diff --git a/contrib/libtcmalloc/include/gperftools/malloc_extension.h b/contrib/libtcmalloc/include/gperftools/malloc_extension.h deleted file mode 100644 index 689b5f17cef..00000000000 --- a/contrib/libtcmalloc/include/gperftools/malloc_extension.h +++ /dev/null @@ -1,434 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat -// -// Extra extensions exported by some malloc implementations. These -// extensions are accessed through a virtual base class so an -// application can link against a malloc that does not implement these -// extensions, and it will get default versions that do nothing. -// -// NOTE FOR C USERS: If you wish to use this functionality from within -// a C program, see malloc_extension_c.h. - -#ifndef BASE_MALLOC_EXTENSION_H_ -#define BASE_MALLOC_EXTENSION_H_ - -#include -// I can't #include config.h in this public API file, but I should -// really use configure (and make malloc_extension.h a .in file) to -// figure out if the system has stdint.h or not. But I'm lazy, so -// for now I'm assuming it's a problem only with MSVC. -#ifndef _MSC_VER -#include -#endif -#include -#include - -// Annoying stuff for windows -- makes sure clients can import these functions -#ifndef PERFTOOLS_DLL_DECL -# ifdef _WIN32 -# define PERFTOOLS_DLL_DECL __declspec(dllimport) -# else -# define PERFTOOLS_DLL_DECL -# endif -#endif - -static const int kMallocHistogramSize = 64; - -// One day, we could support other types of writers (perhaps for C?) -typedef std::string MallocExtensionWriter; - -namespace base { -struct MallocRange; -} - -// Interface to a pluggable system allocator. -class PERFTOOLS_DLL_DECL SysAllocator { - public: - SysAllocator() { - } - virtual ~SysAllocator(); - - // Allocates "size"-byte of memory from system aligned with "alignment". - // Returns NULL if failed. Otherwise, the returned pointer p up to and - // including (p + actual_size -1) have been allocated. - virtual void* Alloc(size_t size, size_t *actual_size, size_t alignment) = 0; -}; - -// The default implementations of the following routines do nothing. -// All implementations should be thread-safe; the current one -// (TCMallocImplementation) is. -class PERFTOOLS_DLL_DECL MallocExtension { - public: - virtual ~MallocExtension(); - - // Call this very early in the program execution -- say, in a global - // constructor -- to set up parameters and state needed by all - // instrumented malloc implemenatations. One example: this routine - // sets environemnt variables to tell STL to use libc's malloc() - // instead of doing its own memory management. This is safe to call - // multiple times, as long as each time is before threads start up. - static void Initialize(); - - // See "verify_memory.h" to see what these routines do - virtual bool VerifyAllMemory(); - virtual bool VerifyNewMemory(const void* p); - virtual bool VerifyArrayNewMemory(const void* p); - virtual bool VerifyMallocMemory(const void* p); - virtual bool MallocMemoryStats(int* blocks, size_t* total, - int histogram[kMallocHistogramSize]); - - // Get a human readable description of the following malloc data structures. - // - Total inuse memory by application. - // - Free memory(thread, central and page heap), - // - Freelist of central cache, each class. - // - Page heap freelist. - // The state is stored as a null-terminated string - // in a prefix of "buffer[0,buffer_length-1]". - // REQUIRES: buffer_length > 0. - virtual void GetStats(char* buffer, int buffer_length); - - // Outputs to "writer" a sample of live objects and the stack traces - // that allocated these objects. The format of the returned output - // is equivalent to the output of the heap profiler and can - // therefore be passed to "pprof". This function is equivalent to - // ReadStackTraces. The main difference is that this function returns - // serialized data appropriately formatted for use by the pprof tool. - // NOTE: by default, tcmalloc does not do any heap sampling, and this - // function will always return an empty sample. To get useful - // data from GetHeapSample, you must also set the environment - // variable TCMALLOC_SAMPLE_PARAMETER to a value such as 524288. - virtual void GetHeapSample(MallocExtensionWriter* writer); - - // Outputs to "writer" the stack traces that caused growth in the - // address space size. The format of the returned output is - // equivalent to the output of the heap profiler and can therefore - // be passed to "pprof". This function is equivalent to - // ReadHeapGrowthStackTraces. The main difference is that this function - // returns serialized data appropriately formatted for use by the - // pprof tool. (This does not depend on, or require, - // TCMALLOC_SAMPLE_PARAMETER.) - virtual void GetHeapGrowthStacks(MallocExtensionWriter* writer); - - // Invokes func(arg, range) for every controlled memory - // range. *range is filled in with information about the range. - // - // This is a best-effort interface useful only for performance - // analysis. The implementation may not call func at all. - typedef void (RangeFunction)(void*, const base::MallocRange*); - virtual void Ranges(void* arg, RangeFunction func); - - // ------------------------------------------------------------------- - // Control operations for getting and setting malloc implementation - // specific parameters. Some currently useful properties: - // - // generic - // ------- - // "generic.current_allocated_bytes" - // Number of bytes currently allocated by application - // This property is not writable. - // - // "generic.heap_size" - // Number of bytes in the heap == - // current_allocated_bytes + - // fragmentation + - // freed memory regions - // This property is not writable. - // - // tcmalloc - // -------- - // "tcmalloc.max_total_thread_cache_bytes" - // Upper limit on total number of bytes stored across all - // per-thread caches. Default: 16MB. - // - // "tcmalloc.current_total_thread_cache_bytes" - // Number of bytes used across all thread caches. - // This property is not writable. - // - // "tcmalloc.central_cache_free_bytes" - // Number of free bytes in the central cache that have been - // assigned to size classes. They always count towards virtual - // memory usage, and unless the underlying memory is swapped out - // by the OS, they also count towards physical memory usage. - // This property is not writable. - // - // "tcmalloc.transfer_cache_free_bytes" - // Number of free bytes that are waiting to be transfered between - // the central cache and a thread cache. They always count - // towards virtual memory usage, and unless the underlying memory - // is swapped out by the OS, they also count towards physical - // memory usage. This property is not writable. - // - // "tcmalloc.thread_cache_free_bytes" - // Number of free bytes in thread caches. They always count - // towards virtual memory usage, and unless the underlying memory - // is swapped out by the OS, they also count towards physical - // memory usage. This property is not writable. - // - // "tcmalloc.pageheap_free_bytes" - // Number of bytes in free, mapped pages in page heap. These - // bytes can be used to fulfill allocation requests. They - // always count towards virtual memory usage, and unless the - // underlying memory is swapped out by the OS, they also count - // towards physical memory usage. This property is not writable. - // - // "tcmalloc.pageheap_unmapped_bytes" - // Number of bytes in free, unmapped pages in page heap. - // These are bytes that have been released back to the OS, - // possibly by one of the MallocExtension "Release" calls. - // They can be used to fulfill allocation requests, but - // typically incur a page fault. They always count towards - // virtual memory usage, and depending on the OS, typically - // do not count towards physical memory usage. This property - // is not writable. - // ------------------------------------------------------------------- - - // Get the named "property"'s value. Returns true if the property - // is known. Returns false if the property is not a valid property - // name for the current malloc implementation. - // REQUIRES: property != NULL; value != NULL - virtual bool GetNumericProperty(const char* property, size_t* value); - - // Set the named "property"'s value. Returns true if the property - // is known and writable. Returns false if the property is not a - // valid property name for the current malloc implementation, or - // is not writable. - // REQUIRES: property != NULL - virtual bool SetNumericProperty(const char* property, size_t value); - - // Mark the current thread as "idle". This routine may optionally - // be called by threads as a hint to the malloc implementation that - // any thread-specific resources should be released. Note: this may - // be an expensive routine, so it should not be called too often. - // - // Also, if the code that calls this routine will go to sleep for - // a while, it should take care to not allocate anything between - // the call to this routine and the beginning of the sleep. - // - // Most malloc implementations ignore this routine. - virtual void MarkThreadIdle(); - - // Mark the current thread as "busy". This routine should be - // called after MarkThreadIdle() if the thread will now do more - // work. If this method is not called, performance may suffer. - // - // Most malloc implementations ignore this routine. - virtual void MarkThreadBusy(); - - // Gets the system allocator used by the malloc extension instance. Returns - // NULL for malloc implementations that do not support pluggable system - // allocators. - virtual SysAllocator* GetSystemAllocator(); - - // Sets the system allocator to the specified. - // - // Users could register their own system allocators for malloc implementation - // that supports pluggable system allocators, such as TCMalloc, by doing: - // alloc = new MyOwnSysAllocator(); - // MallocExtension::instance()->SetSystemAllocator(alloc); - // It's up to users whether to fall back (recommended) to the default - // system allocator (use GetSystemAllocator() above) or not. The caller is - // responsible to any necessary locking. - // See tcmalloc/system-alloc.h for the interface and - // tcmalloc/memfs_malloc.cc for the examples. - // - // It's a no-op for malloc implementations that do not support pluggable - // system allocators. - virtual void SetSystemAllocator(SysAllocator *a); - - // Try to release num_bytes of free memory back to the operating - // system for reuse. Use this extension with caution -- to get this - // memory back may require faulting pages back in by the OS, and - // that may be slow. (Currently only implemented in tcmalloc.) - virtual void ReleaseToSystem(size_t num_bytes); - - // Same as ReleaseToSystem() but release as much memory as possible. - virtual void ReleaseFreeMemory(); - - // Sets the rate at which we release unused memory to the system. - // Zero means we never release memory back to the system. Increase - // this flag to return memory faster; decrease it to return memory - // slower. Reasonable rates are in the range [0,10]. (Currently - // only implemented in tcmalloc). - virtual void SetMemoryReleaseRate(double rate); - - // Gets the release rate. Returns a value < 0 if unknown. - virtual double GetMemoryReleaseRate(); - - // Returns the estimated number of bytes that will be allocated for - // a request of "size" bytes. This is an estimate: an allocation of - // SIZE bytes may reserve more bytes, but will never reserve less. - // (Currently only implemented in tcmalloc, other implementations - // always return SIZE.) - // This is equivalent to malloc_good_size() in OS X. - virtual size_t GetEstimatedAllocatedSize(size_t size); - - // Returns the actual number N of bytes reserved by tcmalloc for the - // pointer p. The client is allowed to use the range of bytes - // [p, p+N) in any way it wishes (i.e. N is the "usable size" of this - // allocation). This number may be equal to or greater than the number - // of bytes requested when p was allocated. - // p must have been allocated by this malloc implementation, - // must not be an interior pointer -- that is, must be exactly - // the pointer returned to by malloc() et al., not some offset - // from that -- and should not have been freed yet. p may be NULL. - // (Currently only implemented in tcmalloc; other implementations - // will return 0.) - // This is equivalent to malloc_size() in OS X, malloc_usable_size() - // in glibc, and _msize() for windows. - virtual size_t GetAllocatedSize(const void* p); - - // Returns kOwned if this malloc implementation allocated the memory - // pointed to by p, or kNotOwned if some other malloc implementation - // allocated it or p is NULL. May also return kUnknownOwnership if - // the malloc implementation does not keep track of ownership. - // REQUIRES: p must be a value returned from a previous call to - // malloc(), calloc(), realloc(), memalign(), posix_memalign(), - // valloc(), pvalloc(), new, or new[], and must refer to memory that - // is currently allocated (so, for instance, you should not pass in - // a pointer after having called free() on it). - enum Ownership { - // NOTE: Enum values MUST be kept in sync with the version in - // malloc_extension_c.h - kUnknownOwnership = 0, - kOwned, - kNotOwned - }; - virtual Ownership GetOwnership(const void* p); - - // The current malloc implementation. Always non-NULL. - static MallocExtension* instance(); - - // Change the malloc implementation. Typically called by the - // malloc implementation during initialization. - static void Register(MallocExtension* implementation); - - // Returns detailed information about malloc's freelists. For each list, - // return a FreeListInfo: - struct FreeListInfo { - size_t min_object_size; - size_t max_object_size; - size_t total_bytes_free; - const char* type; - }; - // Each item in the vector refers to a different freelist. The lists - // are identified by the range of allocations that objects in the - // list can satisfy ([min_object_size, max_object_size]) and the - // type of freelist (see below). The current size of the list is - // returned in total_bytes_free (which count against a processes - // resident and virtual size). - // - // Currently supported types are: - // - // "tcmalloc.page{_unmapped}" - tcmalloc's page heap. An entry for each size - // class in the page heap is returned. Bytes in "page_unmapped" - // are no longer backed by physical memory and do not count against - // the resident size of a process. - // - // "tcmalloc.large{_unmapped}" - tcmalloc's list of objects larger - // than the largest page heap size class. Only one "large" - // entry is returned. There is no upper-bound on the size - // of objects in the large free list; this call returns - // kint64max for max_object_size. Bytes in - // "large_unmapped" are no longer backed by physical memory - // and do not count against the resident size of a process. - // - // "tcmalloc.central" - tcmalloc's central free-list. One entry per - // size-class is returned. Never unmapped. - // - // "debug.free_queue" - free objects queued by the debug allocator - // and not returned to tcmalloc. - // - // "tcmalloc.thread" - tcmalloc's per-thread caches. Never unmapped. - virtual void GetFreeListSizes(std::vector* v); - - // Get a list of stack traces of sampled allocation points. Returns - // a pointer to a "new[]-ed" result array, and stores the sample - // period in "sample_period". - // - // The state is stored as a sequence of adjacent entries - // in the returned array. Each entry has the following form: - // uintptr_t count; // Number of objects with following trace - // uintptr_t size; // Total size of objects with following trace - // uintptr_t depth; // Number of PC values in stack trace - // void* stack[depth]; // PC values that form the stack trace - // - // The list of entries is terminated by a "count" of 0. - // - // It is the responsibility of the caller to "delete[]" the returned array. - // - // May return NULL to indicate no results. - // - // This is an internal extension. Callers should use the more - // convenient "GetHeapSample(string*)" method defined above. - virtual void** ReadStackTraces(int* sample_period); - - // Like ReadStackTraces(), but returns stack traces that caused growth - // in the address space size. - virtual void** ReadHeapGrowthStackTraces(); - - // Returns the size in bytes of the calling threads cache. - virtual size_t GetThreadCacheSize(); - - // Like MarkThreadIdle, but does not destroy the internal data - // structures of the thread cache. When the thread resumes, it wil - // have an empty cache but will not need to pay to reconstruct the - // cache data structures. - virtual void MarkThreadTemporarilyIdle(); -}; - -namespace base { - -// Information passed per range. More fields may be added later. -struct MallocRange { - enum Type { - INUSE, // Application is using this range - FREE, // Range is currently free - UNMAPPED, // Backing physical memory has been returned to the OS - UNKNOWN - // More enum values may be added in the future - }; - - uintptr_t address; // Address of range - size_t length; // Byte length of range - Type type; // Type of this range - double fraction; // Fraction of range that is being used (0 if !INUSE) - - // Perhaps add the following: - // - stack trace if this range was sampled - // - heap growth stack trace if applicable to this range - // - age when allocated (for inuse) or freed (if not in use) -}; - -} // namespace base - -#endif // BASE_MALLOC_EXTENSION_H_ diff --git a/contrib/libtcmalloc/include/gperftools/malloc_extension_c.h b/contrib/libtcmalloc/include/gperftools/malloc_extension_c.h deleted file mode 100644 index 70ff6868ecf..00000000000 --- a/contrib/libtcmalloc/include/gperftools/malloc_extension_c.h +++ /dev/null @@ -1,101 +0,0 @@ -/* Copyright (c) 2008, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * -- - * Author: Craig Silverstein - * - * C shims for the C++ malloc_extension.h. See malloc_extension.h for - * details. Note these C shims always work on - * MallocExtension::instance(); it is not possible to have more than - * one MallocExtension object in C applications. - */ - -#ifndef _MALLOC_EXTENSION_C_H_ -#define _MALLOC_EXTENSION_C_H_ - -#include -#include - -/* Annoying stuff for windows -- makes sure clients can import these fns */ -#ifndef PERFTOOLS_DLL_DECL -# ifdef _WIN32 -# define PERFTOOLS_DLL_DECL __declspec(dllimport) -# else -# define PERFTOOLS_DLL_DECL -# endif -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -#define kMallocExtensionHistogramSize 64 - -PERFTOOLS_DLL_DECL int MallocExtension_VerifyAllMemory(void); -PERFTOOLS_DLL_DECL int MallocExtension_VerifyNewMemory(const void* p); -PERFTOOLS_DLL_DECL int MallocExtension_VerifyArrayNewMemory(const void* p); -PERFTOOLS_DLL_DECL int MallocExtension_VerifyMallocMemory(const void* p); -PERFTOOLS_DLL_DECL int MallocExtension_MallocMemoryStats(int* blocks, size_t* total, - int histogram[kMallocExtensionHistogramSize]); -PERFTOOLS_DLL_DECL void MallocExtension_GetStats(char* buffer, int buffer_length); - -/* TODO(csilvers): write a C version of these routines, that perhaps - * takes a function ptr and a void *. - */ -/* void MallocExtension_GetHeapSample(string* result); */ -/* void MallocExtension_GetHeapGrowthStacks(string* result); */ - -PERFTOOLS_DLL_DECL int MallocExtension_GetNumericProperty(const char* property, size_t* value); -PERFTOOLS_DLL_DECL int MallocExtension_SetNumericProperty(const char* property, size_t value); -PERFTOOLS_DLL_DECL void MallocExtension_MarkThreadIdle(void); -PERFTOOLS_DLL_DECL void MallocExtension_MarkThreadBusy(void); -PERFTOOLS_DLL_DECL void MallocExtension_ReleaseToSystem(size_t num_bytes); -PERFTOOLS_DLL_DECL void MallocExtension_ReleaseFreeMemory(void); -PERFTOOLS_DLL_DECL size_t MallocExtension_GetEstimatedAllocatedSize(size_t size); -PERFTOOLS_DLL_DECL size_t MallocExtension_GetAllocatedSize(const void* p); -PERFTOOLS_DLL_DECL size_t MallocExtension_GetThreadCacheSize(void); -PERFTOOLS_DLL_DECL void MallocExtension_MarkThreadTemporarilyIdle(void); - -/* - * NOTE: These enum values MUST be kept in sync with the version in - * malloc_extension.h - */ -typedef enum { - MallocExtension_kUnknownOwnership = 0, - MallocExtension_kOwned, - MallocExtension_kNotOwned -} MallocExtension_Ownership; - -PERFTOOLS_DLL_DECL MallocExtension_Ownership MallocExtension_GetOwnership(const void* p); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* _MALLOC_EXTENSION_C_H_ */ diff --git a/contrib/libtcmalloc/include/gperftools/malloc_hook.h b/contrib/libtcmalloc/include/gperftools/malloc_hook.h deleted file mode 100644 index b76411fb590..00000000000 --- a/contrib/libtcmalloc/include/gperftools/malloc_hook.h +++ /dev/null @@ -1,359 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat -// -// Some of our malloc implementations can invoke the following hooks whenever -// memory is allocated or deallocated. MallocHook is thread-safe, and things -// you do before calling AddFooHook(MyHook) are visible to any resulting calls -// to MyHook. Hooks must be thread-safe. If you write: -// -// CHECK(MallocHook::AddNewHook(&MyNewHook)); -// -// MyNewHook will be invoked in subsequent calls in the current thread, but -// there are no guarantees on when it might be invoked in other threads. -// -// There are a limited number of slots available for each hook type. Add*Hook -// will return false if there are no slots available. Remove*Hook will return -// false if the given hook was not already installed. -// -// The order in which individual hooks are called in Invoke*Hook is undefined. -// -// It is safe for a hook to remove itself within Invoke*Hook and add other -// hooks. Any hooks added inside a hook invocation (for the same hook type) -// will not be invoked for the current invocation. -// -// One important user of these hooks is the heap profiler. -// -// CAVEAT: If you add new MallocHook::Invoke* calls then those calls must be -// directly in the code of the (de)allocation function that is provided to the -// user and that function must have an ATTRIBUTE_SECTION(malloc_hook) attribute. -// -// Note: the Invoke*Hook() functions are defined in malloc_hook-inl.h. If you -// need to invoke a hook (which you shouldn't unless you're part of tcmalloc), -// be sure to #include malloc_hook-inl.h in addition to malloc_hook.h. -// -// NOTE FOR C USERS: If you want to use malloc_hook functionality from -// a C program, #include malloc_hook_c.h instead of this file. - -#ifndef _MALLOC_HOOK_H_ -#define _MALLOC_HOOK_H_ - -#include -#include -extern "C" { -#include "malloc_hook_c.h" // a C version of the malloc_hook interface -} - -// Annoying stuff for windows -- makes sure clients can import these functions -#ifndef PERFTOOLS_DLL_DECL -# ifdef _WIN32 -# define PERFTOOLS_DLL_DECL __declspec(dllimport) -# else -# define PERFTOOLS_DLL_DECL -# endif -#endif - -// The C++ methods below call the C version (MallocHook_*), and thus -// convert between an int and a bool. Windows complains about this -// (a "performance warning") which we don't care about, so we suppress. -#ifdef _MSC_VER -#pragma warning(push) -#pragma warning(disable:4800) -#endif - -// Note: malloc_hook_c.h defines MallocHook_*Hook and -// MallocHook_{Add,Remove}*Hook. The version of these inside the MallocHook -// class are defined in terms of the malloc_hook_c version. See malloc_hook_c.h -// for details of these types/functions. - -class PERFTOOLS_DLL_DECL MallocHook { - public: - // The NewHook is invoked whenever an object is allocated. - // It may be passed NULL if the allocator returned NULL. - typedef MallocHook_NewHook NewHook; - inline static bool AddNewHook(NewHook hook) { - return MallocHook_AddNewHook(hook); - } - inline static bool RemoveNewHook(NewHook hook) { - return MallocHook_RemoveNewHook(hook); - } - inline static void InvokeNewHook(const void* p, size_t s); - - // The DeleteHook is invoked whenever an object is deallocated. - // It may be passed NULL if the caller is trying to delete NULL. - typedef MallocHook_DeleteHook DeleteHook; - inline static bool AddDeleteHook(DeleteHook hook) { - return MallocHook_AddDeleteHook(hook); - } - inline static bool RemoveDeleteHook(DeleteHook hook) { - return MallocHook_RemoveDeleteHook(hook); - } - inline static void InvokeDeleteHook(const void* p); - - // The PreMmapHook is invoked with mmap or mmap64 arguments just - // before the call is actually made. Such a hook may be useful - // in memory limited contexts, to catch allocations that will exceed - // a memory limit, and take outside actions to increase that limit. - typedef MallocHook_PreMmapHook PreMmapHook; - inline static bool AddPreMmapHook(PreMmapHook hook) { - return MallocHook_AddPreMmapHook(hook); - } - inline static bool RemovePreMmapHook(PreMmapHook hook) { - return MallocHook_RemovePreMmapHook(hook); - } - inline static void InvokePreMmapHook(const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset); - - // The MmapReplacement is invoked after the PreMmapHook but before - // the call is actually made. The MmapReplacement should return true - // if it handled the call, or false if it is still necessary to - // call mmap/mmap64. - // This should be used only by experts, and users must be be - // extremely careful to avoid recursive calls to mmap. The replacement - // should be async signal safe. - // Only one MmapReplacement is supported. After setting an MmapReplacement - // you must call RemoveMmapReplacement before calling SetMmapReplacement - // again. - typedef MallocHook_MmapReplacement MmapReplacement; - inline static bool SetMmapReplacement(MmapReplacement hook) { - return MallocHook_SetMmapReplacement(hook); - } - inline static bool RemoveMmapReplacement(MmapReplacement hook) { - return MallocHook_RemoveMmapReplacement(hook); - } - inline static bool InvokeMmapReplacement(const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset, - void** result); - - - // The MmapHook is invoked whenever a region of memory is mapped. - // It may be passed MAP_FAILED if the mmap failed. - typedef MallocHook_MmapHook MmapHook; - inline static bool AddMmapHook(MmapHook hook) { - return MallocHook_AddMmapHook(hook); - } - inline static bool RemoveMmapHook(MmapHook hook) { - return MallocHook_RemoveMmapHook(hook); - } - inline static void InvokeMmapHook(const void* result, - const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset); - - // The MunmapReplacement is invoked with munmap arguments just before - // the call is actually made. The MunmapReplacement should return true - // if it handled the call, or false if it is still necessary to - // call munmap. - // This should be used only by experts. The replacement should be - // async signal safe. - // Only one MunmapReplacement is supported. After setting an - // MunmapReplacement you must call RemoveMunmapReplacement before - // calling SetMunmapReplacement again. - typedef MallocHook_MunmapReplacement MunmapReplacement; - inline static bool SetMunmapReplacement(MunmapReplacement hook) { - return MallocHook_SetMunmapReplacement(hook); - } - inline static bool RemoveMunmapReplacement(MunmapReplacement hook) { - return MallocHook_RemoveMunmapReplacement(hook); - } - inline static bool InvokeMunmapReplacement(const void* p, - size_t size, - int* result); - - // The MunmapHook is invoked whenever a region of memory is unmapped. - typedef MallocHook_MunmapHook MunmapHook; - inline static bool AddMunmapHook(MunmapHook hook) { - return MallocHook_AddMunmapHook(hook); - } - inline static bool RemoveMunmapHook(MunmapHook hook) { - return MallocHook_RemoveMunmapHook(hook); - } - inline static void InvokeMunmapHook(const void* p, size_t size); - - // The MremapHook is invoked whenever a region of memory is remapped. - typedef MallocHook_MremapHook MremapHook; - inline static bool AddMremapHook(MremapHook hook) { - return MallocHook_AddMremapHook(hook); - } - inline static bool RemoveMremapHook(MremapHook hook) { - return MallocHook_RemoveMremapHook(hook); - } - inline static void InvokeMremapHook(const void* result, - const void* old_addr, - size_t old_size, - size_t new_size, - int flags, - const void* new_addr); - - // The PreSbrkHook is invoked just before sbrk is called -- except when - // the increment is 0. This is because sbrk(0) is often called - // to get the top of the memory stack, and is not actually a - // memory-allocation call. It may be useful in memory-limited contexts, - // to catch allocations that will exceed the limit and take outside - // actions to increase such a limit. - typedef MallocHook_PreSbrkHook PreSbrkHook; - inline static bool AddPreSbrkHook(PreSbrkHook hook) { - return MallocHook_AddPreSbrkHook(hook); - } - inline static bool RemovePreSbrkHook(PreSbrkHook hook) { - return MallocHook_RemovePreSbrkHook(hook); - } - inline static void InvokePreSbrkHook(ptrdiff_t increment); - - // The SbrkHook is invoked whenever sbrk is called -- except when - // the increment is 0. This is because sbrk(0) is often called - // to get the top of the memory stack, and is not actually a - // memory-allocation call. - typedef MallocHook_SbrkHook SbrkHook; - inline static bool AddSbrkHook(SbrkHook hook) { - return MallocHook_AddSbrkHook(hook); - } - inline static bool RemoveSbrkHook(SbrkHook hook) { - return MallocHook_RemoveSbrkHook(hook); - } - inline static void InvokeSbrkHook(const void* result, ptrdiff_t increment); - - // Get the current stack trace. Try to skip all routines up to and - // and including the caller of MallocHook::Invoke*. - // Use "skip_count" (similarly to GetStackTrace from stacktrace.h) - // as a hint about how many routines to skip if better information - // is not available. - inline static int GetCallerStackTrace(void** result, int max_depth, - int skip_count) { - return MallocHook_GetCallerStackTrace(result, max_depth, skip_count); - } - - // Unhooked versions of mmap() and munmap(). These should be used - // only by experts, since they bypass heapchecking, etc. - // Note: These do not run hooks, but they still use the MmapReplacement - // and MunmapReplacement. - static void* UnhookedMMap(void *start, size_t length, int prot, int flags, - int fd, off_t offset); - static int UnhookedMUnmap(void *start, size_t length); - - // The following are DEPRECATED. - inline static NewHook GetNewHook(); - inline static NewHook SetNewHook(NewHook hook) { - return MallocHook_SetNewHook(hook); - } - - inline static DeleteHook GetDeleteHook(); - inline static DeleteHook SetDeleteHook(DeleteHook hook) { - return MallocHook_SetDeleteHook(hook); - } - - inline static PreMmapHook GetPreMmapHook(); - inline static PreMmapHook SetPreMmapHook(PreMmapHook hook) { - return MallocHook_SetPreMmapHook(hook); - } - - inline static MmapHook GetMmapHook(); - inline static MmapHook SetMmapHook(MmapHook hook) { - return MallocHook_SetMmapHook(hook); - } - - inline static MunmapHook GetMunmapHook(); - inline static MunmapHook SetMunmapHook(MunmapHook hook) { - return MallocHook_SetMunmapHook(hook); - } - - inline static MremapHook GetMremapHook(); - inline static MremapHook SetMremapHook(MremapHook hook) { - return MallocHook_SetMremapHook(hook); - } - - inline static PreSbrkHook GetPreSbrkHook(); - inline static PreSbrkHook SetPreSbrkHook(PreSbrkHook hook) { - return MallocHook_SetPreSbrkHook(hook); - } - - inline static SbrkHook GetSbrkHook(); - inline static SbrkHook SetSbrkHook(SbrkHook hook) { - return MallocHook_SetSbrkHook(hook); - } - // End of DEPRECATED methods. - - private: - // Slow path versions of Invoke*Hook. - static void InvokeNewHookSlow(const void* p, size_t s); - static void InvokeDeleteHookSlow(const void* p); - static void InvokePreMmapHookSlow(const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset); - static void InvokeMmapHookSlow(const void* result, - const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset); - static bool InvokeMmapReplacementSlow(const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset, - void** result); - static void InvokeMunmapHookSlow(const void* p, size_t size); - static bool InvokeMunmapReplacementSlow(const void* p, - size_t size, - int* result); - static void InvokeMremapHookSlow(const void* result, - const void* old_addr, - size_t old_size, - size_t new_size, - int flags, - const void* new_addr); - static void InvokePreSbrkHookSlow(ptrdiff_t increment); - static void InvokeSbrkHookSlow(const void* result, ptrdiff_t increment); -}; - -#ifdef _MSC_VER -#pragma warning(pop) -#endif - - -#endif /* _MALLOC_HOOK_H_ */ diff --git a/contrib/libtcmalloc/include/gperftools/malloc_hook_c.h b/contrib/libtcmalloc/include/gperftools/malloc_hook_c.h deleted file mode 100644 index 56337e15e83..00000000000 --- a/contrib/libtcmalloc/include/gperftools/malloc_hook_c.h +++ /dev/null @@ -1,173 +0,0 @@ -/* Copyright (c) 2008, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * -- - * Author: Craig Silverstein - * - * C shims for the C++ malloc_hook.h. See malloc_hook.h for details - * on how to use these. - */ - -#ifndef _MALLOC_HOOK_C_H_ -#define _MALLOC_HOOK_C_H_ - -#include -#include - -/* Annoying stuff for windows; makes sure clients can import these functions */ -#ifndef PERFTOOLS_DLL_DECL -# ifdef _WIN32 -# define PERFTOOLS_DLL_DECL __declspec(dllimport) -# else -# define PERFTOOLS_DLL_DECL -# endif -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -/* Get the current stack trace. Try to skip all routines up to and - * and including the caller of MallocHook::Invoke*. - * Use "skip_count" (similarly to GetStackTrace from stacktrace.h) - * as a hint about how many routines to skip if better information - * is not available. - */ -PERFTOOLS_DLL_DECL -int MallocHook_GetCallerStackTrace(void** result, int max_depth, - int skip_count); - -/* The MallocHook_{Add,Remove}*Hook functions return 1 on success and 0 on - * failure. - */ - -typedef void (*MallocHook_NewHook)(const void* ptr, size_t size); -PERFTOOLS_DLL_DECL -int MallocHook_AddNewHook(MallocHook_NewHook hook); -PERFTOOLS_DLL_DECL -int MallocHook_RemoveNewHook(MallocHook_NewHook hook); - -typedef void (*MallocHook_DeleteHook)(const void* ptr); -PERFTOOLS_DLL_DECL -int MallocHook_AddDeleteHook(MallocHook_DeleteHook hook); -PERFTOOLS_DLL_DECL -int MallocHook_RemoveDeleteHook(MallocHook_DeleteHook hook); - -typedef void (*MallocHook_PreMmapHook)(const void *start, - size_t size, - int protection, - int flags, - int fd, - off_t offset); -PERFTOOLS_DLL_DECL -int MallocHook_AddPreMmapHook(MallocHook_PreMmapHook hook); -PERFTOOLS_DLL_DECL -int MallocHook_RemovePreMmapHook(MallocHook_PreMmapHook hook); - -typedef void (*MallocHook_MmapHook)(const void* result, - const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset); -PERFTOOLS_DLL_DECL -int MallocHook_AddMmapHook(MallocHook_MmapHook hook); -PERFTOOLS_DLL_DECL -int MallocHook_RemoveMmapHook(MallocHook_MmapHook hook); - -typedef int (*MallocHook_MmapReplacement)(const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset, - void** result); -int MallocHook_SetMmapReplacement(MallocHook_MmapReplacement hook); -int MallocHook_RemoveMmapReplacement(MallocHook_MmapReplacement hook); - -typedef void (*MallocHook_MunmapHook)(const void* ptr, size_t size); -PERFTOOLS_DLL_DECL -int MallocHook_AddMunmapHook(MallocHook_MunmapHook hook); -PERFTOOLS_DLL_DECL -int MallocHook_RemoveMunmapHook(MallocHook_MunmapHook hook); - -typedef int (*MallocHook_MunmapReplacement)(const void* ptr, - size_t size, - int* result); -int MallocHook_SetMunmapReplacement(MallocHook_MunmapReplacement hook); -int MallocHook_RemoveMunmapReplacement(MallocHook_MunmapReplacement hook); - -typedef void (*MallocHook_MremapHook)(const void* result, - const void* old_addr, - size_t old_size, - size_t new_size, - int flags, - const void* new_addr); -PERFTOOLS_DLL_DECL -int MallocHook_AddMremapHook(MallocHook_MremapHook hook); -PERFTOOLS_DLL_DECL -int MallocHook_RemoveMremapHook(MallocHook_MremapHook hook); - -typedef void (*MallocHook_PreSbrkHook)(ptrdiff_t increment); -PERFTOOLS_DLL_DECL -int MallocHook_AddPreSbrkHook(MallocHook_PreSbrkHook hook); -PERFTOOLS_DLL_DECL -int MallocHook_RemovePreSbrkHook(MallocHook_PreSbrkHook hook); - -typedef void (*MallocHook_SbrkHook)(const void* result, ptrdiff_t increment); -PERFTOOLS_DLL_DECL -int MallocHook_AddSbrkHook(MallocHook_SbrkHook hook); -PERFTOOLS_DLL_DECL -int MallocHook_RemoveSbrkHook(MallocHook_SbrkHook hook); - -/* The following are DEPRECATED. */ -PERFTOOLS_DLL_DECL -MallocHook_NewHook MallocHook_SetNewHook(MallocHook_NewHook hook); -PERFTOOLS_DLL_DECL -MallocHook_DeleteHook MallocHook_SetDeleteHook(MallocHook_DeleteHook hook); -PERFTOOLS_DLL_DECL -MallocHook_PreMmapHook MallocHook_SetPreMmapHook(MallocHook_PreMmapHook hook); -PERFTOOLS_DLL_DECL -MallocHook_MmapHook MallocHook_SetMmapHook(MallocHook_MmapHook hook); -PERFTOOLS_DLL_DECL -MallocHook_MunmapHook MallocHook_SetMunmapHook(MallocHook_MunmapHook hook); -PERFTOOLS_DLL_DECL -MallocHook_MremapHook MallocHook_SetMremapHook(MallocHook_MremapHook hook); -PERFTOOLS_DLL_DECL -MallocHook_PreSbrkHook MallocHook_SetPreSbrkHook(MallocHook_PreSbrkHook hook); -PERFTOOLS_DLL_DECL -MallocHook_SbrkHook MallocHook_SetSbrkHook(MallocHook_SbrkHook hook); -/* End of DEPRECATED functions. */ - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif /* _MALLOC_HOOK_C_H_ */ diff --git a/contrib/libtcmalloc/include/gperftools/profiler.h b/contrib/libtcmalloc/include/gperftools/profiler.h deleted file mode 100644 index 2d272d616a9..00000000000 --- a/contrib/libtcmalloc/include/gperftools/profiler.h +++ /dev/null @@ -1,169 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2005, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Sanjay Ghemawat - * - * Module for CPU profiling based on periodic pc-sampling. - * - * For full(er) information, see doc/cpuprofile.html - * - * This module is linked into your program with - * no slowdown caused by this unless you activate the profiler - * using one of the following methods: - * - * 1. Before starting the program, set the environment variable - * "CPUPROFILE" to be the name of the file to which the profile - * data should be written. - * - * 2. Programmatically, start and stop the profiler using the - * routines "ProfilerStart(filename)" and "ProfilerStop()". - * - * - * (Note: if using linux 2.4 or earlier, only the main thread may be - * profiled.) - * - * Use pprof to view the resulting profile output. - * % pprof - * % pprof --gv - * - * These functions are thread-safe. - */ - -#ifndef BASE_PROFILER_H_ -#define BASE_PROFILER_H_ - -#include /* For time_t */ - -/* Annoying stuff for windows; makes sure clients can import these functions */ -#ifndef PERFTOOLS_DLL_DECL -# ifdef _WIN32 -# define PERFTOOLS_DLL_DECL __declspec(dllimport) -# else -# define PERFTOOLS_DLL_DECL -# endif -#endif - -/* All this code should be usable from within C apps. */ -#ifdef __cplusplus -extern "C" { -#endif - -/* Profiler options, for use with ProfilerStartWithOptions. To use: - * - * struct ProfilerOptions options; - * memset(&options, 0, sizeof options); - * - * then fill in fields as needed. - * - * This structure is intended to be usable from C code, so no constructor - * is provided to initialize it. (Use memset as described above). - */ -struct ProfilerOptions { - /* Filter function and argument. - * - * If filter_in_thread is not NULL, when a profiling tick is delivered - * the profiler will call: - * - * (*filter_in_thread)(filter_in_thread_arg) - * - * If it returns nonzero, the sample will be included in the profile. - * Note that filter_in_thread runs in a signal handler, so must be - * async-signal-safe. - * - * A typical use would be to set up filter results for each thread - * in the system before starting the profiler, then to make - * filter_in_thread be a very simple function which retrieves those - * results in an async-signal-safe way. Retrieval could be done - * using thread-specific data, or using a shared data structure that - * supports async-signal-safe lookups. - */ - int (*filter_in_thread)(void *arg); - void *filter_in_thread_arg; -}; - -/* Start profiling and write profile info into fname, discarding any - * existing profiling data in that file. - * - * This is equivalent to calling ProfilerStartWithOptions(fname, NULL). - */ -PERFTOOLS_DLL_DECL int ProfilerStart(const char* fname); - -/* Start profiling and write profile into fname, discarding any - * existing profiling data in that file. - * - * The profiler is configured using the options given by 'options'. - * Options which are not specified are given default values. - * - * 'options' may be NULL, in which case all are given default values. - * - * Returns nonzero if profiling was started successfully, or zero else. - */ -PERFTOOLS_DLL_DECL int ProfilerStartWithOptions( - const char *fname, const struct ProfilerOptions *options); - -/* Stop profiling. Can be started again with ProfilerStart(), but - * the currently accumulated profiling data will be cleared. - */ -PERFTOOLS_DLL_DECL void ProfilerStop(void); - -/* Flush any currently buffered profiling state to the profile file. - * Has no effect if the profiler has not been started. - */ -PERFTOOLS_DLL_DECL void ProfilerFlush(void); - - -/* DEPRECATED: these functions were used to enable/disable profiling - * in the current thread, but no longer do anything. - */ -PERFTOOLS_DLL_DECL void ProfilerEnable(void); -PERFTOOLS_DLL_DECL void ProfilerDisable(void); - -/* Returns nonzero if profile is currently enabled, zero if it's not. */ -PERFTOOLS_DLL_DECL int ProfilingIsEnabledForAllThreads(void); - -/* Routine for registering new threads with the profiler. - */ -PERFTOOLS_DLL_DECL void ProfilerRegisterThread(void); - -/* Stores state about profiler's current status into "*state". */ -struct ProfilerState { - int enabled; /* Is profiling currently enabled? */ - time_t start_time; /* If enabled, when was profiling started? */ - char profile_name[1024]; /* Name of profile file being written, or '\0' */ - int samples_gathered; /* Number of samples gathered so far (or 0) */ -}; -PERFTOOLS_DLL_DECL void ProfilerGetCurrentState(struct ProfilerState* state); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif /* BASE_PROFILER_H_ */ diff --git a/contrib/libtcmalloc/include/gperftools/stacktrace.h b/contrib/libtcmalloc/include/gperftools/stacktrace.h deleted file mode 100644 index 2b9c5a13209..00000000000 --- a/contrib/libtcmalloc/include/gperftools/stacktrace.h +++ /dev/null @@ -1,117 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat -// -// Routines to extract the current stack trace. These functions are -// thread-safe. - -#ifndef GOOGLE_STACKTRACE_H_ -#define GOOGLE_STACKTRACE_H_ - -// Annoying stuff for windows -- makes sure clients can import these functions -#ifndef PERFTOOLS_DLL_DECL -# ifdef _WIN32 -# define PERFTOOLS_DLL_DECL __declspec(dllimport) -# else -# define PERFTOOLS_DLL_DECL -# endif -#endif - - -// Skips the most recent "skip_count" stack frames (also skips the -// frame generated for the "GetStackFrames" routine itself), and then -// records the pc values for up to the next "max_depth" frames in -// "result", and the corresponding stack frame sizes in "sizes". -// Returns the number of values recorded in "result"/"sizes". -// -// Example: -// main() { foo(); } -// foo() { bar(); } -// bar() { -// void* result[10]; -// int sizes[10]; -// int depth = GetStackFrames(result, sizes, 10, 1); -// } -// -// The GetStackFrames call will skip the frame for "bar". It will -// return 2 and will produce pc values that map to the following -// procedures: -// result[0] foo -// result[1] main -// (Actually, there may be a few more entries after "main" to account for -// startup procedures.) -// And corresponding stack frame sizes will also be recorded: -// sizes[0] 16 -// sizes[1] 16 -// (Stack frame sizes of 16 above are just for illustration purposes.) -// Stack frame sizes of 0 or less indicate that those frame sizes couldn't -// be identified. -// -// This routine may return fewer stack frame entries than are -// available. Also note that "result" and "sizes" must both be non-NULL. -extern PERFTOOLS_DLL_DECL int GetStackFrames(void** result, int* sizes, int max_depth, - int skip_count); - -// Same as above, but to be used from a signal handler. The "uc" parameter -// should be the pointer to ucontext_t which was passed as the 3rd parameter -// to sa_sigaction signal handler. It may help the unwinder to get a -// better stack trace under certain conditions. The "uc" may safely be NULL. -extern PERFTOOLS_DLL_DECL int GetStackFramesWithContext(void** result, int* sizes, int max_depth, - int skip_count, const void *uc); - -// This is similar to the GetStackFrames routine, except that it returns -// the stack trace only, and not the stack frame sizes as well. -// Example: -// main() { foo(); } -// foo() { bar(); } -// bar() { -// void* result[10]; -// int depth = GetStackTrace(result, 10, 1); -// } -// -// This produces: -// result[0] foo -// result[1] main -// .... ... -// -// "result" must not be NULL. -extern PERFTOOLS_DLL_DECL int GetStackTrace(void** result, int max_depth, - int skip_count); - -// Same as above, but to be used from a signal handler. The "uc" parameter -// should be the pointer to ucontext_t which was passed as the 3rd parameter -// to sa_sigaction signal handler. It may help the unwinder to get a -// better stack trace under certain conditions. The "uc" may safely be NULL. -extern PERFTOOLS_DLL_DECL int GetStackTraceWithContext(void** result, int max_depth, - int skip_count, const void *uc); - -#endif /* GOOGLE_STACKTRACE_H_ */ diff --git a/contrib/libtcmalloc/include/gperftools/tcmalloc.h b/contrib/libtcmalloc/include/gperftools/tcmalloc.h deleted file mode 100644 index a5b39dbffbe..00000000000 --- a/contrib/libtcmalloc/include/gperftools/tcmalloc.h +++ /dev/null @@ -1,160 +0,0 @@ -// -*- Mode: C; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2003, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Sanjay Ghemawat - * .h file by Craig Silverstein - */ - -#ifndef TCMALLOC_TCMALLOC_H_ -#define TCMALLOC_TCMALLOC_H_ - -#include /* for size_t */ - -/* Define the version number so folks can check against it */ -#define TC_VERSION_MAJOR 2 -#define TC_VERSION_MINOR 5 -#define TC_VERSION_PATCH "" -#define TC_VERSION_STRING "gperftools 2.5" - -/* For struct mallinfo, if it's defined. */ -#if !defined(__APPLE__) && !defined(__FreeBSD__) -# include -#else -struct mallinfo { - size_t arena; /* non-mmapped space allocated from system */ - size_t ordblks; /* number of free chunks */ - size_t smblks; /* always 0 */ - size_t hblks; /* always 0 */ - size_t hblkhd; /* space in mmapped regions */ - size_t usmblks; /* maximum total allocated space */ - size_t fsmblks; /* always 0 */ - size_t uordblks; /* total allocated space */ - size_t fordblks; /* total free space */ - size_t keepcost; /* releasable (via malloc_trim) space */ -}; -#endif - -#ifdef __cplusplus -#define PERFTOOLS_THROW throw() -#else -# ifdef __GNUC__ -# define PERFTOOLS_THROW __attribute__((__nothrow__)) -# else -# define PERFTOOLS_THROW -# endif -#endif - -#ifndef PERFTOOLS_DLL_DECL -#define PERFTOOLS_DLL_DECL_DEFINED -# ifdef _WIN32 -# define PERFTOOLS_DLL_DECL __declspec(dllimport) -# else -# define PERFTOOLS_DLL_DECL -# endif -#endif - -#ifdef __cplusplus -namespace std { -struct nothrow_t; -} - -extern "C" { -#endif - /* - * Returns a human-readable version string. If major, minor, - * and/or patch are not NULL, they are set to the major version, - * minor version, and patch-code (a string, usually ""). - */ - PERFTOOLS_DLL_DECL const char* tc_version(int* major, int* minor, - const char** patch) PERFTOOLS_THROW; - - PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) PERFTOOLS_THROW; - PERFTOOLS_DLL_DECL void* tc_malloc_skip_new_handler(size_t size) PERFTOOLS_THROW; - PERFTOOLS_DLL_DECL void tc_free(void* ptr) PERFTOOLS_THROW; - PERFTOOLS_DLL_DECL void tc_free_sized(void *ptr, size_t size) PERFTOOLS_THROW; - PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) PERFTOOLS_THROW; - PERFTOOLS_DLL_DECL void* tc_calloc(size_t nmemb, size_t size) PERFTOOLS_THROW; - PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) PERFTOOLS_THROW; - - PERFTOOLS_DLL_DECL void* tc_memalign(size_t __alignment, - size_t __size) PERFTOOLS_THROW; - PERFTOOLS_DLL_DECL int tc_posix_memalign(void** ptr, - size_t align, size_t size) PERFTOOLS_THROW; - PERFTOOLS_DLL_DECL void* tc_valloc(size_t __size) PERFTOOLS_THROW; - PERFTOOLS_DLL_DECL void* tc_pvalloc(size_t __size) PERFTOOLS_THROW; - - PERFTOOLS_DLL_DECL void tc_malloc_stats(void) PERFTOOLS_THROW; - PERFTOOLS_DLL_DECL int tc_mallopt(int cmd, int value) PERFTOOLS_THROW; -#if 1 - PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) PERFTOOLS_THROW; -#endif - - /* - * This is an alias for MallocExtension::instance()->GetAllocatedSize(). - * It is equivalent to - * OS X: malloc_size() - * glibc: malloc_usable_size() - * Windows: _msize() - */ - PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) PERFTOOLS_THROW; - -#ifdef __cplusplus - PERFTOOLS_DLL_DECL int tc_set_new_mode(int flag) PERFTOOLS_THROW; - PERFTOOLS_DLL_DECL void* tc_new(size_t size); - PERFTOOLS_DLL_DECL void* tc_new_nothrow(size_t size, - const std::nothrow_t&) PERFTOOLS_THROW; - PERFTOOLS_DLL_DECL void tc_delete(void* p) PERFTOOLS_THROW; - PERFTOOLS_DLL_DECL void tc_delete_sized(void* p, size_t size) throw(); - PERFTOOLS_DLL_DECL void tc_delete_nothrow(void* p, - const std::nothrow_t&) PERFTOOLS_THROW; - PERFTOOLS_DLL_DECL void* tc_newarray(size_t size); - PERFTOOLS_DLL_DECL void* tc_newarray_nothrow(size_t size, - const std::nothrow_t&) PERFTOOLS_THROW; - PERFTOOLS_DLL_DECL void tc_deletearray(void* p) PERFTOOLS_THROW; - PERFTOOLS_DLL_DECL void tc_deletearray_sized(void* p, size_t size) throw(); - PERFTOOLS_DLL_DECL void tc_deletearray_nothrow(void* p, - const std::nothrow_t&) PERFTOOLS_THROW; -} -#endif - -/* We're only un-defining those for public */ -#if !defined(GPERFTOOLS_CONFIG_H_) - -#undef PERFTOOLS_THROW - -#ifdef PERFTOOLS_DLL_DECL_DEFINED -#undef PERFTOOLS_DLL_DECL -#undef PERFTOOLS_DLL_DECL_DEFINED -#endif - -#endif /* GPERFTOOLS_CONFIG_H_ */ - -#endif /* #ifndef TCMALLOC_TCMALLOC_H_ */ diff --git a/contrib/libtcmalloc/src/addressmap-inl.h b/contrib/libtcmalloc/src/addressmap-inl.h deleted file mode 100644 index fd1dc5b6ffe..00000000000 --- a/contrib/libtcmalloc/src/addressmap-inl.h +++ /dev/null @@ -1,422 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat -// -// A fast map from addresses to values. Assumes that addresses are -// clustered. The main use is intended to be for heap-profiling. -// May be too memory-hungry for other uses. -// -// We use a user-defined allocator/de-allocator so that we can use -// this data structure during heap-profiling. -// -// IMPLEMENTATION DETAIL: -// -// Some default definitions/parameters: -// * Block -- aligned 128-byte region of the address space -// * Cluster -- aligned 1-MB region of the address space -// * Block-ID -- block-number within a cluster -// * Cluster-ID -- Starting address of cluster divided by cluster size -// -// We use a three-level map to represent the state: -// 1. A hash-table maps from a cluster-ID to the data for that cluster. -// 2. For each non-empty cluster we keep an array indexed by -// block-ID tht points to the first entry in the linked-list -// for the block. -// 3. At the bottom, we keep a singly-linked list of all -// entries in a block (for non-empty blocks). -// -// hash table -// +-------------+ -// | id->cluster |---> ... -// | ... | -// | id->cluster |---> Cluster -// +-------------+ +-------+ Data for one block -// | nil | +------------------------------------+ -// | ----+---|->[addr/value]-->[addr/value]-->... | -// | nil | +------------------------------------+ -// | ----+--> ... -// | nil | -// | ... | -// +-------+ -// -// Note that we require zero-bytes of overhead for completely empty -// clusters. The minimum space requirement for a cluster is the size -// of the hash-table entry plus a pointer value for each block in -// the cluster. Empty blocks impose no extra space requirement. -// -// The cost of a lookup is: -// a. A hash-table lookup to find the cluster -// b. An array access in the cluster structure -// c. A traversal over the linked-list for a block - -#ifndef BASE_ADDRESSMAP_INL_H_ -#define BASE_ADDRESSMAP_INL_H_ - -#include "config.h" -#include -#include -#if defined HAVE_STDINT_H -#include // to get uint16_t (ISO naming madness) -#elif defined HAVE_INTTYPES_H -#include // another place uint16_t might be defined -#else -#include // our last best hope -#endif - -// This class is thread-unsafe -- that is, instances of this class can -// not be accessed concurrently by multiple threads -- because the -// callback function for Iterate() may mutate contained values. If the -// callback functions you pass do not mutate their Value* argument, -// AddressMap can be treated as thread-compatible -- that is, it's -// safe for multiple threads to call "const" methods on this class, -// but not safe for one thread to call const methods on this class -// while another thread is calling non-const methods on the class. -template -class AddressMap { - public: - typedef void* (*Allocator)(size_t size); - typedef void (*DeAllocator)(void* ptr); - typedef const void* Key; - - // Create an AddressMap that uses the specified allocator/deallocator. - // The allocator/deallocator should behave like malloc/free. - // For instance, the allocator does not need to return initialized memory. - AddressMap(Allocator alloc, DeAllocator dealloc); - ~AddressMap(); - - // If the map contains an entry for "key", return it. Else return NULL. - inline const Value* Find(Key key) const; - inline Value* FindMutable(Key key); - - // Insert into the map. Any old value associated - // with key is forgotten. - void Insert(Key key, Value value); - - // Remove any entry for key in the map. If an entry was found - // and removed, stores the associated value in "*removed_value" - // and returns true. Else returns false. - bool FindAndRemove(Key key, Value* removed_value); - - // Similar to Find but we assume that keys are addresses of non-overlapping - // memory ranges whose sizes are given by size_func. - // If the map contains a range into which "key" points - // (at its start or inside of it, but not at the end), - // return the address of the associated value - // and store its key in "*res_key". - // Else return NULL. - // max_size specifies largest range size possibly in existence now. - typedef size_t (*ValueSizeFunc)(const Value& v); - const Value* FindInside(ValueSizeFunc size_func, size_t max_size, - Key key, Key* res_key); - - // Iterate over the address map calling 'callback' - // for all stored key-value pairs and passing 'arg' to it. - // We don't use full Closure/Callback machinery not to add - // unnecessary dependencies to this class with low-level uses. - template - inline void Iterate(void (*callback)(Key, Value*, Type), Type arg) const; - - private: - typedef uintptr_t Number; - - // The implementation assumes that addresses inserted into the map - // will be clustered. We take advantage of this fact by splitting - // up the address-space into blocks and using a linked-list entry - // for each block. - - // Size of each block. There is one linked-list for each block, so - // do not make the block-size too big. Oterwise, a lot of time - // will be spent traversing linked lists. - static const int kBlockBits = 7; - static const int kBlockSize = 1 << kBlockBits; - - // Entry kept in per-block linked-list - struct Entry { - Entry* next; - Key key; - Value value; - }; - - // We further group a sequence of consecutive blocks into a cluster. - // The data for a cluster is represented as a dense array of - // linked-lists, one list per contained block. - static const int kClusterBits = 13; - static const Number kClusterSize = 1 << (kBlockBits + kClusterBits); - static const int kClusterBlocks = 1 << kClusterBits; - - // We use a simple chaining hash-table to represent the clusters. - struct Cluster { - Cluster* next; // Next cluster in hash table chain - Number id; // Cluster ID - Entry* blocks[kClusterBlocks]; // Per-block linked-lists - }; - - // Number of hash-table entries. With the block-size/cluster-size - // defined above, each cluster covers 1 MB, so an 4K entry - // hash-table will give an average hash-chain length of 1 for 4GB of - // in-use memory. - static const int kHashBits = 12; - static const int kHashSize = 1 << 12; - - // Number of entry objects allocated at a time - static const int ALLOC_COUNT = 64; - - Cluster** hashtable_; // The hash-table - Entry* free_; // Free list of unused Entry objects - - // Multiplicative hash function: - // The value "kHashMultiplier" is the bottom 32 bits of - // int((sqrt(5)-1)/2 * 2^32) - // This is a good multiplier as suggested in CLR, Knuth. The hash - // value is taken to be the top "k" bits of the bottom 32 bits - // of the muliplied value. - static const uint32_t kHashMultiplier = 2654435769u; - static int HashInt(Number x) { - // Multiply by a constant and take the top bits of the result. - const uint32_t m = static_cast(x) * kHashMultiplier; - return static_cast(m >> (32 - kHashBits)); - } - - // Find cluster object for specified address. If not found - // and "create" is true, create the object. If not found - // and "create" is false, return NULL. - // - // This method is bitwise-const if create is false. - Cluster* FindCluster(Number address, bool create) { - // Look in hashtable - const Number cluster_id = address >> (kBlockBits + kClusterBits); - const int h = HashInt(cluster_id); - for (Cluster* c = hashtable_[h]; c != NULL; c = c->next) { - if (c->id == cluster_id) { - return c; - } - } - - // Create cluster if necessary - if (create) { - Cluster* c = New(1); - c->id = cluster_id; - c->next = hashtable_[h]; - hashtable_[h] = c; - return c; - } - return NULL; - } - - // Return the block ID for an address within its cluster - static int BlockID(Number address) { - return (address >> kBlockBits) & (kClusterBlocks - 1); - } - - //-------------------------------------------------------------- - // Memory management -- we keep all objects we allocate linked - // together in a singly linked list so we can get rid of them - // when we are all done. Furthermore, we allow the client to - // pass in custom memory allocator/deallocator routines. - //-------------------------------------------------------------- - struct Object { - Object* next; - // The real data starts here - }; - - Allocator alloc_; // The allocator - DeAllocator dealloc_; // The deallocator - Object* allocated_; // List of allocated objects - - // Allocates a zeroed array of T with length "num". Also inserts - // the allocated block into a linked list so it can be deallocated - // when we are all done. - template T* New(int num) { - void* ptr = (*alloc_)(sizeof(Object) + num*sizeof(T)); - memset(ptr, 0, sizeof(Object) + num*sizeof(T)); - Object* obj = reinterpret_cast(ptr); - obj->next = allocated_; - allocated_ = obj; - return reinterpret_cast(reinterpret_cast(ptr) + 1); - } -}; - -// More implementation details follow: - -template -AddressMap::AddressMap(Allocator alloc, DeAllocator dealloc) - : free_(NULL), - alloc_(alloc), - dealloc_(dealloc), - allocated_(NULL) { - hashtable_ = New(kHashSize); -} - -template -AddressMap::~AddressMap() { - // De-allocate all of the objects we allocated - for (Object* obj = allocated_; obj != NULL; /**/) { - Object* next = obj->next; - (*dealloc_)(obj); - obj = next; - } -} - -template -inline const Value* AddressMap::Find(Key key) const { - return const_cast(this)->FindMutable(key); -} - -template -inline Value* AddressMap::FindMutable(Key key) { - const Number num = reinterpret_cast(key); - const Cluster* const c = FindCluster(num, false/*do not create*/); - if (c != NULL) { - for (Entry* e = c->blocks[BlockID(num)]; e != NULL; e = e->next) { - if (e->key == key) { - return &e->value; - } - } - } - return NULL; -} - -template -void AddressMap::Insert(Key key, Value value) { - const Number num = reinterpret_cast(key); - Cluster* const c = FindCluster(num, true/*create*/); - - // Look in linked-list for this block - const int block = BlockID(num); - for (Entry* e = c->blocks[block]; e != NULL; e = e->next) { - if (e->key == key) { - e->value = value; - return; - } - } - - // Create entry - if (free_ == NULL) { - // Allocate a new batch of entries and add to free-list - Entry* array = New(ALLOC_COUNT); - for (int i = 0; i < ALLOC_COUNT-1; i++) { - array[i].next = &array[i+1]; - } - array[ALLOC_COUNT-1].next = free_; - free_ = &array[0]; - } - Entry* e = free_; - free_ = e->next; - e->key = key; - e->value = value; - e->next = c->blocks[block]; - c->blocks[block] = e; -} - -template -bool AddressMap::FindAndRemove(Key key, Value* removed_value) { - const Number num = reinterpret_cast(key); - Cluster* const c = FindCluster(num, false/*do not create*/); - if (c != NULL) { - for (Entry** p = &c->blocks[BlockID(num)]; *p != NULL; p = &(*p)->next) { - Entry* e = *p; - if (e->key == key) { - *removed_value = e->value; - *p = e->next; // Remove e from linked-list - e->next = free_; // Add e to free-list - free_ = e; - return true; - } - } - } - return false; -} - -template -const Value* AddressMap::FindInside(ValueSizeFunc size_func, - size_t max_size, - Key key, - Key* res_key) { - const Number key_num = reinterpret_cast(key); - Number num = key_num; // we'll move this to move back through the clusters - while (1) { - const Cluster* c = FindCluster(num, false/*do not create*/); - if (c != NULL) { - while (1) { - const int block = BlockID(num); - bool had_smaller_key = false; - for (const Entry* e = c->blocks[block]; e != NULL; e = e->next) { - const Number e_num = reinterpret_cast(e->key); - if (e_num <= key_num) { - if (e_num == key_num || // to handle 0-sized ranges - key_num < e_num + (*size_func)(e->value)) { - *res_key = e->key; - return &e->value; - } - had_smaller_key = true; - } - } - if (had_smaller_key) return NULL; // got a range before 'key' - // and it did not contain 'key' - if (block == 0) break; - // try address-wise previous block - num |= kBlockSize - 1; // start at the last addr of prev block - num -= kBlockSize; - if (key_num - num > max_size) return NULL; - } - } - if (num < kClusterSize) return NULL; // first cluster - // go to address-wise previous cluster to try - num |= kClusterSize - 1; // start at the last block of previous cluster - num -= kClusterSize; - if (key_num - num > max_size) return NULL; - // Having max_size to limit the search is crucial: else - // we have to traverse a lot of empty clusters (or blocks). - // We can avoid needing max_size if we put clusters into - // a search tree, but performance suffers considerably - // if we use this approach by using stl::set. - } -} - -template -template -inline void AddressMap::Iterate(void (*callback)(Key, Value*, Type), - Type arg) const { - // We could optimize this by traversing only non-empty clusters and/or blocks - // but it does not speed up heap-checker noticeably. - for (int h = 0; h < kHashSize; ++h) { - for (const Cluster* c = hashtable_[h]; c != NULL; c = c->next) { - for (int b = 0; b < kClusterBlocks; ++b) { - for (Entry* e = c->blocks[b]; e != NULL; e = e->next) { - callback(e->key, &e->value, arg); - } - } - } - } -} - -#endif // BASE_ADDRESSMAP_INL_H_ diff --git a/contrib/libtcmalloc/src/base/arm_instruction_set_select.h b/contrib/libtcmalloc/src/base/arm_instruction_set_select.h deleted file mode 100644 index 6fde685272c..00000000000 --- a/contrib/libtcmalloc/src/base/arm_instruction_set_select.h +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) 2011, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Author: Alexander Levitskiy -// -// Generalizes the plethora of ARM flavors available to an easier to manage set -// Defs reference is at https://wiki.edubuntu.org/ARM/Thumb2PortingHowto - -#ifndef ARM_INSTRUCTION_SET_SELECT_H_ -#define ARM_INSTRUCTION_SET_SELECT_H_ - -#if defined(__ARM_ARCH_8A__) -# define ARMV8 1 -#endif - -#if defined(ARMV8) || \ - defined(__ARM_ARCH_7__) || \ - defined(__ARM_ARCH_7R__) || \ - defined(__ARM_ARCH_7A__) -# define ARMV7 1 -#endif - -#if defined(ARMV7) || \ - defined(__ARM_ARCH_6__) || \ - defined(__ARM_ARCH_6J__) || \ - defined(__ARM_ARCH_6K__) || \ - defined(__ARM_ARCH_6Z__) || \ - defined(__ARM_ARCH_6T2__) || \ - defined(__ARM_ARCH_6ZK__) -# define ARMV6 1 -#endif - -#if defined(ARMV6) || \ - defined(__ARM_ARCH_5T__) || \ - defined(__ARM_ARCH_5E__) || \ - defined(__ARM_ARCH_5TE__) || \ - defined(__ARM_ARCH_5TEJ__) -# define ARMV5 1 -#endif - -#if defined(ARMV5) || \ - defined(__ARM_ARCH_4__) || \ - defined(__ARM_ARCH_4T__) -# define ARMV4 1 -#endif - -#if defined(ARMV4) || \ - defined(__ARM_ARCH_3__) || \ - defined(__ARM_ARCH_3M__) -# define ARMV3 1 -#endif - -#if defined(ARMV3) || \ - defined(__ARM_ARCH_2__) -# define ARMV2 1 -#endif - -#endif // ARM_INSTRUCTION_SET_SELECT_H_ diff --git a/contrib/libtcmalloc/src/base/atomicops-internals-arm-generic.h b/contrib/libtcmalloc/src/base/atomicops-internals-arm-generic.h deleted file mode 100644 index d0f941309bb..00000000000 --- a/contrib/libtcmalloc/src/base/atomicops-internals-arm-generic.h +++ /dev/null @@ -1,228 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2003, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// --- -// -// Author: Lei Zhang, Sasha Levitskiy -// -// This file is an internal atomic implementation, use base/atomicops.h instead. -// -// LinuxKernelCmpxchg is from Google Gears. - -#ifndef BASE_ATOMICOPS_INTERNALS_ARM_GENERIC_H_ -#define BASE_ATOMICOPS_INTERNALS_ARM_GENERIC_H_ - -#include -#include -#include "base/basictypes.h" - -typedef int32_t Atomic32; - -namespace base { -namespace subtle { - -typedef int64_t Atomic64; - -// 0xffff0fc0 is the hard coded address of a function provided by -// the kernel which implements an atomic compare-exchange. On older -// ARM architecture revisions (pre-v6) this may be implemented using -// a syscall. This address is stable, and in active use (hard coded) -// by at least glibc-2.7 and the Android C library. -// pLinuxKernelCmpxchg has both acquire and release barrier sematincs. -typedef Atomic32 (*LinuxKernelCmpxchgFunc)(Atomic32 old_value, - Atomic32 new_value, - volatile Atomic32* ptr); -LinuxKernelCmpxchgFunc pLinuxKernelCmpxchg ATTRIBUTE_WEAK = - (LinuxKernelCmpxchgFunc) 0xffff0fc0; - -typedef void (*LinuxKernelMemoryBarrierFunc)(void); -LinuxKernelMemoryBarrierFunc pLinuxKernelMemoryBarrier ATTRIBUTE_WEAK = - (LinuxKernelMemoryBarrierFunc) 0xffff0fa0; - - -inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 prev_value = *ptr; - do { - if (!pLinuxKernelCmpxchg(old_value, new_value, - const_cast(ptr))) { - return old_value; - } - prev_value = *ptr; - } while (prev_value == old_value); - return prev_value; -} - -inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - Atomic32 old_value; - do { - old_value = *ptr; - } while (pLinuxKernelCmpxchg(old_value, new_value, - const_cast(ptr))); - return old_value; -} - -inline Atomic32 Acquire_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - // pLinuxKernelCmpxchg already has acquire and release barrier semantics. - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic32 Release_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - // pLinuxKernelCmpxchg already has acquire and release barrier semantics. - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; -} - -inline void MemoryBarrier() { - pLinuxKernelMemoryBarrier(); -} - -inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { - MemoryBarrier(); - *ptr = value; -} - -inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { - return *ptr; -} - -inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { - Atomic32 value = *ptr; - MemoryBarrier(); - return value; -} - -inline Atomic32 Release_Load(volatile const Atomic32* ptr) { - MemoryBarrier(); - return *ptr; -} - - -// 64-bit versions are not implemented yet. - -inline void NotImplementedFatalError(const char *function_name) { - fprintf(stderr, "64-bit %s() not implemented on this platform\n", - function_name); - abort(); -} - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - NotImplementedFatalError("NoBarrier_CompareAndSwap"); - return 0; -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - NotImplementedFatalError("NoBarrier_AtomicExchange"); - return 0; -} - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - // pLinuxKernelCmpxchg already has acquire and release barrier semantics. - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - // pLinuxKernelCmpxchg already has acquire and release barrier semantics. - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - NotImplementedFatalError("NoBarrier_Store"); -} - -inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { - NotImplementedFatalError("Acquire_Store64"); -} - -inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { - NotImplementedFatalError("Release_Store"); -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - NotImplementedFatalError("NoBarrier_Load"); - return 0; -} - -inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { - NotImplementedFatalError("Atomic64 Acquire_Load"); - return 0; -} - -inline Atomic64 Release_Load(volatile const Atomic64* ptr) { - NotImplementedFatalError("Atomic64 Release_Load"); - return 0; -} - -inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - NotImplementedFatalError("Atomic64 Acquire_CompareAndSwap"); - return 0; -} - -inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - NotImplementedFatalError("Atomic64 Release_CompareAndSwap"); - return 0; -} - -} // namespace base::subtle -} // namespace base - -#endif // BASE_ATOMICOPS_INTERNALS_ARM_GENERIC_H_ diff --git a/contrib/libtcmalloc/src/base/atomicops-internals-arm-v6plus.h b/contrib/libtcmalloc/src/base/atomicops-internals-arm-v6plus.h deleted file mode 100644 index 35f10481b04..00000000000 --- a/contrib/libtcmalloc/src/base/atomicops-internals-arm-v6plus.h +++ /dev/null @@ -1,330 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2011, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// --- -// -// Author: Sasha Levitskiy -// based on atomicops-internals by Sanjay Ghemawat -// -// This file is an internal atomic implementation, use base/atomicops.h instead. -// -// This code implements ARM atomics for architectures V6 and newer. - -#ifndef BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ -#define BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ - -#include -#include -#include "base/basictypes.h" // For COMPILE_ASSERT - -// The LDREXD and STREXD instructions in ARM all v7 variants or above. In v6, -// only some variants support it. For simplicity, we only use exclusive -// 64-bit load/store in V7 or above. -#if defined(ARMV7) -# define BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD -#endif - -typedef int32_t Atomic32; - -namespace base { -namespace subtle { - -typedef int64_t Atomic64; - -// 32-bit low-level ops - -inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 oldval, res; - do { - __asm__ __volatile__( - "ldrex %1, [%3]\n" - "mov %0, #0\n" - "teq %1, %4\n" - // The following IT (if-then) instruction is needed for the subsequent - // conditional instruction STREXEQ when compiling in THUMB mode. - // In ARM mode, the compiler/assembler will not generate any code for it. - "it eq\n" - "strexeq %0, %5, [%3]\n" - : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr) - : "r" (ptr), "Ir" (old_value), "r" (new_value) - : "cc"); - } while (res); - return oldval; -} - -inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - Atomic32 tmp, old; - __asm__ __volatile__( - "1:\n" - "ldrex %1, [%2]\n" - "strex %0, %3, [%2]\n" - "teq %0, #0\n" - "bne 1b" - : "=&r" (tmp), "=&r" (old) - : "r" (ptr), "r" (new_value) - : "cc", "memory"); - return old; -} - -inline void MemoryBarrier() { -#if !defined(ARMV7) - uint32_t dest = 0; - __asm__ __volatile__("mcr p15,0,%0,c7,c10,5" :"=&r"(dest) : : "memory"); -#else - __asm__ __volatile__("dmb" : : : "memory"); -#endif -} - -inline Atomic32 Acquire_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - Atomic32 old_value = NoBarrier_AtomicExchange(ptr, new_value); - MemoryBarrier(); - return old_value; -} - -inline Atomic32 Release_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - MemoryBarrier(); - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); - MemoryBarrier(); - return value; -} - -inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - MemoryBarrier(); - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; -} - -inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { - MemoryBarrier(); - *ptr = value; -} - -inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { - return *ptr; -} - -inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { - Atomic32 value = *ptr; - MemoryBarrier(); - return value; -} - -inline Atomic32 Release_Load(volatile const Atomic32* ptr) { - MemoryBarrier(); - return *ptr; -} - -// 64-bit versions are only available if LDREXD and STREXD instructions -// are available. -#ifdef BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD - -#define BASE_HAS_ATOMIC64 1 - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 oldval, res; - do { - __asm__ __volatile__( - "ldrexd %1, [%3]\n" - "mov %0, #0\n" - "teq %Q1, %Q4\n" - // The following IT (if-then) instructions are needed for the subsequent - // conditional instructions when compiling in THUMB mode. - // In ARM mode, the compiler/assembler will not generate any code for it. - "it eq\n" - "teqeq %R1, %R4\n" - "it eq\n" - "strexdeq %0, %5, [%3]\n" - : "=&r" (res), "=&r" (oldval), "+Q" (*ptr) - : "r" (ptr), "Ir" (old_value), "r" (new_value) - : "cc"); - } while (res); - return oldval; -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - int store_failed; - Atomic64 old; - __asm__ __volatile__( - "1:\n" - "ldrexd %1, [%2]\n" - "strexd %0, %3, [%2]\n" - "teq %0, #0\n" - "bne 1b" - : "=&r" (store_failed), "=&r" (old) - : "r" (ptr), "r" (new_value) - : "cc", "memory"); - return old; -} - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - Atomic64 old_value = NoBarrier_AtomicExchange(ptr, new_value); - MemoryBarrier(); - return old_value; -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - MemoryBarrier(); - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - int store_failed; - Atomic64 dummy; - __asm__ __volatile__( - "1:\n" - // Dummy load to lock cache line. - "ldrexd %1, [%3]\n" - "strexd %0, %2, [%3]\n" - "teq %0, #0\n" - "bne 1b" - : "=&r" (store_failed), "=&r"(dummy) - : "r"(value), "r" (ptr) - : "cc", "memory"); -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - Atomic64 res; - __asm__ __volatile__( - "ldrexd %0, [%1]\n" - "clrex\n" - : "=r" (res) - : "r"(ptr), "Q"(*ptr)); - return res; -} - -#else // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD - -inline void NotImplementedFatalError(const char *function_name) { - fprintf(stderr, "64-bit %s() not implemented on this platform\n", - function_name); - abort(); -} - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - NotImplementedFatalError("NoBarrier_CompareAndSwap"); - return 0; -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - NotImplementedFatalError("NoBarrier_AtomicExchange"); - return 0; -} - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - NotImplementedFatalError("Acquire_AtomicExchange"); - return 0; -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - NotImplementedFatalError("Release_AtomicExchange"); - return 0; -} - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - NotImplementedFatalError("NoBarrier_Store"); -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - NotImplementedFatalError("NoBarrier_Load"); - return 0; -} - -#endif // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD - -inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { - NoBarrier_Store(ptr, value); - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { - MemoryBarrier(); - NoBarrier_Store(ptr, value); -} - -inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { - Atomic64 value = NoBarrier_Load(ptr); - MemoryBarrier(); - return value; -} - -inline Atomic64 Release_Load(volatile const Atomic64* ptr) { - MemoryBarrier(); - return NoBarrier_Load(ptr); -} - -inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); - MemoryBarrier(); - return value; -} - -inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - MemoryBarrier(); - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -} // namespace subtle ends -} // namespace base ends - -#endif // BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ diff --git a/contrib/libtcmalloc/src/base/atomicops-internals-gcc.h b/contrib/libtcmalloc/src/base/atomicops-internals-gcc.h deleted file mode 100644 index f8d27863cb7..00000000000 --- a/contrib/libtcmalloc/src/base/atomicops-internals-gcc.h +++ /dev/null @@ -1,203 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2014, Linaro -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// --- -// -// Author: Riku Voipio, riku.voipio@linaro.org -// -// atomic primitives implemented with gcc atomic intrinsics: -// http://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html -// - -#ifndef BASE_ATOMICOPS_INTERNALS_GCC_GENERIC_H_ -#define BASE_ATOMICOPS_INTERNALS_GCC_GENERIC_H_ - -#include -#include -#include "base/basictypes.h" - -typedef int32_t Atomic32; - -namespace base { -namespace subtle { - -typedef int64_t Atomic64; - -inline void MemoryBarrier() { - __sync_synchronize(); -} - -inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 prev_value = old_value; - __atomic_compare_exchange_n(ptr, &prev_value, new_value, - 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); - return prev_value; -} - -inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - return __atomic_exchange_n(const_cast(ptr), new_value, __ATOMIC_RELAXED); -} - -inline Atomic32 Acquire_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - return __atomic_exchange_n(const_cast(ptr), new_value, __ATOMIC_ACQUIRE); -} - -inline Atomic32 Release_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - return __atomic_exchange_n(const_cast(ptr), new_value, __ATOMIC_RELEASE); -} - -inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 prev_value = old_value; - __atomic_compare_exchange_n(ptr, &prev_value, new_value, - 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); - return prev_value; -} - -inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 prev_value = old_value; - __atomic_compare_exchange_n(ptr, &prev_value, new_value, - 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED); - return prev_value; -} - -inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; -} - -inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { - MemoryBarrier(); - *ptr = value; -} - -inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { - return *ptr; -} - -inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { - Atomic32 value = *ptr; - MemoryBarrier(); - return value; -} - -inline Atomic32 Release_Load(volatile const Atomic32* ptr) { - MemoryBarrier(); - return *ptr; -} - -// 64-bit versions - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 prev_value = old_value; - __atomic_compare_exchange_n(ptr, &prev_value, new_value, - 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); - return prev_value; -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - return __atomic_exchange_n(const_cast(ptr), new_value, __ATOMIC_RELAXED); -} - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - return __atomic_exchange_n(const_cast(ptr), new_value, __ATOMIC_ACQUIRE); -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - return __atomic_exchange_n(const_cast(ptr), new_value, __ATOMIC_RELEASE); -} - -inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 prev_value = old_value; - __atomic_compare_exchange_n(ptr, &prev_value, new_value, - 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); - return prev_value; -} - -inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 prev_value = old_value; - __atomic_compare_exchange_n(ptr, &prev_value, new_value, - 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED); - return prev_value; -} - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - *ptr = value; -} - -inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { - *ptr = value; - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { - MemoryBarrier(); - *ptr = value; -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - return *ptr; -} - -inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { - Atomic64 value = *ptr; - MemoryBarrier(); - return value; -} - -inline Atomic64 Release_Load(volatile const Atomic64* ptr) { - MemoryBarrier(); - return *ptr; -} - -} // namespace base::subtle -} // namespace base - -#endif // BASE_ATOMICOPS_INTERNALS_GCC_GENERIC_H_ diff --git a/contrib/libtcmalloc/src/base/atomicops-internals-linuxppc.h b/contrib/libtcmalloc/src/base/atomicops-internals-linuxppc.h deleted file mode 100644 index b52fdf0d1ec..00000000000 --- a/contrib/libtcmalloc/src/base/atomicops-internals-linuxppc.h +++ /dev/null @@ -1,437 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2008, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - */ - -// Implementation of atomic operations for ppc-linux. This file should not -// be included directly. Clients should instead include -// "base/atomicops.h". - -#ifndef BASE_ATOMICOPS_INTERNALS_LINUXPPC_H_ -#define BASE_ATOMICOPS_INTERNALS_LINUXPPC_H_ - -typedef int32_t Atomic32; - -#ifdef __PPC64__ -#define BASE_HAS_ATOMIC64 1 -#endif - -namespace base { -namespace subtle { - -static inline void _sync(void) { - __asm__ __volatile__("sync": : : "memory"); -} - -static inline void _lwsync(void) { - // gcc defines __NO_LWSYNC__ when appropriate; see - // http://gcc.gnu.org/ml/gcc-patches/2006-11/msg01238.html -#ifdef __NO_LWSYNC__ - __asm__ __volatile__("msync": : : "memory"); -#else - __asm__ __volatile__("lwsync": : : "memory"); -#endif -} - -static inline void _isync(void) { - __asm__ __volatile__("isync": : : "memory"); -} - -static inline Atomic32 OSAtomicAdd32(Atomic32 amount, Atomic32 *value) { - Atomic32 t; - __asm__ __volatile__( -"1: lwarx %0,0,%3\n\ - add %0,%2,%0\n\ - stwcx. %0,0,%3 \n\ - bne- 1b" - : "=&r" (t), "+m" (*value) - : "r" (amount), "r" (value) - : "cc"); - return t; -} - -static inline Atomic32 OSAtomicAdd32Barrier(Atomic32 amount, Atomic32 *value) { - Atomic32 t; - _lwsync(); - t = OSAtomicAdd32(amount, value); - // This is based on the code snippet in the architecture manual (Vol - // 2, Appendix B). It's a little tricky: correctness depends on the - // fact that the code right before this (in OSAtomicAdd32) has a - // conditional branch with a data dependency on the update. - // Otherwise, we'd have to use sync. - _isync(); - return t; -} - -static inline bool OSAtomicCompareAndSwap32(Atomic32 old_value, - Atomic32 new_value, - Atomic32 *value) { - Atomic32 prev; - __asm__ __volatile__( -"1: lwarx %0,0,%2\n\ - cmpw 0,%0,%3\n\ - bne- 2f\n\ - stwcx. %4,0,%2\n\ - bne- 1b\n\ -2:" - : "=&r" (prev), "+m" (*value) - : "r" (value), "r" (old_value), "r" (new_value) - : "cc"); - return prev == old_value; -} - -static inline Atomic32 OSAtomicCompareAndSwap32Acquire(Atomic32 old_value, - Atomic32 new_value, - Atomic32 *value) { - Atomic32 t; - t = OSAtomicCompareAndSwap32(old_value, new_value, value); - // This is based on the code snippet in the architecture manual (Vol - // 2, Appendix B). It's a little tricky: correctness depends on the - // fact that the code right before this (in - // OSAtomicCompareAndSwap32) has a conditional branch with a data - // dependency on the update. Otherwise, we'd have to use sync. - _isync(); - return t; -} - -static inline Atomic32 OSAtomicCompareAndSwap32Release(Atomic32 old_value, - Atomic32 new_value, - Atomic32 *value) { - _lwsync(); - return OSAtomicCompareAndSwap32(old_value, new_value, value); -} - -typedef int64_t Atomic64; - -inline void MemoryBarrier() { - // This can't be _lwsync(); we need to order the immediately - // preceding stores against any load that may follow, but lwsync - // doesn't guarantee that. - _sync(); -} - -// 32-bit Versions. - -inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32 *ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 prev_value; - do { - if (OSAtomicCompareAndSwap32(old_value, new_value, - const_cast(ptr))) { - return old_value; - } - prev_value = *ptr; - } while (prev_value == old_value); - return prev_value; -} - -inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32 *ptr, - Atomic32 new_value) { - Atomic32 old_value; - do { - old_value = *ptr; - } while (!OSAtomicCompareAndSwap32(old_value, new_value, - const_cast(ptr))); - return old_value; -} - -inline Atomic32 Acquire_AtomicExchange(volatile Atomic32 *ptr, - Atomic32 new_value) { - Atomic32 old_value; - do { - old_value = *ptr; - } while (!OSAtomicCompareAndSwap32Acquire(old_value, new_value, - const_cast(ptr))); - return old_value; -} - -inline Atomic32 Release_AtomicExchange(volatile Atomic32 *ptr, - Atomic32 new_value) { - Atomic32 old_value; - do { - old_value = *ptr; - } while (!OSAtomicCompareAndSwap32Release(old_value, new_value, - const_cast(ptr))); - return old_value; -} - -inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32 *ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 prev_value; - do { - if (OSAtomicCompareAndSwap32Acquire(old_value, new_value, - const_cast(ptr))) { - return old_value; - } - prev_value = *ptr; - } while (prev_value == old_value); - return prev_value; -} - -inline Atomic32 Release_CompareAndSwap(volatile Atomic32 *ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 prev_value; - do { - if (OSAtomicCompareAndSwap32Release(old_value, new_value, - const_cast(ptr))) { - return old_value; - } - prev_value = *ptr; - } while (prev_value == old_value); - return prev_value; -} - -#ifdef __PPC64__ - -// 64-bit Versions. - -static inline Atomic64 OSAtomicAdd64(Atomic64 amount, Atomic64 *value) { - Atomic64 t; - __asm__ __volatile__( -"1: ldarx %0,0,%3\n\ - add %0,%2,%0\n\ - stdcx. %0,0,%3 \n\ - bne- 1b" - : "=&r" (t), "+m" (*value) - : "r" (amount), "r" (value) - : "cc"); - return t; -} - -static inline Atomic64 OSAtomicAdd64Barrier(Atomic64 amount, Atomic64 *value) { - Atomic64 t; - _lwsync(); - t = OSAtomicAdd64(amount, value); - // This is based on the code snippet in the architecture manual (Vol - // 2, Appendix B). It's a little tricky: correctness depends on the - // fact that the code right before this (in OSAtomicAdd64) has a - // conditional branch with a data dependency on the update. - // Otherwise, we'd have to use sync. - _isync(); - return t; -} - -static inline bool OSAtomicCompareAndSwap64(Atomic64 old_value, - Atomic64 new_value, - Atomic64 *value) { - Atomic64 prev; - __asm__ __volatile__( -"1: ldarx %0,0,%2\n\ - cmpd 0,%0,%3\n\ - bne- 2f\n\ - stdcx. %4,0,%2\n\ - bne- 1b\n\ -2:" - : "=&r" (prev), "+m" (*value) - : "r" (value), "r" (old_value), "r" (new_value) - : "cc"); - return prev == old_value; -} - -static inline Atomic64 OSAtomicCompareAndSwap64Acquire(Atomic64 old_value, - Atomic64 new_value, - Atomic64 *value) { - Atomic64 t; - t = OSAtomicCompareAndSwap64(old_value, new_value, value); - // This is based on the code snippet in the architecture manual (Vol - // 2, Appendix B). It's a little tricky: correctness depends on the - // fact that the code right before this (in - // OSAtomicCompareAndSwap64) has a conditional branch with a data - // dependency on the update. Otherwise, we'd have to use sync. - _isync(); - return t; -} - -static inline Atomic64 OSAtomicCompareAndSwap64Release(Atomic64 old_value, - Atomic64 new_value, - Atomic64 *value) { - _lwsync(); - return OSAtomicCompareAndSwap64(old_value, new_value, value); -} - - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64 *ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 prev_value; - do { - if (OSAtomicCompareAndSwap64(old_value, new_value, - const_cast(ptr))) { - return old_value; - } - prev_value = *ptr; - } while (prev_value == old_value); - return prev_value; -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64 *ptr, - Atomic64 new_value) { - Atomic64 old_value; - do { - old_value = *ptr; - } while (!OSAtomicCompareAndSwap64(old_value, new_value, - const_cast(ptr))); - return old_value; -} - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64 *ptr, - Atomic64 new_value) { - Atomic64 old_value; - do { - old_value = *ptr; - } while (!OSAtomicCompareAndSwap64Acquire(old_value, new_value, - const_cast(ptr))); - return old_value; -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64 *ptr, - Atomic64 new_value) { - Atomic64 old_value; - do { - old_value = *ptr; - } while (!OSAtomicCompareAndSwap64Release(old_value, new_value, - const_cast(ptr))); - return old_value; -} - -inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64 *ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 prev_value; - do { - if (OSAtomicCompareAndSwap64Acquire(old_value, new_value, - const_cast(ptr))) { - return old_value; - } - prev_value = *ptr; - } while (prev_value == old_value); - return prev_value; -} - -inline Atomic64 Release_CompareAndSwap(volatile Atomic64 *ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 prev_value; - do { - if (OSAtomicCompareAndSwap64Release(old_value, new_value, - const_cast(ptr))) { - return old_value; - } - prev_value = *ptr; - } while (prev_value == old_value); - return prev_value; -} - -#endif - -inline void NoBarrier_Store(volatile Atomic32 *ptr, Atomic32 value) { - *ptr = value; -} - -inline void Acquire_Store(volatile Atomic32 *ptr, Atomic32 value) { - *ptr = value; - // This can't be _lwsync(); we need to order the immediately - // preceding stores against any load that may follow, but lwsync - // doesn't guarantee that. - _sync(); -} - -inline void Release_Store(volatile Atomic32 *ptr, Atomic32 value) { - _lwsync(); - *ptr = value; -} - -inline Atomic32 NoBarrier_Load(volatile const Atomic32 *ptr) { - return *ptr; -} - -inline Atomic32 Acquire_Load(volatile const Atomic32 *ptr) { - Atomic32 value = *ptr; - _lwsync(); - return value; -} - -inline Atomic32 Release_Load(volatile const Atomic32 *ptr) { - // This can't be _lwsync(); we need to order the immediately - // preceding stores against any load that may follow, but lwsync - // doesn't guarantee that. - _sync(); - return *ptr; -} - -#ifdef __PPC64__ - -// 64-bit Versions. - -inline void NoBarrier_Store(volatile Atomic64 *ptr, Atomic64 value) { - *ptr = value; -} - -inline void Acquire_Store(volatile Atomic64 *ptr, Atomic64 value) { - *ptr = value; - // This can't be _lwsync(); we need to order the immediately - // preceding stores against any load that may follow, but lwsync - // doesn't guarantee that. - _sync(); -} - -inline void Release_Store(volatile Atomic64 *ptr, Atomic64 value) { - _lwsync(); - *ptr = value; -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64 *ptr) { - return *ptr; -} - -inline Atomic64 Acquire_Load(volatile const Atomic64 *ptr) { - Atomic64 value = *ptr; - _lwsync(); - return value; -} - -inline Atomic64 Release_Load(volatile const Atomic64 *ptr) { - // This can't be _lwsync(); we need to order the immediately - // preceding stores against any load that may follow, but lwsync - // doesn't guarantee that. - _sync(); - return *ptr; -} - -#endif - -} // namespace base::subtle -} // namespace base - -#endif // BASE_ATOMICOPS_INTERNALS_LINUXPPC_H_ diff --git a/contrib/libtcmalloc/src/base/atomicops-internals-macosx.h b/contrib/libtcmalloc/src/base/atomicops-internals-macosx.h deleted file mode 100644 index b5130d4f4d7..00000000000 --- a/contrib/libtcmalloc/src/base/atomicops-internals-macosx.h +++ /dev/null @@ -1,370 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2006, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -// Implementation of atomic operations for Mac OS X. This file should not -// be included directly. Clients should instead include -// "base/atomicops.h". - -#ifndef BASE_ATOMICOPS_INTERNALS_MACOSX_H_ -#define BASE_ATOMICOPS_INTERNALS_MACOSX_H_ - -typedef int32_t Atomic32; - -// MacOS uses long for intptr_t, AtomicWord and Atomic32 are always different -// on the Mac, even when they are the same size. Similarly, on __ppc64__, -// AtomicWord and Atomic64 are always different. Thus, we need explicit -// casting. -#ifdef __LP64__ -#define AtomicWordCastType base::subtle::Atomic64 -#else -#define AtomicWordCastType Atomic32 -#endif - -#if defined(__LP64__) || defined(__i386__) -#define BASE_HAS_ATOMIC64 1 // Use only in tests and base/atomic* -#endif - -#include - -namespace base { -namespace subtle { - -#if !defined(__LP64__) && defined(__ppc__) - -// The Mac 64-bit OSAtomic implementations are not available for 32-bit PowerPC, -// while the underlying assembly instructions are available only some -// implementations of PowerPC. - -// The following inline functions will fail with the error message at compile -// time ONLY IF they are called. So it is safe to use this header if user -// code only calls AtomicWord and Atomic32 operations. -// -// NOTE(vchen): Implementation notes to implement the atomic ops below may -// be found in "PowerPC Virtual Environment Architecture, Book II, -// Version 2.02", January 28, 2005, Appendix B, page 46. Unfortunately, -// extra care must be taken to ensure data are properly 8-byte aligned, and -// that data are returned correctly according to Mac OS X ABI specs. - -inline int64_t OSAtomicCompareAndSwap64( - int64_t oldValue, int64_t newValue, int64_t *theValue) { - __asm__ __volatile__( - "_OSAtomicCompareAndSwap64_not_supported_for_32_bit_ppc\n\t"); - return 0; -} - -inline int64_t OSAtomicAdd64(int64_t theAmount, int64_t *theValue) { - __asm__ __volatile__( - "_OSAtomicAdd64_not_supported_for_32_bit_ppc\n\t"); - return 0; -} - -inline int64_t OSAtomicCompareAndSwap64Barrier( - int64_t oldValue, int64_t newValue, int64_t *theValue) { - int64_t prev = OSAtomicCompareAndSwap64(oldValue, newValue, theValue); - OSMemoryBarrier(); - return prev; -} - -inline int64_t OSAtomicAdd64Barrier( - int64_t theAmount, int64_t *theValue) { - int64_t new_val = OSAtomicAdd64(theAmount, theValue); - OSMemoryBarrier(); - return new_val; -} -#endif - -typedef int64_t Atomic64; - -inline void MemoryBarrier() { - OSMemoryBarrier(); -} - -// 32-bit Versions. - -inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32 *ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 prev_value; - do { - if (OSAtomicCompareAndSwap32(old_value, new_value, - const_cast(ptr))) { - return old_value; - } - prev_value = *ptr; - } while (prev_value == old_value); - return prev_value; -} - -inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32 *ptr, - Atomic32 new_value) { - Atomic32 old_value; - do { - old_value = *ptr; - } while (!OSAtomicCompareAndSwap32(old_value, new_value, - const_cast(ptr))); - return old_value; -} - -inline Atomic32 Acquire_AtomicExchange(volatile Atomic32 *ptr, - Atomic32 new_value) { - Atomic32 old_value; - do { - old_value = *ptr; - } while (!OSAtomicCompareAndSwap32Barrier(old_value, new_value, - const_cast(ptr))); - return old_value; -} - -inline Atomic32 Release_AtomicExchange(volatile Atomic32 *ptr, - Atomic32 new_value) { - return Acquire_AtomicExchange(ptr, new_value); -} - -inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32 *ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 prev_value; - do { - if (OSAtomicCompareAndSwap32Barrier(old_value, new_value, - const_cast(ptr))) { - return old_value; - } - prev_value = *ptr; - } while (prev_value == old_value); - return prev_value; -} - -inline Atomic32 Release_CompareAndSwap(volatile Atomic32 *ptr, - Atomic32 old_value, - Atomic32 new_value) { - return Acquire_CompareAndSwap(ptr, old_value, new_value); -} - -inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; -} - -inline void Acquire_Store(volatile Atomic32 *ptr, Atomic32 value) { - *ptr = value; - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic32 *ptr, Atomic32 value) { - MemoryBarrier(); - *ptr = value; -} - -inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { - return *ptr; -} - -inline Atomic32 Acquire_Load(volatile const Atomic32 *ptr) { - Atomic32 value = *ptr; - MemoryBarrier(); - return value; -} - -inline Atomic32 Release_Load(volatile const Atomic32 *ptr) { - MemoryBarrier(); - return *ptr; -} - -// 64-bit version - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64 *ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 prev_value; - do { - if (OSAtomicCompareAndSwap64(old_value, new_value, - const_cast(ptr))) { - return old_value; - } - prev_value = *ptr; - } while (prev_value == old_value); - return prev_value; -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64 *ptr, - Atomic64 new_value) { - Atomic64 old_value; - do { - old_value = *ptr; - } while (!OSAtomicCompareAndSwap64(old_value, new_value, - const_cast(ptr))); - return old_value; -} - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64 *ptr, - Atomic64 new_value) { - Atomic64 old_value; - do { - old_value = *ptr; - } while (!OSAtomicCompareAndSwap64Barrier(old_value, new_value, - const_cast(ptr))); - return old_value; -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64 *ptr, - Atomic64 new_value) { - return Acquire_AtomicExchange(ptr, new_value); -} - -inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64 *ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 prev_value; - do { - if (OSAtomicCompareAndSwap64Barrier(old_value, new_value, - const_cast(ptr))) { - return old_value; - } - prev_value = *ptr; - } while (prev_value == old_value); - return prev_value; -} - -inline Atomic64 Release_CompareAndSwap(volatile Atomic64 *ptr, - Atomic64 old_value, - Atomic64 new_value) { - // The lib kern interface does not distinguish between - // Acquire and Release memory barriers; they are equivalent. - return Acquire_CompareAndSwap(ptr, old_value, new_value); -} - -#ifdef __LP64__ - -// 64-bit implementation on 64-bit platform - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - *ptr = value; -} - -inline void Acquire_Store(volatile Atomic64 *ptr, Atomic64 value) { - *ptr = value; - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic64 *ptr, Atomic64 value) { - MemoryBarrier(); - *ptr = value; -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - return *ptr; -} - -inline Atomic64 Acquire_Load(volatile const Atomic64 *ptr) { - Atomic64 value = *ptr; - MemoryBarrier(); - return value; -} - -inline Atomic64 Release_Load(volatile const Atomic64 *ptr) { - MemoryBarrier(); - return *ptr; -} - -#else - -// 64-bit implementation on 32-bit platform - -#if defined(__ppc__) - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - __asm__ __volatile__( - "_NoBarrier_Store_not_supported_for_32_bit_ppc\n\t"); -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - __asm__ __volatile__( - "_NoBarrier_Load_not_supported_for_32_bit_ppc\n\t"); - return 0; -} - -#elif defined(__i386__) - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - __asm__ __volatile__("movq %1, %%mm0\n\t" // Use mmx reg for 64-bit atomic - "movq %%mm0, %0\n\t" // moves (ptr could be read-only) - "emms\n\t" // Reset FP registers - : "=m" (*ptr) - : "m" (value) - : // mark the FP stack and mmx registers as clobbered - "st", "st(1)", "st(2)", "st(3)", "st(4)", - "st(5)", "st(6)", "st(7)", "mm0", "mm1", - "mm2", "mm3", "mm4", "mm5", "mm6", "mm7"); - -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - Atomic64 value; - __asm__ __volatile__("movq %1, %%mm0\n\t" // Use mmx reg for 64-bit atomic - "movq %%mm0, %0\n\t" // moves (ptr could be read-only) - "emms\n\t" // Reset FP registers - : "=m" (value) - : "m" (*ptr) - : // mark the FP stack and mmx registers as clobbered - "st", "st(1)", "st(2)", "st(3)", "st(4)", - "st(5)", "st(6)", "st(7)", "mm0", "mm1", - "mm2", "mm3", "mm4", "mm5", "mm6", "mm7"); - - return value; -} -#endif - - -inline void Acquire_Store(volatile Atomic64 *ptr, Atomic64 value) { - NoBarrier_Store(ptr, value); - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic64 *ptr, Atomic64 value) { - MemoryBarrier(); - NoBarrier_Store(ptr, value); -} - -inline Atomic64 Acquire_Load(volatile const Atomic64 *ptr) { - Atomic64 value = NoBarrier_Load(ptr); - MemoryBarrier(); - return value; -} - -inline Atomic64 Release_Load(volatile const Atomic64 *ptr) { - MemoryBarrier(); - return NoBarrier_Load(ptr); -} -#endif // __LP64__ - -} // namespace base::subtle -} // namespace base - -#endif // BASE_ATOMICOPS_INTERNALS_MACOSX_H_ diff --git a/contrib/libtcmalloc/src/base/atomicops-internals-mips.h b/contrib/libtcmalloc/src/base/atomicops-internals-mips.h deleted file mode 100644 index 4bfd7f6c70d..00000000000 --- a/contrib/libtcmalloc/src/base/atomicops-internals-mips.h +++ /dev/null @@ -1,323 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2013, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -// Author: Jovan Zelincevic -// based on atomicops-internals by Sanjay Ghemawat - -// This file is an internal atomic implementation, use base/atomicops.h instead. -// -// This code implements MIPS atomics. - -#ifndef BASE_ATOMICOPS_INTERNALS_MIPS_H_ -#define BASE_ATOMICOPS_INTERNALS_MIPS_H_ - -#if (_MIPS_ISA == _MIPS_ISA_MIPS64) -#define BASE_HAS_ATOMIC64 1 -#endif - -typedef int32_t Atomic32; - -namespace base { -namespace subtle { - -// Atomically execute: -// result = *ptr; -// if (*ptr == old_value) -// *ptr = new_value; -// return result; -// -// I.e., replace "*ptr" with "new_value" if "*ptr" used to be "old_value". -// Always return the old value of "*ptr" -// -// This routine implies no memory barriers. -inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) -{ - Atomic32 prev, tmp; - __asm__ volatile( - ".set push \n" - ".set noreorder \n" - - "1: \n" - "ll %0, %5 \n" // prev = *ptr - "bne %0, %3, 2f \n" // if (prev != old_value) goto 2 - " move %2, %4 \n" // tmp = new_value - "sc %2, %1 \n" // *ptr = tmp (with atomic check) - "beqz %2, 1b \n" // start again on atomic error - " nop \n" // delay slot nop - "2: \n" - - ".set pop \n" - : "=&r" (prev), "=m" (*ptr), - "=&r" (tmp) - : "Ir" (old_value), "r" (new_value), - "m" (*ptr) - : "memory" - ); - return prev; -} - -// Atomically store new_value into *ptr, returning the previous value held in -// *ptr. This routine implies no memory barriers. -inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) -{ - Atomic32 temp, old; - __asm__ volatile( - ".set push \n" - ".set noreorder \n" - - "1: \n" - "ll %1, %2 \n" // old = *ptr - "move %0, %3 \n" // temp = new_value - "sc %0, %2 \n" // *ptr = temp (with atomic check) - "beqz %0, 1b \n" // start again on atomic error - " nop \n" // delay slot nop - - ".set pop \n" - : "=&r" (temp), "=&r" (old), - "=m" (*ptr) - : "r" (new_value), "m" (*ptr) - : "memory" - ); - return old; -} - -inline void MemoryBarrier() -{ - __asm__ volatile("sync" : : : "memory"); -} - -// "Acquire" operations -// ensure that no later memory access can be reordered ahead of the operation. -// "Release" operations ensure that no previous memory access can be reordered -// after the operation. "Barrier" operations have both "Acquire" and "Release" -// semantics. A MemoryBarrier() has "Barrier" semantics, but does no memory -// access. -inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) -{ - Atomic32 res = NoBarrier_CompareAndSwap(ptr, old_value, new_value); - MemoryBarrier(); - return res; -} - -inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) -{ - MemoryBarrier(); - Atomic32 res = NoBarrier_CompareAndSwap(ptr, old_value, new_value); - return res; -} - -inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) -{ - *ptr = value; -} - -inline Atomic32 Acquire_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) -{ - Atomic32 old_value = NoBarrier_AtomicExchange(ptr, new_value); - MemoryBarrier(); - return old_value; -} - -inline Atomic32 Release_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) -{ - MemoryBarrier(); - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) -{ - *ptr = value; - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) -{ - MemoryBarrier(); - *ptr = value; -} - -inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) -{ - return *ptr; -} - -inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) -{ - Atomic32 value = *ptr; - MemoryBarrier(); - return value; -} - -inline Atomic32 Release_Load(volatile const Atomic32* ptr) -{ - MemoryBarrier(); - return *ptr; -} - -#if (_MIPS_ISA == _MIPS_ISA_MIPS64) || (_MIPS_SIM == _MIPS_SIM_ABI64) - -typedef int64_t Atomic64; - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) -{ - Atomic64 prev, tmp; - __asm__ volatile( - ".set push \n" - ".set noreorder \n" - - "1: \n" - "lld %0, %5 \n" // prev = *ptr - "bne %0, %3, 2f \n" // if (prev != old_value) goto 2 - " move %2, %4 \n" // tmp = new_value - "scd %2, %1 \n" // *ptr = tmp (with atomic check) - "beqz %2, 1b \n" // start again on atomic error - " nop \n" // delay slot nop - "2: \n" - - ".set pop \n" - : "=&r" (prev), "=m" (*ptr), - "=&r" (tmp) - : "Ir" (old_value), "r" (new_value), - "m" (*ptr) - : "memory" - ); - return prev; -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) -{ - Atomic64 temp, old; - __asm__ volatile( - ".set push \n" - ".set noreorder \n" - - "1: \n" - "lld %1, %2 \n" // old = *ptr - "move %0, %3 \n" // temp = new_value - "scd %0, %2 \n" // *ptr = temp (with atomic check) - "beqz %0, 1b \n" // start again on atomic error - " nop \n" // delay slot nop - - ".set pop \n" - : "=&r" (temp), "=&r" (old), - "=m" (*ptr) - : "r" (new_value), "m" (*ptr) - : "memory" - ); - return old; -} - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) -{ - Atomic64 old_value = NoBarrier_AtomicExchange(ptr, new_value); - MemoryBarrier(); - return old_value; -} - -inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) -{ - Atomic64 res = NoBarrier_CompareAndSwap(ptr, old_value, new_value); - MemoryBarrier(); - return res; -} - -inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) -{ - MemoryBarrier(); - Atomic64 res = NoBarrier_CompareAndSwap(ptr, old_value, new_value); - return res; -} - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) -{ - *ptr = value; -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) -{ - MemoryBarrier(); - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) -{ - *ptr = value; - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) -{ - MemoryBarrier(); - *ptr = value; -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) -{ - return *ptr; -} - -inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) -{ - Atomic64 value = *ptr; - MemoryBarrier(); - return value; -} - -inline Atomic64 Release_Load(volatile const Atomic64* ptr) -{ - MemoryBarrier(); - return *ptr; -} - -#endif - -} // namespace base::subtle -} // namespace base - -#endif // BASE_ATOMICOPS_INTERNALS_MIPS_H_ diff --git a/contrib/libtcmalloc/src/base/atomicops-internals-windows.h b/contrib/libtcmalloc/src/base/atomicops-internals-windows.h deleted file mode 100644 index 93ced8770d4..00000000000 --- a/contrib/libtcmalloc/src/base/atomicops-internals-windows.h +++ /dev/null @@ -1,457 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2006, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Sanjay Ghemawat - */ - -// Implementation of atomic operations using Windows API -// functions. This file should not be included directly. Clients -// should instead include "base/atomicops.h". - -#ifndef BASE_ATOMICOPS_INTERNALS_WINDOWS_H_ -#define BASE_ATOMICOPS_INTERNALS_WINDOWS_H_ - -#include -#include -#include "base/basictypes.h" // For COMPILE_ASSERT - -typedef int32 Atomic32; - -#if defined(_WIN64) -#define BASE_HAS_ATOMIC64 1 // Use only in tests and base/atomic* -#endif - -namespace base { -namespace subtle { - -typedef int64 Atomic64; - -// 32-bit low-level operations on any platform - -extern "C" { -// We use windows intrinsics when we can (they seem to be supported -// well on MSVC 8.0 and above). Unfortunately, in some -// environments, and have conflicting -// declarations of some other intrinsics, breaking compilation: -// http://connect.microsoft.com/VisualStudio/feedback/details/262047 -// Therefore, we simply declare the relevant intrinsics ourself. - -// MinGW has a bug in the header files where it doesn't indicate the -// first argument is volatile -- they're not up to date. See -// http://readlist.com/lists/lists.sourceforge.net/mingw-users/0/3861.html -// We have to const_cast away the volatile to avoid compiler warnings. -// TODO(csilvers): remove this once MinGW has updated MinGW/include/winbase.h -#if defined(__MINGW32__) -inline LONG FastInterlockedCompareExchange(volatile LONG* ptr, - LONG newval, LONG oldval) { - return ::InterlockedCompareExchange(const_cast(ptr), newval, oldval); -} -inline LONG FastInterlockedExchange(volatile LONG* ptr, LONG newval) { - return ::InterlockedExchange(const_cast(ptr), newval); -} -inline LONG FastInterlockedExchangeAdd(volatile LONG* ptr, LONG increment) { - return ::InterlockedExchangeAdd(const_cast(ptr), increment); -} - -#elif _MSC_VER >= 1400 // intrinsics didn't work so well before MSVC 8.0 -// Unfortunately, in some environments, and -// have conflicting declarations of some intrinsics, breaking -// compilation. So we declare the intrinsics we need ourselves. See -// http://connect.microsoft.com/VisualStudio/feedback/details/262047 -LONG _InterlockedCompareExchange(volatile LONG* ptr, LONG newval, LONG oldval); -#pragma intrinsic(_InterlockedCompareExchange) -inline LONG FastInterlockedCompareExchange(volatile LONG* ptr, - LONG newval, LONG oldval) { - return _InterlockedCompareExchange(ptr, newval, oldval); -} - -LONG _InterlockedExchange(volatile LONG* ptr, LONG newval); -#pragma intrinsic(_InterlockedExchange) -inline LONG FastInterlockedExchange(volatile LONG* ptr, LONG newval) { - return _InterlockedExchange(ptr, newval); -} - -LONG _InterlockedExchangeAdd(volatile LONG* ptr, LONG increment); -#pragma intrinsic(_InterlockedExchangeAdd) -inline LONG FastInterlockedExchangeAdd(volatile LONG* ptr, LONG increment) { - return _InterlockedExchangeAdd(ptr, increment); -} - -#else -inline LONG FastInterlockedCompareExchange(volatile LONG* ptr, - LONG newval, LONG oldval) { - return ::InterlockedCompareExchange(ptr, newval, oldval); -} -inline LONG FastInterlockedExchange(volatile LONG* ptr, LONG newval) { - return ::InterlockedExchange(ptr, newval); -} -inline LONG FastInterlockedExchangeAdd(volatile LONG* ptr, LONG increment) { - return ::InterlockedExchangeAdd(ptr, increment); -} - -#endif // ifdef __MINGW32__ -} // extern "C" - -inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - LONG result = FastInterlockedCompareExchange( - reinterpret_cast(ptr), - static_cast(new_value), - static_cast(old_value)); - return static_cast(result); -} - -inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - LONG result = FastInterlockedExchange( - reinterpret_cast(ptr), - static_cast(new_value)); - return static_cast(result); -} - -inline Atomic32 Acquire_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - // FastInterlockedExchange has both acquire and release memory barriers. - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic32 Release_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - // FastInterlockedExchange has both acquire and release memory barriers. - return NoBarrier_AtomicExchange(ptr, new_value); -} - -} // namespace base::subtle -} // namespace base - - -// In msvc8/vs2005, winnt.h already contains a definition for -// MemoryBarrier in the global namespace. Add it there for earlier -// versions and forward to it from within the namespace. -#if !(defined(_MSC_VER) && _MSC_VER >= 1400) -inline void MemoryBarrier() { - Atomic32 value = 0; - base::subtle::NoBarrier_AtomicExchange(&value, 0); - // actually acts as a barrier in thisd implementation -} -#endif - -namespace base { -namespace subtle { - -inline void MemoryBarrier() { - ::MemoryBarrier(); -} - -inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; -} - -inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { - Acquire_AtomicExchange(ptr, value); -} - -inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; // works w/o barrier for current Intel chips as of June 2005 - // See comments in Atomic64 version of Release_Store() below. -} - -inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { - return *ptr; -} - -inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { - Atomic32 value = *ptr; - return value; -} - -inline Atomic32 Release_Load(volatile const Atomic32* ptr) { - MemoryBarrier(); - return *ptr; -} - -// 64-bit operations - -#if defined(_WIN64) || defined(__MINGW64__) - -// 64-bit low-level operations on 64-bit platform. - -COMPILE_ASSERT(sizeof(Atomic64) == sizeof(PVOID), atomic_word_is_atomic); - -// These are the intrinsics needed for 64-bit operations. Similar to the -// 32-bit case above. - -extern "C" { -#if defined(__MINGW64__) -inline PVOID FastInterlockedCompareExchangePointer(volatile PVOID* ptr, - PVOID newval, PVOID oldval) { - return ::InterlockedCompareExchangePointer(const_cast(ptr), - newval, oldval); -} -inline PVOID FastInterlockedExchangePointer(volatile PVOID* ptr, PVOID newval) { - return ::InterlockedExchangePointer(const_cast(ptr), newval); -} -inline LONGLONG FastInterlockedExchangeAdd64(volatile LONGLONG* ptr, - LONGLONG increment) { - return ::InterlockedExchangeAdd64(const_cast(ptr), increment); -} - -#elif _MSC_VER >= 1400 // intrinsics didn't work so well before MSVC 8.0 -// Like above, we need to declare the intrinsics ourselves. -PVOID _InterlockedCompareExchangePointer(volatile PVOID* ptr, - PVOID newval, PVOID oldval); -#pragma intrinsic(_InterlockedCompareExchangePointer) -inline PVOID FastInterlockedCompareExchangePointer(volatile PVOID* ptr, - PVOID newval, PVOID oldval) { - return _InterlockedCompareExchangePointer(const_cast(ptr), - newval, oldval); -} - -PVOID _InterlockedExchangePointer(volatile PVOID* ptr, PVOID newval); -#pragma intrinsic(_InterlockedExchangePointer) -inline PVOID FastInterlockedExchangePointer(volatile PVOID* ptr, PVOID newval) { - return _InterlockedExchangePointer(const_cast(ptr), newval); -} - -LONGLONG _InterlockedExchangeAdd64(volatile LONGLONG* ptr, LONGLONG increment); -#pragma intrinsic(_InterlockedExchangeAdd64) -inline LONGLONG FastInterlockedExchangeAdd64(volatile LONGLONG* ptr, - LONGLONG increment) { - return _InterlockedExchangeAdd64(const_cast(ptr), increment); -} - -#else -inline PVOID FastInterlockedCompareExchangePointer(volatile PVOID* ptr, - PVOID newval, PVOID oldval) { - return ::InterlockedCompareExchangePointer(ptr, newval, oldval); -} -inline PVOID FastInterlockedExchangePointer(volatile PVOID* ptr, PVOID newval) { - return ::InterlockedExchangePointer(ptr, newval); -} -inline LONGLONG FastInterlockedExchangeAdd64(volatile LONGLONG* ptr, - LONGLONG increment) { - return ::InterlockedExchangeAdd64(ptr, increment); -} - -#endif // ifdef __MINGW64__ -} // extern "C" - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - PVOID result = FastInterlockedCompareExchangePointer( - reinterpret_cast(ptr), - reinterpret_cast(new_value), reinterpret_cast(old_value)); - return reinterpret_cast(result); -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - PVOID result = FastInterlockedExchangePointer( - reinterpret_cast(ptr), - reinterpret_cast(new_value)); - return reinterpret_cast(result); -} - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - *ptr = value; -} - -inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { - NoBarrier_AtomicExchange(ptr, value); - // acts as a barrier in this implementation -} - -inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { - *ptr = value; // works w/o barrier for current Intel chips as of June 2005 - - // When new chips come out, check: - // IA-32 Intel Architecture Software Developer's Manual, Volume 3: - // System Programming Guide, Chatper 7: Multiple-processor management, - // Section 7.2, Memory Ordering. - // Last seen at: - // http://developer.intel.com/design/pentium4/manuals/index_new.htm -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - return *ptr; -} - -inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { - Atomic64 value = *ptr; - return value; -} - -inline Atomic64 Release_Load(volatile const Atomic64* ptr) { - MemoryBarrier(); - return *ptr; -} - -#else // defined(_WIN64) || defined(__MINGW64__) - -// 64-bit low-level operations on 32-bit platform - -// TODO(vchen): The GNU assembly below must be converted to MSVC inline -// assembly. Then the file should be renamed to ...-x86-msvc.h, probably. - -inline void NotImplementedFatalError(const char *function_name) { - fprintf(stderr, "64-bit %s() not implemented on this platform\n", - function_name); - abort(); -} - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { -#if 0 // Not implemented - Atomic64 prev; - __asm__ __volatile__("movl (%3), %%ebx\n\t" // Move 64-bit new_value into - "movl 4(%3), %%ecx\n\t" // ecx:ebx - "lock; cmpxchg8b %1\n\t" // If edx:eax (old_value) same - : "=A" (prev) // as contents of ptr: - : "m" (*ptr), // ecx:ebx => ptr - "0" (old_value), // else: - "r" (&new_value) // old *ptr => edx:eax - : "memory", "%ebx", "%ecx"); - return prev; -#else - NotImplementedFatalError("NoBarrier_CompareAndSwap"); - return 0; -#endif -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { -#if 0 // Not implemented - __asm__ __volatile__( - "movl (%2), %%ebx\n\t" // Move 64-bit new_value into - "movl 4(%2), %%ecx\n\t" // ecx:ebx - "0:\n\t" - "movl %1, %%eax\n\t" // Read contents of ptr into - "movl 4%1, %%edx\n\t" // edx:eax - "lock; cmpxchg8b %1\n\t" // Attempt cmpxchg; if *ptr - "jnz 0b\n\t" // is no longer edx:eax, loop - : "=A" (new_value) - : "m" (*ptr), - "r" (&new_value) - : "memory", "%ebx", "%ecx"); - return new_value; // Now it's the previous value. -#else - NotImplementedFatalError("NoBarrier_AtomicExchange"); - return 0; -#endif -} - -inline void NoBarrier_Store(volatile Atomic64* ptrValue, Atomic64 value) -{ - __asm { - movq mm0, value; // Use mmx reg for 64-bit atomic moves - mov eax, ptrValue; - movq [eax], mm0; - emms; // Empty mmx state to enable FP registers - } -} - -inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { - NoBarrier_AtomicExchange(ptr, value); - // acts as a barrier in this implementation -} - -inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { - NoBarrier_Store(ptr, value); -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptrValue) -{ - Atomic64 value; - __asm { - mov eax, ptrValue; - movq mm0, [eax]; // Use mmx reg for 64-bit atomic moves - movq value, mm0; - emms; // Empty mmx state to enable FP registers - } - return value; -} - -inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { - Atomic64 value = NoBarrier_Load(ptr); - return value; -} - -inline Atomic64 Release_Load(volatile const Atomic64* ptr) { - MemoryBarrier(); - return NoBarrier_Load(ptr); -} - -#endif // defined(_WIN64) || defined(__MINGW64__) - - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - // FastInterlockedExchange has both acquire and release memory barriers. - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - // FastInterlockedExchange has both acquire and release memory barriers. - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -} // namespace base::subtle -} // namespace base - -#endif // BASE_ATOMICOPS_INTERNALS_WINDOWS_H_ diff --git a/contrib/libtcmalloc/src/base/atomicops-internals-x86.cc b/contrib/libtcmalloc/src/base/atomicops-internals-x86.cc deleted file mode 100644 index c3391e78234..00000000000 --- a/contrib/libtcmalloc/src/base/atomicops-internals-x86.cc +++ /dev/null @@ -1,112 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2007, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * This module gets enough CPU information to optimize the - * atomicops module on x86. - */ - -#include "base/atomicops.h" -#include "base/basictypes.h" -#include "base/googleinit.h" -#include "base/logging.h" -#include - -// This file only makes sense with atomicops-internals-x86.h -- it -// depends on structs that are defined in that file. If atomicops.h -// doesn't sub-include that file, then we aren't needed, and shouldn't -// try to do anything. -#ifdef BASE_ATOMICOPS_INTERNALS_X86_H_ - -// Inline cpuid instruction. In PIC compilations, %ebx contains the address -// of the global offset table. To avoid breaking such executables, this code -// must preserve that register's value across cpuid instructions. -#if defined(__i386__) -#define cpuid(a, b, c, d, inp) \ - asm ("mov %%ebx, %%edi\n" \ - "cpuid\n" \ - "xchg %%edi, %%ebx\n" \ - : "=a" (a), "=D" (b), "=c" (c), "=d" (d) : "a" (inp)) -#elif defined (__x86_64__) -#define cpuid(a, b, c, d, inp) \ - asm ("mov %%rbx, %%rdi\n" \ - "cpuid\n" \ - "xchg %%rdi, %%rbx\n" \ - : "=a" (a), "=D" (b), "=c" (c), "=d" (d) : "a" (inp)) -#endif - -#if defined(cpuid) // initialize the struct only on x86 - -// Set the flags so that code will run correctly and conservatively -// until InitGoogle() is called. -struct AtomicOps_x86CPUFeatureStruct AtomicOps_Internalx86CPUFeatures = { - false, // no SSE2 - false // no cmpxchg16b -}; - -// Initialize the AtomicOps_Internalx86CPUFeatures struct. -static void AtomicOps_Internalx86CPUFeaturesInit() { - uint32 eax; - uint32 ebx; - uint32 ecx; - uint32 edx; - - // Get vendor string (issue CPUID with eax = 0) - cpuid(eax, ebx, ecx, edx, 0); - char vendor[13]; - memcpy(vendor, &ebx, 4); - memcpy(vendor + 4, &edx, 4); - memcpy(vendor + 8, &ecx, 4); - vendor[12] = 0; - - // get feature flags in ecx/edx, and family/model in eax - cpuid(eax, ebx, ecx, edx, 1); - - int family = (eax >> 8) & 0xf; // family and model fields - int model = (eax >> 4) & 0xf; - if (family == 0xf) { // use extended family and model fields - family += (eax >> 20) & 0xff; - model += ((eax >> 16) & 0xf) << 4; - } - - // edx bit 26 is SSE2 which we use to tell use whether we can use mfence - AtomicOps_Internalx86CPUFeatures.has_sse2 = ((edx >> 26) & 1); - - // ecx bit 13 indicates whether the cmpxchg16b instruction is supported - AtomicOps_Internalx86CPUFeatures.has_cmpxchg16b = ((ecx >> 13) & 1); -} - -REGISTER_MODULE_INITIALIZER(atomicops_x86, { - AtomicOps_Internalx86CPUFeaturesInit(); -}); - -#endif - -#endif /* ifdef BASE_ATOMICOPS_INTERNALS_X86_H_ */ diff --git a/contrib/libtcmalloc/src/base/atomicops-internals-x86.h b/contrib/libtcmalloc/src/base/atomicops-internals-x86.h deleted file mode 100644 index e441ac7e673..00000000000 --- a/contrib/libtcmalloc/src/base/atomicops-internals-x86.h +++ /dev/null @@ -1,391 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2006, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Sanjay Ghemawat - */ - -// Implementation of atomic operations for x86. This file should not -// be included directly. Clients should instead include -// "base/atomicops.h". - -#ifndef BASE_ATOMICOPS_INTERNALS_X86_H_ -#define BASE_ATOMICOPS_INTERNALS_X86_H_ -#include "base/basictypes.h" - -typedef int32_t Atomic32; -#define BASE_HAS_ATOMIC64 1 // Use only in tests and base/atomic* - - -// NOTE(vchen): x86 does not need to define AtomicWordCastType, because it -// already matches Atomic32 or Atomic64, depending on the platform. - - -// This struct is not part of the public API of this module; clients may not -// use it. -// Features of this x86. Values may not be correct before main() is run, -// but are set conservatively. -struct AtomicOps_x86CPUFeatureStruct { - bool has_sse2; // Processor has SSE2. - bool has_cmpxchg16b; // Processor supports cmpxchg16b instruction. -}; - -ATTRIBUTE_VISIBILITY_HIDDEN -extern struct AtomicOps_x86CPUFeatureStruct AtomicOps_Internalx86CPUFeatures; - - -#define ATOMICOPS_COMPILER_BARRIER() __asm__ __volatile__("" : : : "memory") - - -namespace base { -namespace subtle { - -typedef int64_t Atomic64; - -// 32-bit low-level operations on any platform. - -inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 prev; - __asm__ __volatile__("lock; cmpxchgl %1,%2" - : "=a" (prev) - : "q" (new_value), "m" (*ptr), "0" (old_value) - : "memory"); - return prev; -} - -inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - __asm__ __volatile__("xchgl %1,%0" // The lock prefix is implicit for xchg. - : "=r" (new_value) - : "m" (*ptr), "0" (new_value) - : "memory"); - return new_value; // Now it's the previous value. -} - -inline Atomic32 Acquire_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - Atomic32 old_val = NoBarrier_AtomicExchange(ptr, new_value); - return old_val; -} - -inline Atomic32 Release_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - // xchgl already has release memory barrier semantics. - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 x = NoBarrier_CompareAndSwap(ptr, old_value, new_value); - return x; -} - -inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; -} - -#if defined(__x86_64__) - -// 64-bit implementations of memory barrier can be simpler, because it -// "mfence" is guaranteed to exist. -inline void MemoryBarrier() { - __asm__ __volatile__("mfence" : : : "memory"); -} - -inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; - MemoryBarrier(); -} - -#else - -inline void MemoryBarrier() { - if (AtomicOps_Internalx86CPUFeatures.has_sse2) { - __asm__ __volatile__("mfence" : : : "memory"); - } else { // mfence is faster but not present on PIII - Atomic32 x = 0; - Acquire_AtomicExchange(&x, 0); - } -} - -inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { - if (AtomicOps_Internalx86CPUFeatures.has_sse2) { - *ptr = value; - __asm__ __volatile__("mfence" : : : "memory"); - } else { - Acquire_AtomicExchange(ptr, value); - } -} -#endif - -inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { - ATOMICOPS_COMPILER_BARRIER(); - *ptr = value; // An x86 store acts as a release barrier. - // See comments in Atomic64 version of Release_Store(), below. -} - -inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { - return *ptr; -} - -inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { - Atomic32 value = *ptr; // An x86 load acts as a acquire barrier. - // See comments in Atomic64 version of Release_Store(), below. - ATOMICOPS_COMPILER_BARRIER(); - return value; -} - -inline Atomic32 Release_Load(volatile const Atomic32* ptr) { - MemoryBarrier(); - return *ptr; -} - -#if defined(__x86_64__) - -// 64-bit low-level operations on 64-bit platform. - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 prev; - __asm__ __volatile__("lock; cmpxchgq %1,%2" - : "=a" (prev) - : "q" (new_value), "m" (*ptr), "0" (old_value) - : "memory"); - return prev; -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - __asm__ __volatile__("xchgq %1,%0" // The lock prefix is implicit for xchg. - : "=r" (new_value) - : "m" (*ptr), "0" (new_value) - : "memory"); - return new_value; // Now it's the previous value. -} - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - Atomic64 old_val = NoBarrier_AtomicExchange(ptr, new_value); - return old_val; -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - // xchgq already has release memory barrier semantics. - return NoBarrier_AtomicExchange(ptr, new_value); -} - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - *ptr = value; -} - -inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { - *ptr = value; - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { - ATOMICOPS_COMPILER_BARRIER(); - - *ptr = value; // An x86 store acts as a release barrier - // for current AMD/Intel chips as of Jan 2008. - // See also Acquire_Load(), below. - - // When new chips come out, check: - // IA-32 Intel Architecture Software Developer's Manual, Volume 3: - // System Programming Guide, Chatper 7: Multiple-processor management, - // Section 7.2, Memory Ordering. - // Last seen at: - // http://developer.intel.com/design/pentium4/manuals/index_new.htm - // - // x86 stores/loads fail to act as barriers for a few instructions (clflush - // maskmovdqu maskmovq movntdq movnti movntpd movntps movntq) but these are - // not generated by the compiler, and are rare. Users of these instructions - // need to know about cache behaviour in any case since all of these involve - // either flushing cache lines or non-temporal cache hints. -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - return *ptr; -} - -inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { - Atomic64 value = *ptr; // An x86 load acts as a acquire barrier, - // for current AMD/Intel chips as of Jan 2008. - // See also Release_Store(), above. - ATOMICOPS_COMPILER_BARRIER(); - return value; -} - -inline Atomic64 Release_Load(volatile const Atomic64* ptr) { - MemoryBarrier(); - return *ptr; -} - -#else // defined(__x86_64__) - -// 64-bit low-level operations on 32-bit platform. - -#if !((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) -// For compilers older than gcc 4.1, we use inline asm. -// -// Potential pitfalls: -// -// 1. %ebx points to Global offset table (GOT) with -fPIC. -// We need to preserve this register. -// 2. When explicit registers are used in inline asm, the -// compiler may not be aware of it and might try to reuse -// the same register for another argument which has constraints -// that allow it ("r" for example). - -inline Atomic64 __sync_val_compare_and_swap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 prev; - __asm__ __volatile__("push %%ebx\n\t" - "movl (%3), %%ebx\n\t" // Move 64-bit new_value into - "movl 4(%3), %%ecx\n\t" // ecx:ebx - "lock; cmpxchg8b (%1)\n\t"// If edx:eax (old_value) same - "pop %%ebx\n\t" - : "=A" (prev) // as contents of ptr: - : "D" (ptr), // ecx:ebx => ptr - "0" (old_value), // else: - "S" (&new_value) // old *ptr => edx:eax - : "memory", "%ecx"); - return prev; -} -#endif // Compiler < gcc-4.1 - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_val, - Atomic64 new_val) { - return __sync_val_compare_and_swap(ptr, old_val, new_val); -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_val) { - Atomic64 old_val; - - do { - old_val = *ptr; - } while (__sync_val_compare_and_swap(ptr, old_val, new_val) != old_val); - - return old_val; -} - -inline Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_val) { - Atomic64 old_val = NoBarrier_AtomicExchange(ptr, new_val); - return old_val; -} - -inline Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_val) { - return NoBarrier_AtomicExchange(ptr, new_val); -} - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - __asm__ __volatile__("movq %1, %%mm0\n\t" // Use mmx reg for 64-bit atomic - "movq %%mm0, %0\n\t" // moves (ptr could be read-only) - "emms\n\t" // Empty mmx state/Reset FP regs - : "=m" (*ptr) - : "m" (value) - : // mark the FP stack and mmx registers as clobbered - "st", "st(1)", "st(2)", "st(3)", "st(4)", - "st(5)", "st(6)", "st(7)", "mm0", "mm1", - "mm2", "mm3", "mm4", "mm5", "mm6", "mm7"); -} - -inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { - NoBarrier_Store(ptr, value); - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { - ATOMICOPS_COMPILER_BARRIER(); - NoBarrier_Store(ptr, value); -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - Atomic64 value; - __asm__ __volatile__("movq %1, %%mm0\n\t" // Use mmx reg for 64-bit atomic - "movq %%mm0, %0\n\t" // moves (ptr could be read-only) - "emms\n\t" // Empty mmx state/Reset FP regs - : "=m" (value) - : "m" (*ptr) - : // mark the FP stack and mmx registers as clobbered - "st", "st(1)", "st(2)", "st(3)", "st(4)", - "st(5)", "st(6)", "st(7)", "mm0", "mm1", - "mm2", "mm3", "mm4", "mm5", "mm6", "mm7"); - return value; -} - -inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { - Atomic64 value = NoBarrier_Load(ptr); - ATOMICOPS_COMPILER_BARRIER(); - return value; -} - -inline Atomic64 Release_Load(volatile const Atomic64* ptr) { - MemoryBarrier(); - return NoBarrier_Load(ptr); -} - -#endif // defined(__x86_64__) - -inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - Atomic64 x = NoBarrier_CompareAndSwap(ptr, old_value, new_value); - return x; -} - -inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -} // namespace base::subtle -} // namespace base - -#undef ATOMICOPS_COMPILER_BARRIER - -#endif // BASE_ATOMICOPS_INTERNALS_X86_H_ diff --git a/contrib/libtcmalloc/src/base/atomicops.h b/contrib/libtcmalloc/src/base/atomicops.h deleted file mode 100644 index 46a4b9bb7a2..00000000000 --- a/contrib/libtcmalloc/src/base/atomicops.h +++ /dev/null @@ -1,399 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2006, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Sanjay Ghemawat - */ - -// For atomic operations on statistics counters, see atomic_stats_counter.h. -// For atomic operations on sequence numbers, see atomic_sequence_num.h. -// For atomic operations on reference counts, see atomic_refcount.h. - -// Some fast atomic operations -- typically with machine-dependent -// implementations. This file may need editing as Google code is -// ported to different architectures. - -// The routines exported by this module are subtle. If you use them, even if -// you get the code right, it will depend on careful reasoning about atomicity -// and memory ordering; it will be less readable, and harder to maintain. If -// you plan to use these routines, you should have a good reason, such as solid -// evidence that performance would otherwise suffer, or there being no -// alternative. You should assume only properties explicitly guaranteed by the -// specifications in this file. You are almost certainly _not_ writing code -// just for the x86; if you assume x86 semantics, x86 hardware bugs and -// implementations on other archtectures will cause your code to break. If you -// do not know what you are doing, avoid these routines, and use a Mutex. -// -// These following lower-level operations are typically useful only to people -// implementing higher-level synchronization operations like spinlocks, -// mutexes, and condition-variables. They combine CompareAndSwap(), a load, or -// a store with appropriate memory-ordering instructions. "Acquire" operations -// ensure that no later memory access can be reordered ahead of the operation. -// "Release" operations ensure that no previous memory access can be reordered -// after the operation. "Barrier" operations have both "Acquire" and "Release" -// semantics. A MemoryBarrier() has "Barrier" semantics, but does no memory -// access. -// -// It is incorrect to make direct assignments to/from an atomic variable. -// You should use one of the Load or Store routines. The NoBarrier -// versions are provided when no barriers are needed: -// NoBarrier_Store() -// NoBarrier_Load() -// Although there are currently no compiler enforcement, you are encouraged -// to use these. Moreover, if you choose to use base::subtle::Atomic64 type, -// you MUST use one of the Load or Store routines to get correct behavior -// on 32-bit platforms. -// -// The intent is eventually to put all of these routines in namespace -// base::subtle - -#ifndef THREAD_ATOMICOPS_H_ -#define THREAD_ATOMICOPS_H_ - -#include "../config.h" -#ifdef HAVE_STDINT_H -#include -#endif - -// ------------------------------------------------------------------------ -// Include the platform specific implementations of the types -// and operations listed below. Implementations are to provide Atomic32 -// and Atomic64 operations. If there is a mismatch between intptr_t and -// the Atomic32 or Atomic64 types for a platform, the platform-specific header -// should define the macro, AtomicWordCastType in a clause similar to the -// following: -// #if ...pointers are 64 bits... -// # define AtomicWordCastType base::subtle::Atomic64 -// #else -// # define AtomicWordCastType Atomic32 -// #endif -// TODO(csilvers): figure out ARCH_PIII/ARCH_K8 (perhaps via ./configure?) -// ------------------------------------------------------------------------ - -#include "base/arm_instruction_set_select.h" -#define GCC_VERSION (__GNUC__ * 10000 \ - + __GNUC_MINOR__ * 100 \ - + __GNUC_PATCHLEVEL__) - -#define CLANG_VERSION (__clang_major__ * 10000 \ - + __clang_minor__ * 100 \ - + __clang_patchlevel__) - -#if defined(TCMALLOC_PREFER_GCC_ATOMICS) && defined(__GNUC__) && GCC_VERSION >= 40700 -#include "base/atomicops-internals-gcc.h" -#elif defined(TCMALLOC_PREFER_GCC_ATOMICS) && defined(__clang__) && CLANG_VERSION >= 30400 -#include "base/atomicops-internals-gcc.h" -#elif defined(__MACH__) && defined(__APPLE__) -#include "base/atomicops-internals-macosx.h" -#elif defined(__GNUC__) && defined(ARMV6) -#include "base/atomicops-internals-arm-v6plus.h" -#elif defined(ARMV3) -#include "base/atomicops-internals-arm-generic.h" -#elif defined(__GNUC__) && (defined(__i386) || defined(__x86_64__)) -#include "base/atomicops-internals-x86.h" -#elif defined(_WIN32) -#include "base/atomicops-internals-windows.h" -#elif defined(__linux__) && defined(__PPC__) -#include "base/atomicops-internals-linuxppc.h" -#elif defined(__GNUC__) && defined(__mips__) -#include "base/atomicops-internals-mips.h" -#elif defined(__GNUC__) && GCC_VERSION >= 40700 -#include "base/atomicops-internals-gcc.h" -#elif defined(__clang__) && CLANG_VERSION >= 30400 -#include "base/atomicops-internals-gcc.h" -#else -#error You need to implement atomic operations for this architecture -#endif - -// Signed type that can hold a pointer and supports the atomic ops below, as -// well as atomic loads and stores. Instances must be naturally-aligned. -typedef intptr_t AtomicWord; - -#ifdef AtomicWordCastType -// ------------------------------------------------------------------------ -// This section is needed only when explicit type casting is required to -// cast AtomicWord to one of the basic atomic types (Atomic64 or Atomic32). -// It also serves to document the AtomicWord interface. -// ------------------------------------------------------------------------ - -namespace base { -namespace subtle { - -// Atomically execute: -// result = *ptr; -// if (*ptr == old_value) -// *ptr = new_value; -// return result; -// -// I.e., replace "*ptr" with "new_value" if "*ptr" used to be "old_value". -// Always return the old value of "*ptr" -// -// This routine implies no memory barriers. -inline AtomicWord NoBarrier_CompareAndSwap(volatile AtomicWord* ptr, - AtomicWord old_value, - AtomicWord new_value) { - return NoBarrier_CompareAndSwap( - reinterpret_cast(ptr), - old_value, new_value); -} - -// Atomically store new_value into *ptr, returning the previous value held in -// *ptr. This routine implies no memory barriers. -inline AtomicWord NoBarrier_AtomicExchange(volatile AtomicWord* ptr, - AtomicWord new_value) { - return NoBarrier_AtomicExchange( - reinterpret_cast(ptr), new_value); -} - -inline AtomicWord Acquire_AtomicExchange(volatile AtomicWord* ptr, - AtomicWord new_value) { - return Acquire_AtomicExchange( - reinterpret_cast(ptr), new_value); -} - -inline AtomicWord Release_AtomicExchange(volatile AtomicWord* ptr, - AtomicWord new_value) { - return Release_AtomicExchange( - reinterpret_cast(ptr), new_value); -} - -inline AtomicWord Acquire_CompareAndSwap(volatile AtomicWord* ptr, - AtomicWord old_value, - AtomicWord new_value) { - return base::subtle::Acquire_CompareAndSwap( - reinterpret_cast(ptr), - old_value, new_value); -} - -inline AtomicWord Release_CompareAndSwap(volatile AtomicWord* ptr, - AtomicWord old_value, - AtomicWord new_value) { - return base::subtle::Release_CompareAndSwap( - reinterpret_cast(ptr), - old_value, new_value); -} - -inline void NoBarrier_Store(volatile AtomicWord *ptr, AtomicWord value) { - NoBarrier_Store( - reinterpret_cast(ptr), value); -} - -inline void Acquire_Store(volatile AtomicWord* ptr, AtomicWord value) { - return base::subtle::Acquire_Store( - reinterpret_cast(ptr), value); -} - -inline void Release_Store(volatile AtomicWord* ptr, AtomicWord value) { - return base::subtle::Release_Store( - reinterpret_cast(ptr), value); -} - -inline AtomicWord NoBarrier_Load(volatile const AtomicWord *ptr) { - return NoBarrier_Load( - reinterpret_cast(ptr)); -} - -inline AtomicWord Acquire_Load(volatile const AtomicWord* ptr) { - return base::subtle::Acquire_Load( - reinterpret_cast(ptr)); -} - -inline AtomicWord Release_Load(volatile const AtomicWord* ptr) { - return base::subtle::Release_Load( - reinterpret_cast(ptr)); -} - -} // namespace base::subtle -} // namespace base -#endif // AtomicWordCastType - -// ------------------------------------------------------------------------ -// Commented out type definitions and method declarations for documentation -// of the interface provided by this module. -// ------------------------------------------------------------------------ - -#if 0 - -// Signed 32-bit type that supports the atomic ops below, as well as atomic -// loads and stores. Instances must be naturally aligned. This type differs -// from AtomicWord in 64-bit binaries where AtomicWord is 64-bits. -typedef int32_t Atomic32; - -// Corresponding operations on Atomic32 -namespace base { -namespace subtle { - -// Signed 64-bit type that supports the atomic ops below, as well as atomic -// loads and stores. Instances must be naturally aligned. This type differs -// from AtomicWord in 32-bit binaries where AtomicWord is 32-bits. -typedef int64_t Atomic64; - -Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value); -Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, Atomic32 new_value); -Atomic32 Acquire_AtomicExchange(volatile Atomic32* ptr, Atomic32 new_value); -Atomic32 Release_AtomicExchange(volatile Atomic32* ptr, Atomic32 new_value); -Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value); -Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value); -void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value); -void Acquire_Store(volatile Atomic32* ptr, Atomic32 value); -void Release_Store(volatile Atomic32* ptr, Atomic32 value); -Atomic32 NoBarrier_Load(volatile const Atomic32* ptr); -Atomic32 Acquire_Load(volatile const Atomic32* ptr); -Atomic32 Release_Load(volatile const Atomic32* ptr); - -// Corresponding operations on Atomic64 -Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value); -Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, Atomic64 new_value); -Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, Atomic64 new_value); -Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, Atomic64 new_value); - -Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value); -Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value); -void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value); -void Acquire_Store(volatile Atomic64* ptr, Atomic64 value); -void Release_Store(volatile Atomic64* ptr, Atomic64 value); -Atomic64 NoBarrier_Load(volatile const Atomic64* ptr); -Atomic64 Acquire_Load(volatile const Atomic64* ptr); -Atomic64 Release_Load(volatile const Atomic64* ptr); -} // namespace base::subtle -} // namespace base - -void MemoryBarrier(); - -#endif // 0 - - -// ------------------------------------------------------------------------ -// The following are to be deprecated when all uses have been changed to -// use the base::subtle namespace. -// ------------------------------------------------------------------------ - -#ifdef AtomicWordCastType -// AtomicWord versions to be deprecated -inline AtomicWord Acquire_CompareAndSwap(volatile AtomicWord* ptr, - AtomicWord old_value, - AtomicWord new_value) { - return base::subtle::Acquire_CompareAndSwap(ptr, old_value, new_value); -} - -inline AtomicWord Release_CompareAndSwap(volatile AtomicWord* ptr, - AtomicWord old_value, - AtomicWord new_value) { - return base::subtle::Release_CompareAndSwap(ptr, old_value, new_value); -} - -inline void Acquire_Store(volatile AtomicWord* ptr, AtomicWord value) { - return base::subtle::Acquire_Store(ptr, value); -} - -inline void Release_Store(volatile AtomicWord* ptr, AtomicWord value) { - return base::subtle::Release_Store(ptr, value); -} - -inline AtomicWord Acquire_Load(volatile const AtomicWord* ptr) { - return base::subtle::Acquire_Load(ptr); -} - -inline AtomicWord Release_Load(volatile const AtomicWord* ptr) { - return base::subtle::Release_Load(ptr); -} -#endif // AtomicWordCastType - -// 32-bit Acquire/Release operations to be deprecated. - -inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - return base::subtle::Acquire_CompareAndSwap(ptr, old_value, new_value); -} -inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - return base::subtle::Release_CompareAndSwap(ptr, old_value, new_value); -} -inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { - base::subtle::Acquire_Store(ptr, value); -} -inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { - return base::subtle::Release_Store(ptr, value); -} -inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { - return base::subtle::Acquire_Load(ptr); -} -inline Atomic32 Release_Load(volatile const Atomic32* ptr) { - return base::subtle::Release_Load(ptr); -} - -#ifdef BASE_HAS_ATOMIC64 - -// 64-bit Acquire/Release operations to be deprecated. - -inline base::subtle::Atomic64 Acquire_CompareAndSwap( - volatile base::subtle::Atomic64* ptr, - base::subtle::Atomic64 old_value, base::subtle::Atomic64 new_value) { - return base::subtle::Acquire_CompareAndSwap(ptr, old_value, new_value); -} -inline base::subtle::Atomic64 Release_CompareAndSwap( - volatile base::subtle::Atomic64* ptr, - base::subtle::Atomic64 old_value, base::subtle::Atomic64 new_value) { - return base::subtle::Release_CompareAndSwap(ptr, old_value, new_value); -} -inline void Acquire_Store( - volatile base::subtle::Atomic64* ptr, base::subtle::Atomic64 value) { - base::subtle::Acquire_Store(ptr, value); -} -inline void Release_Store( - volatile base::subtle::Atomic64* ptr, base::subtle::Atomic64 value) { - return base::subtle::Release_Store(ptr, value); -} -inline base::subtle::Atomic64 Acquire_Load( - volatile const base::subtle::Atomic64* ptr) { - return base::subtle::Acquire_Load(ptr); -} -inline base::subtle::Atomic64 Release_Load( - volatile const base::subtle::Atomic64* ptr) { - return base::subtle::Release_Load(ptr); -} - -#endif // BASE_HAS_ATOMIC64 - -#endif // THREAD_ATOMICOPS_H_ diff --git a/contrib/libtcmalloc/src/base/basictypes.h b/contrib/libtcmalloc/src/base/basictypes.h deleted file mode 100644 index a81d0466c27..00000000000 --- a/contrib/libtcmalloc/src/base/basictypes.h +++ /dev/null @@ -1,408 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#ifndef _BASICTYPES_H_ -#define _BASICTYPES_H_ - -#include "../config.h" -#include // for memcpy() -#ifdef HAVE_INTTYPES_H -#include // gets us PRId64, etc -#endif - -// To use this in an autoconf setting, make sure you run the following -// autoconf macros: -// AC_HEADER_STDC /* for stdint_h and inttypes_h */ -// AC_CHECK_TYPES([__int64]) /* defined in some windows platforms */ - -#ifdef HAVE_INTTYPES_H -#include // uint16_t might be here; PRId64 too. -#endif -#ifdef HAVE_STDINT_H -#include // to get uint16_t (ISO naming madness) -#endif -#include // our last best hope for uint16_t - -// Standard typedefs -// All Google code is compiled with -funsigned-char to make "char" -// unsigned. Google code therefore doesn't need a "uchar" type. -// TODO(csilvers): how do we make sure unsigned-char works on non-gcc systems? -typedef signed char schar; -typedef int8_t int8; -typedef int16_t int16; -typedef int32_t int32; -typedef int64_t int64; - -// NOTE: unsigned types are DANGEROUS in loops and other arithmetical -// places. Use the signed types unless your variable represents a bit -// pattern (eg a hash value) or you really need the extra bit. Do NOT -// use 'unsigned' to express "this value should always be positive"; -// use assertions for this. - -typedef uint8_t uint8; -typedef uint16_t uint16; -typedef uint32_t uint32; -typedef uint64_t uint64; - -const uint16 kuint16max = ( (uint16) 0xFFFF); -const uint32 kuint32max = ( (uint32) 0xFFFFFFFF); -const uint64 kuint64max = ( (((uint64) kuint32max) << 32) | kuint32max ); - -const int8 kint8max = ( ( int8) 0x7F); -const int16 kint16max = ( ( int16) 0x7FFF); -const int32 kint32max = ( ( int32) 0x7FFFFFFF); -const int64 kint64max = ( ((( int64) kint32max) << 32) | kuint32max ); - -const int8 kint8min = ( ( int8) 0x80); -const int16 kint16min = ( ( int16) 0x8000); -const int32 kint32min = ( ( int32) 0x80000000); -const int64 kint64min = ( (((uint64) kint32min) << 32) | 0 ); - -// Define the "portable" printf and scanf macros, if they're not -// already there (via the inttypes.h we #included above, hopefully). -// Mostly it's old systems that don't support inttypes.h, so we assume -// they're 32 bit. -#ifndef PRIx64 -#define PRIx64 "llx" -#endif -#ifndef SCNx64 -#define SCNx64 "llx" -#endif -#ifndef PRId64 -#define PRId64 "lld" -#endif -#ifndef SCNd64 -#define SCNd64 "lld" -#endif -#ifndef PRIu64 -#define PRIu64 "llu" -#endif -#ifndef PRIxPTR -#define PRIxPTR "lx" -#endif - -// Also allow for printing of a pthread_t. -#define GPRIuPTHREAD "lu" -#define GPRIxPTHREAD "lx" -#if defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__APPLE__) || defined(__FreeBSD__) -#define PRINTABLE_PTHREAD(pthreadt) reinterpret_cast(pthreadt) -#else -#define PRINTABLE_PTHREAD(pthreadt) pthreadt -#endif - -// A macro to disallow the evil copy constructor and operator= functions -// This should be used in the private: declarations for a class -#define DISALLOW_EVIL_CONSTRUCTORS(TypeName) \ - TypeName(const TypeName&); \ - void operator=(const TypeName&) - -// An alternate name that leaves out the moral judgment... :-) -#define DISALLOW_COPY_AND_ASSIGN(TypeName) DISALLOW_EVIL_CONSTRUCTORS(TypeName) - -// The COMPILE_ASSERT macro can be used to verify that a compile time -// expression is true. For example, you could use it to verify the -// size of a static array: -// -// COMPILE_ASSERT(sizeof(num_content_type_names) == sizeof(int), -// content_type_names_incorrect_size); -// -// or to make sure a struct is smaller than a certain size: -// -// COMPILE_ASSERT(sizeof(foo) < 128, foo_too_large); -// -// The second argument to the macro is the name of the variable. If -// the expression is false, most compilers will issue a warning/error -// containing the name of the variable. -// -// Implementation details of COMPILE_ASSERT: -// -// - COMPILE_ASSERT works by defining an array type that has -1 -// elements (and thus is invalid) when the expression is false. -// -// - The simpler definition -// -// #define COMPILE_ASSERT(expr, msg) typedef char msg[(expr) ? 1 : -1] -// -// does not work, as gcc supports variable-length arrays whose sizes -// are determined at run-time (this is gcc's extension and not part -// of the C++ standard). As a result, gcc fails to reject the -// following code with the simple definition: -// -// int foo; -// COMPILE_ASSERT(foo, msg); // not supposed to compile as foo is -// // not a compile-time constant. -// -// - By using the type CompileAssert<(bool(expr))>, we ensures that -// expr is a compile-time constant. (Template arguments must be -// determined at compile-time.) -// -// - The outter parentheses in CompileAssert<(bool(expr))> are necessary -// to work around a bug in gcc 3.4.4 and 4.0.1. If we had written -// -// CompileAssert -// -// instead, these compilers will refuse to compile -// -// COMPILE_ASSERT(5 > 0, some_message); -// -// (They seem to think the ">" in "5 > 0" marks the end of the -// template argument list.) -// -// - The array size is (bool(expr) ? 1 : -1), instead of simply -// -// ((expr) ? 1 : -1). -// -// This is to avoid running into a bug in MS VC 7.1, which -// causes ((0.0) ? 1 : -1) to incorrectly evaluate to 1. - -template -struct CompileAssert { -}; - -#ifdef HAVE___ATTRIBUTE__ -# define ATTRIBUTE_UNUSED __attribute__((unused)) -#else -# define ATTRIBUTE_UNUSED -#endif - -#if defined(HAVE___ATTRIBUTE__) && defined(HAVE_TLS) -#define ATTR_INITIAL_EXEC __attribute__ ((tls_model ("initial-exec"))) -#else -#define ATTR_INITIAL_EXEC -#endif - -#define COMPILE_ASSERT(expr, msg) \ - typedef CompileAssert<(bool(expr))> msg[bool(expr) ? 1 : -1] ATTRIBUTE_UNUSED - -#define arraysize(a) (sizeof(a) / sizeof(*(a))) - -#define OFFSETOF_MEMBER(strct, field) \ - (reinterpret_cast(&reinterpret_cast(16)->field) - \ - reinterpret_cast(16)) - -// bit_cast implements the equivalent of -// "*reinterpret_cast(&source)". -// -// The reinterpret_cast method would produce undefined behavior -// according to ISO C++ specification section 3.10 -15 -. -// bit_cast<> calls memcpy() which is blessed by the standard, -// especially by the example in section 3.9. -// -// Fortunately memcpy() is very fast. In optimized mode, with a -// constant size, gcc 2.95.3, gcc 4.0.1, and msvc 7.1 produce inline -// code with the minimal amount of data movement. On a 32-bit system, -// memcpy(d,s,4) compiles to one load and one store, and memcpy(d,s,8) -// compiles to two loads and two stores. - -template -inline Dest bit_cast(const Source& source) { - COMPILE_ASSERT(sizeof(Dest) == sizeof(Source), bitcasting_unequal_sizes); - Dest dest; - memcpy(&dest, &source, sizeof(dest)); - return dest; -} - -// bit_store implements the equivalent of -// "dest = *reinterpret_cast(&source)". -// -// This prevents undefined behavior when the dest pointer is unaligned. -template -inline void bit_store(Dest *dest, const Source *source) { - COMPILE_ASSERT(sizeof(Dest) == sizeof(Source), bitcasting_unequal_sizes); - memcpy(dest, source, sizeof(Dest)); -} - -#ifdef HAVE___ATTRIBUTE__ -# define ATTRIBUTE_WEAK __attribute__((weak)) -# define ATTRIBUTE_NOINLINE __attribute__((noinline)) -#else -# define ATTRIBUTE_WEAK -# define ATTRIBUTE_NOINLINE -#endif - -#if defined(HAVE___ATTRIBUTE__) && defined(__ELF__) -# define ATTRIBUTE_VISIBILITY_HIDDEN __attribute__((visibility("hidden"))) -#else -# define ATTRIBUTE_VISIBILITY_HIDDEN -#endif - -// Section attributes are supported for both ELF and Mach-O, but in -// very different ways. Here's the API we provide: -// 1) ATTRIBUTE_SECTION: put this with the declaration of all functions -// you want to be in the same linker section -// 2) DEFINE_ATTRIBUTE_SECTION_VARS: must be called once per unique -// name. You want to make sure this is executed before any -// DECLARE_ATTRIBUTE_SECTION_VARS; the easiest way is to put them -// in the same .cc file. Put this call at the global level. -// 3) INIT_ATTRIBUTE_SECTION_VARS: you can scatter calls to this in -// multiple places to help ensure execution before any -// DECLARE_ATTRIBUTE_SECTION_VARS. You must have at least one -// DEFINE, but you can have many INITs. Put each in its own scope. -// 4) DECLARE_ATTRIBUTE_SECTION_VARS: must be called before using -// ATTRIBUTE_SECTION_START or ATTRIBUTE_SECTION_STOP on a name. -// Put this call at the global level. -// 5) ATTRIBUTE_SECTION_START/ATTRIBUTE_SECTION_STOP: call this to say -// where in memory a given section is. All functions declared with -// ATTRIBUTE_SECTION are guaranteed to be between START and STOP. - -#if defined(HAVE___ATTRIBUTE__) && defined(__ELF__) -# define ATTRIBUTE_SECTION(name) __attribute__ ((section (#name))) - - // Weak section declaration to be used as a global declaration - // for ATTRIBUTE_SECTION_START|STOP(name) to compile and link - // even without functions with ATTRIBUTE_SECTION(name). -# define DECLARE_ATTRIBUTE_SECTION_VARS(name) \ - extern char __start_##name[] ATTRIBUTE_WEAK; \ - extern char __stop_##name[] ATTRIBUTE_WEAK -# define INIT_ATTRIBUTE_SECTION_VARS(name) // no-op for ELF -# define DEFINE_ATTRIBUTE_SECTION_VARS(name) // no-op for ELF - - // Return void* pointers to start/end of a section of code with functions - // having ATTRIBUTE_SECTION(name), or 0 if no such function exists. - // One must DECLARE_ATTRIBUTE_SECTION(name) for this to compile and link. -# define ATTRIBUTE_SECTION_START(name) (reinterpret_cast(__start_##name)) -# define ATTRIBUTE_SECTION_STOP(name) (reinterpret_cast(__stop_##name)) -# define HAVE_ATTRIBUTE_SECTION_START 1 - -#elif defined(HAVE___ATTRIBUTE__) && defined(__MACH__) -# define ATTRIBUTE_SECTION(name) __attribute__ ((section ("__TEXT, " #name))) - -#include -#include -class AssignAttributeStartEnd { - public: - AssignAttributeStartEnd(const char* name, char** pstart, char** pend) { - // Find out what dynamic library name is defined in - if (_dyld_present()) { - for (int i = _dyld_image_count() - 1; i >= 0; --i) { - const mach_header* hdr = _dyld_get_image_header(i); -#ifdef MH_MAGIC_64 - if (hdr->magic == MH_MAGIC_64) { - uint64_t len; - *pstart = getsectdatafromheader_64((mach_header_64*)hdr, - "__TEXT", name, &len); - if (*pstart) { // NULL if not defined in this dynamic library - *pstart += _dyld_get_image_vmaddr_slide(i); // correct for reloc - *pend = *pstart + len; - return; - } - } -#endif - if (hdr->magic == MH_MAGIC) { - uint32_t len; - *pstart = getsectdatafromheader(hdr, "__TEXT", name, &len); - if (*pstart) { // NULL if not defined in this dynamic library - *pstart += _dyld_get_image_vmaddr_slide(i); // correct for reloc - *pend = *pstart + len; - return; - } - } - } - } - // If we get here, not defined in a dll at all. See if defined statically. - unsigned long len; // don't ask me why this type isn't uint32_t too... - *pstart = getsectdata("__TEXT", name, &len); - *pend = *pstart + len; - } -}; - -#define DECLARE_ATTRIBUTE_SECTION_VARS(name) \ - extern char* __start_##name; \ - extern char* __stop_##name - -#define INIT_ATTRIBUTE_SECTION_VARS(name) \ - DECLARE_ATTRIBUTE_SECTION_VARS(name); \ - static const AssignAttributeStartEnd __assign_##name( \ - #name, &__start_##name, &__stop_##name) - -#define DEFINE_ATTRIBUTE_SECTION_VARS(name) \ - char* __start_##name, *__stop_##name; \ - INIT_ATTRIBUTE_SECTION_VARS(name) - -# define ATTRIBUTE_SECTION_START(name) (reinterpret_cast(__start_##name)) -# define ATTRIBUTE_SECTION_STOP(name) (reinterpret_cast(__stop_##name)) -# define HAVE_ATTRIBUTE_SECTION_START 1 - -#else // not HAVE___ATTRIBUTE__ && __ELF__, nor HAVE___ATTRIBUTE__ && __MACH__ -# define ATTRIBUTE_SECTION(name) -# define DECLARE_ATTRIBUTE_SECTION_VARS(name) -# define INIT_ATTRIBUTE_SECTION_VARS(name) -# define DEFINE_ATTRIBUTE_SECTION_VARS(name) -# define ATTRIBUTE_SECTION_START(name) (reinterpret_cast(0)) -# define ATTRIBUTE_SECTION_STOP(name) (reinterpret_cast(0)) - -#endif // HAVE___ATTRIBUTE__ and __ELF__ or __MACH__ - -#if defined(HAVE___ATTRIBUTE__) -# if (defined(__i386__) || defined(__x86_64__)) -# define CACHELINE_ALIGNED __attribute__((aligned(64))) -# elif (defined(__PPC__) || defined(__PPC64__)) -# define CACHELINE_ALIGNED __attribute__((aligned(16))) -# elif (defined(__arm__)) -# define CACHELINE_ALIGNED __attribute__((aligned(64))) - // some ARMs have shorter cache lines (ARM1176JZF-S is 32 bytes for example) but obviously 64-byte aligned implies 32-byte aligned -# elif (defined(__mips__)) -# define CACHELINE_ALIGNED __attribute__((aligned(128))) -# elif (defined(__aarch64__)) -# define CACHELINE_ALIGNED __attribute__((aligned(64))) - // implementation specific, Cortex-A53 and 57 should have 64 bytes -# elif (defined(__s390__)) -# define CACHELINE_ALIGNED __attribute__((aligned(256))) -# else -# error Could not determine cache line length - unknown architecture -# endif -#else -# define CACHELINE_ALIGNED -#endif // defined(HAVE___ATTRIBUTE__) && (__i386__ || __x86_64__) - -// Structure for discovering alignment -union MemoryAligner { - void* p; - double d; - size_t s; -} CACHELINE_ALIGNED; - -// The following enum should be used only as a constructor argument to indicate -// that the variable has static storage class, and that the constructor should -// do nothing to its state. It indicates to the reader that it is legal to -// declare a static nistance of the class, provided the constructor is given -// the base::LINKER_INITIALIZED argument. Normally, it is unsafe to declare a -// static variable that has a constructor or a destructor because invocation -// order is undefined. However, IF the type can be initialized by filling with -// zeroes (which the loader does for static variables), AND the destructor also -// does nothing to the storage, then a constructor declared as -// explicit MyClass(base::LinkerInitialized x) {} -// and invoked as -// static MyClass my_variable_name(base::LINKER_INITIALIZED); -namespace base { -enum LinkerInitialized { LINKER_INITIALIZED }; -} - -#endif // _BASICTYPES_H_ diff --git a/contrib/libtcmalloc/src/base/commandlineflags.h b/contrib/libtcmalloc/src/base/commandlineflags.h deleted file mode 100644 index e940edd3791..00000000000 --- a/contrib/libtcmalloc/src/base/commandlineflags.h +++ /dev/null @@ -1,166 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// This file is a compatibility layer that defines Google's version of -// command line flags that are used for configuration. -// -// We put flags into their own namespace. It is purposefully -// named in an opaque way that people should have trouble typing -// directly. The idea is that DEFINE puts the flag in the weird -// namespace, and DECLARE imports the flag from there into the -// current namespace. The net result is to force people to use -// DECLARE to get access to a flag, rather than saying -// extern bool FLAGS_logtostderr; -// or some such instead. We want this so we can put extra -// functionality (like sanity-checking) in DECLARE if we want, -// and make sure it is picked up everywhere. -// -// We also put the type of the variable in the namespace, so that -// people can't DECLARE_int32 something that they DEFINE_bool'd -// elsewhere. -#ifndef BASE_COMMANDLINEFLAGS_H_ -#define BASE_COMMANDLINEFLAGS_H_ - -#include "../config.h" -#include -#include // for memchr -#include // for getenv -#include "base/basictypes.h" - -#define DECLARE_VARIABLE(type, name) \ - namespace FLAG__namespace_do_not_use_directly_use_DECLARE_##type##_instead { \ - extern PERFTOOLS_DLL_DECL type FLAGS_##name; \ - } \ - using FLAG__namespace_do_not_use_directly_use_DECLARE_##type##_instead::FLAGS_##name - -#define DEFINE_VARIABLE(type, name, value, meaning) \ - namespace FLAG__namespace_do_not_use_directly_use_DECLARE_##type##_instead { \ - PERFTOOLS_DLL_DECL type FLAGS_##name(value); \ - char FLAGS_no##name; \ - } \ - using FLAG__namespace_do_not_use_directly_use_DECLARE_##type##_instead::FLAGS_##name - -// bool specialization -#define DECLARE_bool(name) \ - DECLARE_VARIABLE(bool, name) -#define DEFINE_bool(name, value, meaning) \ - DEFINE_VARIABLE(bool, name, value, meaning) - -// int32 specialization -#define DECLARE_int32(name) \ - DECLARE_VARIABLE(int32, name) -#define DEFINE_int32(name, value, meaning) \ - DEFINE_VARIABLE(int32, name, value, meaning) - -// int64 specialization -#define DECLARE_int64(name) \ - DECLARE_VARIABLE(int64, name) -#define DEFINE_int64(name, value, meaning) \ - DEFINE_VARIABLE(int64, name, value, meaning) - -#define DECLARE_uint64(name) \ - DECLARE_VARIABLE(uint64, name) -#define DEFINE_uint64(name, value, meaning) \ - DEFINE_VARIABLE(uint64, name, value, meaning) - -// double specialization -#define DECLARE_double(name) \ - DECLARE_VARIABLE(double, name) -#define DEFINE_double(name, value, meaning) \ - DEFINE_VARIABLE(double, name, value, meaning) - -// Special case for string, because we have to specify the namespace -// std::string, which doesn't play nicely with our FLAG__namespace hackery. -#define DECLARE_string(name) \ - namespace FLAG__namespace_do_not_use_directly_use_DECLARE_string_instead { \ - extern std::string FLAGS_##name; \ - } \ - using FLAG__namespace_do_not_use_directly_use_DECLARE_string_instead::FLAGS_##name -#define DEFINE_string(name, value, meaning) \ - namespace FLAG__namespace_do_not_use_directly_use_DECLARE_string_instead { \ - std::string FLAGS_##name(value); \ - char FLAGS_no##name; \ - } \ - using FLAG__namespace_do_not_use_directly_use_DECLARE_string_instead::FLAGS_##name - -// implemented in sysinfo.cc -namespace tcmalloc { - namespace commandlineflags { - - inline bool StringToBool(const char *value, bool def) { - if (!value) { - return def; - } - return memchr("tTyY1\0", value[0], 6) != NULL; - } - - inline int StringToInt(const char *value, int def) { - if (!value) { - return def; - } - return strtol(value, NULL, 10); - } - - inline long long StringToLongLong(const char *value, long long def) { - if (!value) { - return def; - } - return strtoll(value, NULL, 10); - } - - inline double StringToDouble(const char *value, double def) { - if (!value) { - return def; - } - return strtod(value, NULL); - } - } -} - -// These macros (could be functions, but I don't want to bother with a .cc -// file), make it easier to initialize flags from the environment. - -#define EnvToString(envname, dflt) \ - (!getenv(envname) ? (dflt) : getenv(envname)) - -#define EnvToBool(envname, dflt) \ - tcmalloc::commandlineflags::StringToBool(getenv(envname), dflt) - -#define EnvToInt(envname, dflt) \ - tcmalloc::commandlineflags::StringToInt(getenv(envname), dflt) - -#define EnvToInt64(envname, dflt) \ - tcmalloc::commandlineflags::StringToLongLong(getenv(envname), dflt) - -#define EnvToDouble(envname, dflt) \ - tcmalloc::commandlineflags::StringToDouble(getenv(envname), dflt) - -#endif // BASE_COMMANDLINEFLAGS_H_ diff --git a/contrib/libtcmalloc/src/base/dynamic_annotations.c b/contrib/libtcmalloc/src/base/dynamic_annotations.c deleted file mode 100644 index 87bd2ecde97..00000000000 --- a/contrib/libtcmalloc/src/base/dynamic_annotations.c +++ /dev/null @@ -1,179 +0,0 @@ -/* Copyright (c) 2008-2009, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Kostya Serebryany - */ - -#ifdef __cplusplus -# error "This file should be built as pure C to avoid name mangling" -#endif - -#include "config.h" -#include -#include - -#include "base/dynamic_annotations.h" -#include "getenv_safe.h" // for TCMallocGetenvSafe - -#ifdef __GNUC__ -/* valgrind.h uses gcc extensions so it won't build with other compilers */ -# ifdef HAVE_VALGRIND_H /* prefer the user's copy if they have it */ -# include -# else /* otherwise just use the copy that we have */ -# include "third_party/valgrind.h" -# endif -#endif - -/* Compiler-based ThreadSanitizer defines - DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL = 1 - and provides its own definitions of the functions. */ - -#ifndef DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL -# define DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL 0 -#endif - -/* Each function is empty and called (via a macro) only in debug mode. - The arguments are captured by dynamic tools at runtime. */ - -#if DYNAMIC_ANNOTATIONS_ENABLED == 1 \ - && DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL == 0 - -void AnnotateRWLockCreate(const char *file, int line, - const volatile void *lock){} -void AnnotateRWLockDestroy(const char *file, int line, - const volatile void *lock){} -void AnnotateRWLockAcquired(const char *file, int line, - const volatile void *lock, long is_w){} -void AnnotateRWLockReleased(const char *file, int line, - const volatile void *lock, long is_w){} -void AnnotateBarrierInit(const char *file, int line, - const volatile void *barrier, long count, - long reinitialization_allowed) {} -void AnnotateBarrierWaitBefore(const char *file, int line, - const volatile void *barrier) {} -void AnnotateBarrierWaitAfter(const char *file, int line, - const volatile void *barrier) {} -void AnnotateBarrierDestroy(const char *file, int line, - const volatile void *barrier) {} - -void AnnotateCondVarWait(const char *file, int line, - const volatile void *cv, - const volatile void *lock){} -void AnnotateCondVarSignal(const char *file, int line, - const volatile void *cv){} -void AnnotateCondVarSignalAll(const char *file, int line, - const volatile void *cv){} -void AnnotatePublishMemoryRange(const char *file, int line, - const volatile void *address, - long size){} -void AnnotateUnpublishMemoryRange(const char *file, int line, - const volatile void *address, - long size){} -void AnnotatePCQCreate(const char *file, int line, - const volatile void *pcq){} -void AnnotatePCQDestroy(const char *file, int line, - const volatile void *pcq){} -void AnnotatePCQPut(const char *file, int line, - const volatile void *pcq){} -void AnnotatePCQGet(const char *file, int line, - const volatile void *pcq){} -void AnnotateNewMemory(const char *file, int line, - const volatile void *mem, - long size){} -void AnnotateExpectRace(const char *file, int line, - const volatile void *mem, - const char *description){} -void AnnotateBenignRace(const char *file, int line, - const volatile void *mem, - const char *description){} -void AnnotateBenignRaceSized(const char *file, int line, - const volatile void *mem, - long size, - const char *description) {} -void AnnotateMutexIsUsedAsCondVar(const char *file, int line, - const volatile void *mu){} -void AnnotateTraceMemory(const char *file, int line, - const volatile void *arg){} -void AnnotateThreadName(const char *file, int line, - const char *name){} -void AnnotateIgnoreReadsBegin(const char *file, int line){} -void AnnotateIgnoreReadsEnd(const char *file, int line){} -void AnnotateIgnoreWritesBegin(const char *file, int line){} -void AnnotateIgnoreWritesEnd(const char *file, int line){} -void AnnotateEnableRaceDetection(const char *file, int line, int enable){} -void AnnotateNoOp(const char *file, int line, - const volatile void *arg){} -void AnnotateFlushState(const char *file, int line){} - -#endif /* DYNAMIC_ANNOTATIONS_ENABLED == 1 - && DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL == 0 */ - -#if DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL == 0 - -static int GetRunningOnValgrind(void) { -#ifdef RUNNING_ON_VALGRIND - if (RUNNING_ON_VALGRIND) return 1; -#endif - const char *running_on_valgrind_str = TCMallocGetenvSafe("RUNNING_ON_VALGRIND"); - if (running_on_valgrind_str) { - return strcmp(running_on_valgrind_str, "0") != 0; - } - return 0; -} - -/* See the comments in dynamic_annotations.h */ -int RunningOnValgrind(void) { - static volatile int running_on_valgrind = -1; - int local_running_on_valgrind = running_on_valgrind; - /* C doesn't have thread-safe initialization of statics, and we - don't want to depend on pthread_once here, so hack it. */ - ANNOTATE_BENIGN_RACE(&running_on_valgrind, "safe hack"); - if (local_running_on_valgrind == -1) - running_on_valgrind = local_running_on_valgrind = GetRunningOnValgrind(); - return local_running_on_valgrind; -} - -#endif /* DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL == 0 */ - -/* See the comments in dynamic_annotations.h */ -double ValgrindSlowdown(void) { - /* Same initialization hack as in RunningOnValgrind(). */ - static volatile double slowdown = 0.0; - double local_slowdown = slowdown; - ANNOTATE_BENIGN_RACE(&slowdown, "safe hack"); - if (RunningOnValgrind() == 0) { - return 1.0; - } - if (local_slowdown == 0.0) { - char *env = getenv("VALGRIND_SLOWDOWN"); - slowdown = local_slowdown = env ? atof(env) : 50.0; - } - return local_slowdown; -} diff --git a/contrib/libtcmalloc/src/base/dynamic_annotations.h b/contrib/libtcmalloc/src/base/dynamic_annotations.h deleted file mode 100644 index 4669315ced3..00000000000 --- a/contrib/libtcmalloc/src/base/dynamic_annotations.h +++ /dev/null @@ -1,627 +0,0 @@ -/* Copyright (c) 2008, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Kostya Serebryany - */ - -/* This file defines dynamic annotations for use with dynamic analysis - tool such as valgrind, PIN, etc. - - Dynamic annotation is a source code annotation that affects - the generated code (that is, the annotation is not a comment). - Each such annotation is attached to a particular - instruction and/or to a particular object (address) in the program. - - The annotations that should be used by users are macros in all upper-case - (e.g., ANNOTATE_NEW_MEMORY). - - Actual implementation of these macros may differ depending on the - dynamic analysis tool being used. - - See http://code.google.com/p/data-race-test/ for more information. - - This file supports the following dynamic analysis tools: - - None (DYNAMIC_ANNOTATIONS_ENABLED is not defined or zero). - Macros are defined empty. - - ThreadSanitizer, Helgrind, DRD (DYNAMIC_ANNOTATIONS_ENABLED is 1). - Macros are defined as calls to non-inlinable empty functions - that are intercepted by Valgrind. */ - -#ifndef BASE_DYNAMIC_ANNOTATIONS_H_ -#define BASE_DYNAMIC_ANNOTATIONS_H_ - -#ifndef DYNAMIC_ANNOTATIONS_ENABLED -# define DYNAMIC_ANNOTATIONS_ENABLED 0 -#endif - -#if DYNAMIC_ANNOTATIONS_ENABLED != 0 - - /* ------------------------------------------------------------- - Annotations useful when implementing condition variables such as CondVar, - using conditional critical sections (Await/LockWhen) and when constructing - user-defined synchronization mechanisms. - - The annotations ANNOTATE_HAPPENS_BEFORE() and ANNOTATE_HAPPENS_AFTER() can - be used to define happens-before arcs in user-defined synchronization - mechanisms: the race detector will infer an arc from the former to the - latter when they share the same argument pointer. - - Example 1 (reference counting): - - void Unref() { - ANNOTATE_HAPPENS_BEFORE(&refcount_); - if (AtomicDecrementByOne(&refcount_) == 0) { - ANNOTATE_HAPPENS_AFTER(&refcount_); - delete this; - } - } - - Example 2 (message queue): - - void MyQueue::Put(Type *e) { - MutexLock lock(&mu_); - ANNOTATE_HAPPENS_BEFORE(e); - PutElementIntoMyQueue(e); - } - - Type *MyQueue::Get() { - MutexLock lock(&mu_); - Type *e = GetElementFromMyQueue(); - ANNOTATE_HAPPENS_AFTER(e); - return e; - } - - Note: when possible, please use the existing reference counting and message - queue implementations instead of inventing new ones. */ - - /* Report that wait on the condition variable at address "cv" has succeeded - and the lock at address "lock" is held. */ - #define ANNOTATE_CONDVAR_LOCK_WAIT(cv, lock) \ - AnnotateCondVarWait(__FILE__, __LINE__, cv, lock) - - /* Report that wait on the condition variable at "cv" has succeeded. Variant - w/o lock. */ - #define ANNOTATE_CONDVAR_WAIT(cv) \ - AnnotateCondVarWait(__FILE__, __LINE__, cv, NULL) - - /* Report that we are about to signal on the condition variable at address - "cv". */ - #define ANNOTATE_CONDVAR_SIGNAL(cv) \ - AnnotateCondVarSignal(__FILE__, __LINE__, cv) - - /* Report that we are about to signal_all on the condition variable at "cv". */ - #define ANNOTATE_CONDVAR_SIGNAL_ALL(cv) \ - AnnotateCondVarSignalAll(__FILE__, __LINE__, cv) - - /* Annotations for user-defined synchronization mechanisms. */ - #define ANNOTATE_HAPPENS_BEFORE(obj) ANNOTATE_CONDVAR_SIGNAL(obj) - #define ANNOTATE_HAPPENS_AFTER(obj) ANNOTATE_CONDVAR_WAIT(obj) - - /* Report that the bytes in the range [pointer, pointer+size) are about - to be published safely. The race checker will create a happens-before - arc from the call ANNOTATE_PUBLISH_MEMORY_RANGE(pointer, size) to - subsequent accesses to this memory. - Note: this annotation may not work properly if the race detector uses - sampling, i.e. does not observe all memory accesses. - */ - #define ANNOTATE_PUBLISH_MEMORY_RANGE(pointer, size) \ - AnnotatePublishMemoryRange(__FILE__, __LINE__, pointer, size) - - /* DEPRECATED. Don't use it. */ - #define ANNOTATE_UNPUBLISH_MEMORY_RANGE(pointer, size) \ - AnnotateUnpublishMemoryRange(__FILE__, __LINE__, pointer, size) - - /* DEPRECATED. Don't use it. */ - #define ANNOTATE_SWAP_MEMORY_RANGE(pointer, size) \ - do { \ - ANNOTATE_UNPUBLISH_MEMORY_RANGE(pointer, size); \ - ANNOTATE_PUBLISH_MEMORY_RANGE(pointer, size); \ - } while (0) - - /* Instruct the tool to create a happens-before arc between mu->Unlock() and - mu->Lock(). This annotation may slow down the race detector and hide real - races. Normally it is used only when it would be difficult to annotate each - of the mutex's critical sections individually using the annotations above. - This annotation makes sense only for hybrid race detectors. For pure - happens-before detectors this is a no-op. For more details see - http://code.google.com/p/data-race-test/wiki/PureHappensBeforeVsHybrid . */ - #define ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX(mu) \ - AnnotateMutexIsUsedAsCondVar(__FILE__, __LINE__, mu) - - /* Deprecated. Use ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX. */ - #define ANNOTATE_MUTEX_IS_USED_AS_CONDVAR(mu) \ - AnnotateMutexIsUsedAsCondVar(__FILE__, __LINE__, mu) - - /* ------------------------------------------------------------- - Annotations useful when defining memory allocators, or when memory that - was protected in one way starts to be protected in another. */ - - /* Report that a new memory at "address" of size "size" has been allocated. - This might be used when the memory has been retrieved from a free list and - is about to be reused, or when a the locking discipline for a variable - changes. */ - #define ANNOTATE_NEW_MEMORY(address, size) \ - AnnotateNewMemory(__FILE__, __LINE__, address, size) - - /* ------------------------------------------------------------- - Annotations useful when defining FIFO queues that transfer data between - threads. */ - - /* Report that the producer-consumer queue (such as ProducerConsumerQueue) at - address "pcq" has been created. The ANNOTATE_PCQ_* annotations - should be used only for FIFO queues. For non-FIFO queues use - ANNOTATE_HAPPENS_BEFORE (for put) and ANNOTATE_HAPPENS_AFTER (for get). */ - #define ANNOTATE_PCQ_CREATE(pcq) \ - AnnotatePCQCreate(__FILE__, __LINE__, pcq) - - /* Report that the queue at address "pcq" is about to be destroyed. */ - #define ANNOTATE_PCQ_DESTROY(pcq) \ - AnnotatePCQDestroy(__FILE__, __LINE__, pcq) - - /* Report that we are about to put an element into a FIFO queue at address - "pcq". */ - #define ANNOTATE_PCQ_PUT(pcq) \ - AnnotatePCQPut(__FILE__, __LINE__, pcq) - - /* Report that we've just got an element from a FIFO queue at address "pcq". */ - #define ANNOTATE_PCQ_GET(pcq) \ - AnnotatePCQGet(__FILE__, __LINE__, pcq) - - /* ------------------------------------------------------------- - Annotations that suppress errors. It is usually better to express the - program's synchronization using the other annotations, but these can - be used when all else fails. */ - - /* Report that we may have a benign race at "pointer", with size - "sizeof(*(pointer))". "pointer" must be a non-void* pointer. Insert at the - point where "pointer" has been allocated, preferably close to the point - where the race happens. See also ANNOTATE_BENIGN_RACE_STATIC. */ - #define ANNOTATE_BENIGN_RACE(pointer, description) \ - AnnotateBenignRaceSized(__FILE__, __LINE__, pointer, \ - sizeof(*(pointer)), description) - - /* Same as ANNOTATE_BENIGN_RACE(address, description), but applies to - the memory range [address, address+size). */ - #define ANNOTATE_BENIGN_RACE_SIZED(address, size, description) \ - AnnotateBenignRaceSized(__FILE__, __LINE__, address, size, description) - - /* Request the analysis tool to ignore all reads in the current thread - until ANNOTATE_IGNORE_READS_END is called. - Useful to ignore intentional racey reads, while still checking - other reads and all writes. - See also ANNOTATE_UNPROTECTED_READ. */ - #define ANNOTATE_IGNORE_READS_BEGIN() \ - AnnotateIgnoreReadsBegin(__FILE__, __LINE__) - - /* Stop ignoring reads. */ - #define ANNOTATE_IGNORE_READS_END() \ - AnnotateIgnoreReadsEnd(__FILE__, __LINE__) - - /* Similar to ANNOTATE_IGNORE_READS_BEGIN, but ignore writes. */ - #define ANNOTATE_IGNORE_WRITES_BEGIN() \ - AnnotateIgnoreWritesBegin(__FILE__, __LINE__) - - /* Stop ignoring writes. */ - #define ANNOTATE_IGNORE_WRITES_END() \ - AnnotateIgnoreWritesEnd(__FILE__, __LINE__) - - /* Start ignoring all memory accesses (reads and writes). */ - #define ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN() \ - do {\ - ANNOTATE_IGNORE_READS_BEGIN();\ - ANNOTATE_IGNORE_WRITES_BEGIN();\ - }while(0)\ - - /* Stop ignoring all memory accesses. */ - #define ANNOTATE_IGNORE_READS_AND_WRITES_END() \ - do {\ - ANNOTATE_IGNORE_WRITES_END();\ - ANNOTATE_IGNORE_READS_END();\ - }while(0)\ - - /* Enable (enable!=0) or disable (enable==0) race detection for all threads. - This annotation could be useful if you want to skip expensive race analysis - during some period of program execution, e.g. during initialization. */ - #define ANNOTATE_ENABLE_RACE_DETECTION(enable) \ - AnnotateEnableRaceDetection(__FILE__, __LINE__, enable) - - /* ------------------------------------------------------------- - Annotations useful for debugging. */ - - /* Request to trace every access to "address". */ - #define ANNOTATE_TRACE_MEMORY(address) \ - AnnotateTraceMemory(__FILE__, __LINE__, address) - - /* Report the current thread name to a race detector. */ - #define ANNOTATE_THREAD_NAME(name) \ - AnnotateThreadName(__FILE__, __LINE__, name) - - /* ------------------------------------------------------------- - Annotations useful when implementing locks. They are not - normally needed by modules that merely use locks. - The "lock" argument is a pointer to the lock object. */ - - /* Report that a lock has been created at address "lock". */ - #define ANNOTATE_RWLOCK_CREATE(lock) \ - AnnotateRWLockCreate(__FILE__, __LINE__, lock) - - /* Report that the lock at address "lock" is about to be destroyed. */ - #define ANNOTATE_RWLOCK_DESTROY(lock) \ - AnnotateRWLockDestroy(__FILE__, __LINE__, lock) - - /* Report that the lock at address "lock" has been acquired. - is_w=1 for writer lock, is_w=0 for reader lock. */ - #define ANNOTATE_RWLOCK_ACQUIRED(lock, is_w) \ - AnnotateRWLockAcquired(__FILE__, __LINE__, lock, is_w) - - /* Report that the lock at address "lock" is about to be released. */ - #define ANNOTATE_RWLOCK_RELEASED(lock, is_w) \ - AnnotateRWLockReleased(__FILE__, __LINE__, lock, is_w) - - /* ------------------------------------------------------------- - Annotations useful when implementing barriers. They are not - normally needed by modules that merely use barriers. - The "barrier" argument is a pointer to the barrier object. */ - - /* Report that the "barrier" has been initialized with initial "count". - If 'reinitialization_allowed' is true, initialization is allowed to happen - multiple times w/o calling barrier_destroy() */ - #define ANNOTATE_BARRIER_INIT(barrier, count, reinitialization_allowed) \ - AnnotateBarrierInit(__FILE__, __LINE__, barrier, count, \ - reinitialization_allowed) - - /* Report that we are about to enter barrier_wait("barrier"). */ - #define ANNOTATE_BARRIER_WAIT_BEFORE(barrier) \ - AnnotateBarrierWaitBefore(__FILE__, __LINE__, barrier) - - /* Report that we just exited barrier_wait("barrier"). */ - #define ANNOTATE_BARRIER_WAIT_AFTER(barrier) \ - AnnotateBarrierWaitAfter(__FILE__, __LINE__, barrier) - - /* Report that the "barrier" has been destroyed. */ - #define ANNOTATE_BARRIER_DESTROY(barrier) \ - AnnotateBarrierDestroy(__FILE__, __LINE__, barrier) - - /* ------------------------------------------------------------- - Annotations useful for testing race detectors. */ - - /* Report that we expect a race on the variable at "address". - Use only in unit tests for a race detector. */ - #define ANNOTATE_EXPECT_RACE(address, description) \ - AnnotateExpectRace(__FILE__, __LINE__, address, description) - - /* A no-op. Insert where you like to test the interceptors. */ - #define ANNOTATE_NO_OP(arg) \ - AnnotateNoOp(__FILE__, __LINE__, arg) - - /* Force the race detector to flush its state. The actual effect depends on - * the implementation of the detector. */ - #define ANNOTATE_FLUSH_STATE() \ - AnnotateFlushState(__FILE__, __LINE__) - - -#else /* DYNAMIC_ANNOTATIONS_ENABLED == 0 */ - - #define ANNOTATE_RWLOCK_CREATE(lock) /* empty */ - #define ANNOTATE_RWLOCK_DESTROY(lock) /* empty */ - #define ANNOTATE_RWLOCK_ACQUIRED(lock, is_w) /* empty */ - #define ANNOTATE_RWLOCK_RELEASED(lock, is_w) /* empty */ - #define ANNOTATE_BARRIER_INIT(barrier, count, reinitialization_allowed) /* */ - #define ANNOTATE_BARRIER_WAIT_BEFORE(barrier) /* empty */ - #define ANNOTATE_BARRIER_WAIT_AFTER(barrier) /* empty */ - #define ANNOTATE_BARRIER_DESTROY(barrier) /* empty */ - #define ANNOTATE_CONDVAR_LOCK_WAIT(cv, lock) /* empty */ - #define ANNOTATE_CONDVAR_WAIT(cv) /* empty */ - #define ANNOTATE_CONDVAR_SIGNAL(cv) /* empty */ - #define ANNOTATE_CONDVAR_SIGNAL_ALL(cv) /* empty */ - #define ANNOTATE_HAPPENS_BEFORE(obj) /* empty */ - #define ANNOTATE_HAPPENS_AFTER(obj) /* empty */ - #define ANNOTATE_PUBLISH_MEMORY_RANGE(address, size) /* empty */ - #define ANNOTATE_UNPUBLISH_MEMORY_RANGE(address, size) /* empty */ - #define ANNOTATE_SWAP_MEMORY_RANGE(address, size) /* empty */ - #define ANNOTATE_PCQ_CREATE(pcq) /* empty */ - #define ANNOTATE_PCQ_DESTROY(pcq) /* empty */ - #define ANNOTATE_PCQ_PUT(pcq) /* empty */ - #define ANNOTATE_PCQ_GET(pcq) /* empty */ - #define ANNOTATE_NEW_MEMORY(address, size) /* empty */ - #define ANNOTATE_EXPECT_RACE(address, description) /* empty */ - #define ANNOTATE_BENIGN_RACE(address, description) /* empty */ - #define ANNOTATE_BENIGN_RACE_SIZED(address, size, description) /* empty */ - #define ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX(mu) /* empty */ - #define ANNOTATE_MUTEX_IS_USED_AS_CONDVAR(mu) /* empty */ - #define ANNOTATE_TRACE_MEMORY(arg) /* empty */ - #define ANNOTATE_THREAD_NAME(name) /* empty */ - #define ANNOTATE_IGNORE_READS_BEGIN() /* empty */ - #define ANNOTATE_IGNORE_READS_END() /* empty */ - #define ANNOTATE_IGNORE_WRITES_BEGIN() /* empty */ - #define ANNOTATE_IGNORE_WRITES_END() /* empty */ - #define ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN() /* empty */ - #define ANNOTATE_IGNORE_READS_AND_WRITES_END() /* empty */ - #define ANNOTATE_ENABLE_RACE_DETECTION(enable) /* empty */ - #define ANNOTATE_NO_OP(arg) /* empty */ - #define ANNOTATE_FLUSH_STATE() /* empty */ - -#endif /* DYNAMIC_ANNOTATIONS_ENABLED */ - -/* Macro definitions for GCC attributes that allow static thread safety - analysis to recognize and use some of the dynamic annotations as - escape hatches. - TODO(lcwu): remove the check for __SUPPORT_DYN_ANNOTATION__ once the - default crosstool/GCC supports these GCC attributes. */ - -#define ANNOTALYSIS_STATIC_INLINE -#define ANNOTALYSIS_SEMICOLON_OR_EMPTY_BODY ; -#define ANNOTALYSIS_IGNORE_READS_BEGIN -#define ANNOTALYSIS_IGNORE_READS_END -#define ANNOTALYSIS_IGNORE_WRITES_BEGIN -#define ANNOTALYSIS_IGNORE_WRITES_END -#define ANNOTALYSIS_UNPROTECTED_READ - -#if defined(__GNUC__) && (!defined(SWIG)) && (!defined(__clang__)) && \ - defined(__SUPPORT_TS_ANNOTATION__) && defined(__SUPPORT_DYN_ANNOTATION__) - -#if DYNAMIC_ANNOTATIONS_ENABLED == 0 -#define ANNOTALYSIS_ONLY 1 -#undef ANNOTALYSIS_STATIC_INLINE -#define ANNOTALYSIS_STATIC_INLINE static inline -#undef ANNOTALYSIS_SEMICOLON_OR_EMPTY_BODY -#define ANNOTALYSIS_SEMICOLON_OR_EMPTY_BODY { (void)file; (void)line; } -#endif - -/* Only emit attributes when annotalysis is enabled. */ -#if defined(__SUPPORT_TS_ANNOTATION__) && defined(__SUPPORT_DYN_ANNOTATION__) -#undef ANNOTALYSIS_IGNORE_READS_BEGIN -#define ANNOTALYSIS_IGNORE_READS_BEGIN __attribute__ ((ignore_reads_begin)) -#undef ANNOTALYSIS_IGNORE_READS_END -#define ANNOTALYSIS_IGNORE_READS_END __attribute__ ((ignore_reads_end)) -#undef ANNOTALYSIS_IGNORE_WRITES_BEGIN -#define ANNOTALYSIS_IGNORE_WRITES_BEGIN __attribute__ ((ignore_writes_begin)) -#undef ANNOTALYSIS_IGNORE_WRITES_END -#define ANNOTALYSIS_IGNORE_WRITES_END __attribute__ ((ignore_writes_end)) -#undef ANNOTALYSIS_UNPROTECTED_READ -#define ANNOTALYSIS_UNPROTECTED_READ __attribute__ ((unprotected_read)) -#endif - -#endif // defined(__GNUC__) && (!defined(SWIG)) && (!defined(__clang__)) - -/* Use the macros above rather than using these functions directly. */ -#ifdef __cplusplus -extern "C" { -#endif -void AnnotateRWLockCreate(const char *file, int line, - const volatile void *lock); -void AnnotateRWLockDestroy(const char *file, int line, - const volatile void *lock); -void AnnotateRWLockAcquired(const char *file, int line, - const volatile void *lock, long is_w); -void AnnotateRWLockReleased(const char *file, int line, - const volatile void *lock, long is_w); -void AnnotateBarrierInit(const char *file, int line, - const volatile void *barrier, long count, - long reinitialization_allowed); -void AnnotateBarrierWaitBefore(const char *file, int line, - const volatile void *barrier); -void AnnotateBarrierWaitAfter(const char *file, int line, - const volatile void *barrier); -void AnnotateBarrierDestroy(const char *file, int line, - const volatile void *barrier); -void AnnotateCondVarWait(const char *file, int line, - const volatile void *cv, - const volatile void *lock); -void AnnotateCondVarSignal(const char *file, int line, - const volatile void *cv); -void AnnotateCondVarSignalAll(const char *file, int line, - const volatile void *cv); -void AnnotatePublishMemoryRange(const char *file, int line, - const volatile void *address, - long size); -void AnnotateUnpublishMemoryRange(const char *file, int line, - const volatile void *address, - long size); -void AnnotatePCQCreate(const char *file, int line, - const volatile void *pcq); -void AnnotatePCQDestroy(const char *file, int line, - const volatile void *pcq); -void AnnotatePCQPut(const char *file, int line, - const volatile void *pcq); -void AnnotatePCQGet(const char *file, int line, - const volatile void *pcq); -void AnnotateNewMemory(const char *file, int line, - const volatile void *address, - long size); -void AnnotateExpectRace(const char *file, int line, - const volatile void *address, - const char *description); -void AnnotateBenignRace(const char *file, int line, - const volatile void *address, - const char *description); -void AnnotateBenignRaceSized(const char *file, int line, - const volatile void *address, - long size, - const char *description); -void AnnotateMutexIsUsedAsCondVar(const char *file, int line, - const volatile void *mu); -void AnnotateTraceMemory(const char *file, int line, - const volatile void *arg); -void AnnotateThreadName(const char *file, int line, - const char *name); -ANNOTALYSIS_STATIC_INLINE -void AnnotateIgnoreReadsBegin(const char *file, int line) - ANNOTALYSIS_IGNORE_READS_BEGIN ANNOTALYSIS_SEMICOLON_OR_EMPTY_BODY -ANNOTALYSIS_STATIC_INLINE -void AnnotateIgnoreReadsEnd(const char *file, int line) - ANNOTALYSIS_IGNORE_READS_END ANNOTALYSIS_SEMICOLON_OR_EMPTY_BODY -ANNOTALYSIS_STATIC_INLINE -void AnnotateIgnoreWritesBegin(const char *file, int line) - ANNOTALYSIS_IGNORE_WRITES_BEGIN ANNOTALYSIS_SEMICOLON_OR_EMPTY_BODY -ANNOTALYSIS_STATIC_INLINE -void AnnotateIgnoreWritesEnd(const char *file, int line) - ANNOTALYSIS_IGNORE_WRITES_END ANNOTALYSIS_SEMICOLON_OR_EMPTY_BODY -void AnnotateEnableRaceDetection(const char *file, int line, int enable); -void AnnotateNoOp(const char *file, int line, - const volatile void *arg); -void AnnotateFlushState(const char *file, int line); - -/* Return non-zero value if running under valgrind. - - If "valgrind.h" is included into dynamic_annotations.c, - the regular valgrind mechanism will be used. - See http://valgrind.org/docs/manual/manual-core-adv.html about - RUNNING_ON_VALGRIND and other valgrind "client requests". - The file "valgrind.h" may be obtained by doing - svn co svn://svn.valgrind.org/valgrind/trunk/include - - If for some reason you can't use "valgrind.h" or want to fake valgrind, - there are two ways to make this function return non-zero: - - Use environment variable: export RUNNING_ON_VALGRIND=1 - - Make your tool intercept the function RunningOnValgrind() and - change its return value. - */ -int RunningOnValgrind(void); - -/* ValgrindSlowdown returns: - * 1.0, if (RunningOnValgrind() == 0) - * 50.0, if (RunningOnValgrind() != 0 && getenv("VALGRIND_SLOWDOWN") == NULL) - * atof(getenv("VALGRIND_SLOWDOWN")) otherwise - This function can be used to scale timeout values: - EXAMPLE: - for (;;) { - DoExpensiveBackgroundTask(); - SleepForSeconds(5 * ValgrindSlowdown()); - } - */ -double ValgrindSlowdown(void); - -#ifdef __cplusplus -} -#endif - -#if DYNAMIC_ANNOTATIONS_ENABLED != 0 && defined(__cplusplus) - - /* ANNOTATE_UNPROTECTED_READ is the preferred way to annotate racey reads. - - Instead of doing - ANNOTATE_IGNORE_READS_BEGIN(); - ... = x; - ANNOTATE_IGNORE_READS_END(); - one can use - ... = ANNOTATE_UNPROTECTED_READ(x); */ - template - inline T ANNOTATE_UNPROTECTED_READ(const volatile T &x) - ANNOTALYSIS_UNPROTECTED_READ { - ANNOTATE_IGNORE_READS_BEGIN(); - T res = x; - ANNOTATE_IGNORE_READS_END(); - return res; - } - /* Apply ANNOTATE_BENIGN_RACE_SIZED to a static variable. */ - #define ANNOTATE_BENIGN_RACE_STATIC(static_var, description) \ - namespace { \ - class static_var ## _annotator { \ - public: \ - static_var ## _annotator() { \ - ANNOTATE_BENIGN_RACE_SIZED(&static_var, \ - sizeof(static_var), \ - # static_var ": " description); \ - } \ - }; \ - static static_var ## _annotator the ## static_var ## _annotator;\ - } -#else /* DYNAMIC_ANNOTATIONS_ENABLED == 0 */ - - #define ANNOTATE_UNPROTECTED_READ(x) (x) - #define ANNOTATE_BENIGN_RACE_STATIC(static_var, description) /* empty */ - -#endif /* DYNAMIC_ANNOTATIONS_ENABLED */ - -/* Annotalysis, a GCC based static analyzer, is able to understand and use - some of the dynamic annotations defined in this file. However, dynamic - annotations are usually disabled in the opt mode (to avoid additional - runtime overheads) while Annotalysis only works in the opt mode. - In order for Annotalysis to use these dynamic annotations when they - are disabled, we re-define these annotations here. Note that unlike the - original macro definitions above, these macros are expanded to calls to - static inline functions so that the compiler will be able to remove the - calls after the analysis. */ - -#ifdef ANNOTALYSIS_ONLY - - #undef ANNOTALYSIS_ONLY - - /* Undefine and re-define the macros that the static analyzer understands. */ - #undef ANNOTATE_IGNORE_READS_BEGIN - #define ANNOTATE_IGNORE_READS_BEGIN() \ - AnnotateIgnoreReadsBegin(__FILE__, __LINE__) - - #undef ANNOTATE_IGNORE_READS_END - #define ANNOTATE_IGNORE_READS_END() \ - AnnotateIgnoreReadsEnd(__FILE__, __LINE__) - - #undef ANNOTATE_IGNORE_WRITES_BEGIN - #define ANNOTATE_IGNORE_WRITES_BEGIN() \ - AnnotateIgnoreWritesBegin(__FILE__, __LINE__) - - #undef ANNOTATE_IGNORE_WRITES_END - #define ANNOTATE_IGNORE_WRITES_END() \ - AnnotateIgnoreWritesEnd(__FILE__, __LINE__) - - #undef ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN - #define ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN() \ - do { \ - ANNOTATE_IGNORE_READS_BEGIN(); \ - ANNOTATE_IGNORE_WRITES_BEGIN(); \ - }while(0) \ - - #undef ANNOTATE_IGNORE_READS_AND_WRITES_END - #define ANNOTATE_IGNORE_READS_AND_WRITES_END() \ - do { \ - ANNOTATE_IGNORE_WRITES_END(); \ - ANNOTATE_IGNORE_READS_END(); \ - }while(0) \ - - #if defined(__cplusplus) - #undef ANNOTATE_UNPROTECTED_READ - template - inline T ANNOTATE_UNPROTECTED_READ(const volatile T &x) - ANNOTALYSIS_UNPROTECTED_READ { - ANNOTATE_IGNORE_READS_BEGIN(); - T res = x; - ANNOTATE_IGNORE_READS_END(); - return res; - } - #endif /* __cplusplus */ - -#endif /* ANNOTALYSIS_ONLY */ - -/* Undefine the macros intended only in this file. */ -#undef ANNOTALYSIS_STATIC_INLINE -#undef ANNOTALYSIS_SEMICOLON_OR_EMPTY_BODY - -#endif /* BASE_DYNAMIC_ANNOTATIONS_H_ */ diff --git a/contrib/libtcmalloc/src/base/elf_mem_image.cc b/contrib/libtcmalloc/src/base/elf_mem_image.cc deleted file mode 100644 index d9605609e3a..00000000000 --- a/contrib/libtcmalloc/src/base/elf_mem_image.cc +++ /dev/null @@ -1,443 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Paul Pluzhnikov -// -// Allow dynamic symbol lookup in an in-memory Elf image. -// - -#include "base/elf_mem_image.h" - -#ifdef HAVE_ELF_MEM_IMAGE // defined in elf_mem_image.h - -#include // for size_t, ptrdiff_t -#include "base/logging.h" - -// From binutils/include/elf/common.h (this doesn't appear to be documented -// anywhere else). -// -// /* This flag appears in a Versym structure. It means that the symbol -// is hidden, and is only visible with an explicit version number. -// This is a GNU extension. */ -// #define VERSYM_HIDDEN 0x8000 -// -// /* This is the mask for the rest of the Versym information. */ -// #define VERSYM_VERSION 0x7fff - -#define VERSYM_VERSION 0x7fff - -#if __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wunused-const-variable" -#endif - -namespace base { - -namespace { -template class ElfClass { - public: - static const int kElfClass = -1; - static int ElfBind(const ElfW(Sym) *) { - CHECK(false); // << "Unexpected word size"; - return 0; - } - static int ElfType(const ElfW(Sym) *) { - CHECK(false); // << "Unexpected word size"; - return 0; - } -}; - -template <> class ElfClass<32> { - public: - static const int kElfClass = ELFCLASS32; - static int ElfBind(const ElfW(Sym) *symbol) { - return ELF32_ST_BIND(symbol->st_info); - } - static int ElfType(const ElfW(Sym) *symbol) { - return ELF32_ST_TYPE(symbol->st_info); - } -}; - -template <> class ElfClass<64> { - public: - static const int kElfClass = ELFCLASS64; - static int ElfBind(const ElfW(Sym) *symbol) { - return ELF64_ST_BIND(symbol->st_info); - } - static int ElfType(const ElfW(Sym) *symbol) { - return ELF64_ST_TYPE(symbol->st_info); - } -}; - -typedef ElfClass<__WORDSIZE> CurrentElfClass; - -// Extract an element from one of the ELF tables, cast it to desired type. -// This is just a simple arithmetic and a glorified cast. -// Callers are responsible for bounds checking. -template -const T* GetTableElement(const ElfW(Ehdr) *ehdr, - ElfW(Off) table_offset, - ElfW(Word) element_size, - size_t index) { - return reinterpret_cast(reinterpret_cast(ehdr) - + table_offset - + index * element_size); -} -} // namespace - -const void *const ElfMemImage::kInvalidBase = - reinterpret_cast(~0L); - -ElfMemImage::ElfMemImage(const void *base) { - CHECK(base != kInvalidBase); - Init(base); -} - -int ElfMemImage::GetNumSymbols() const { - if (!hash_) { - return 0; - } - // See http://www.caldera.com/developers/gabi/latest/ch5.dynamic.html#hash - return hash_[1]; -} - -const ElfW(Sym) *ElfMemImage::GetDynsym(int index) const { - CHECK_LT(index, GetNumSymbols()); - return dynsym_ + index; -} - -const ElfW(Versym) *ElfMemImage::GetVersym(int index) const { - CHECK_LT(index, GetNumSymbols()); - return versym_ + index; -} - -const ElfW(Phdr) *ElfMemImage::GetPhdr(int index) const { - CHECK_LT(index, ehdr_->e_phnum); - return GetTableElement(ehdr_, - ehdr_->e_phoff, - ehdr_->e_phentsize, - index); -} - -const char *ElfMemImage::GetDynstr(ElfW(Word) offset) const { - CHECK_LT(offset, strsize_); - return dynstr_ + offset; -} - -const void *ElfMemImage::GetSymAddr(const ElfW(Sym) *sym) const { - if (sym->st_shndx == SHN_UNDEF || sym->st_shndx >= SHN_LORESERVE) { - // Symbol corresponds to "special" (e.g. SHN_ABS) section. - return reinterpret_cast(sym->st_value); - } - CHECK_LT(link_base_, sym->st_value); - return GetTableElement(ehdr_, 0, 1, sym->st_value) - link_base_; -} - -const ElfW(Verdef) *ElfMemImage::GetVerdef(int index) const { - CHECK_LE(index, verdefnum_); - const ElfW(Verdef) *version_definition = verdef_; - while (version_definition->vd_ndx < index && version_definition->vd_next) { - const char *const version_definition_as_char = - reinterpret_cast(version_definition); - version_definition = - reinterpret_cast(version_definition_as_char + - version_definition->vd_next); - } - return version_definition->vd_ndx == index ? version_definition : NULL; -} - -const ElfW(Verdaux) *ElfMemImage::GetVerdefAux( - const ElfW(Verdef) *verdef) const { - return reinterpret_cast(verdef+1); -} - -const char *ElfMemImage::GetVerstr(ElfW(Word) offset) const { - CHECK_LT(offset, strsize_); - return dynstr_ + offset; -} - -void ElfMemImage::Init(const void *base) { - ehdr_ = NULL; - dynsym_ = NULL; - dynstr_ = NULL; - versym_ = NULL; - verdef_ = NULL; - hash_ = NULL; - strsize_ = 0; - verdefnum_ = 0; - link_base_ = ~0L; // Sentinel: PT_LOAD .p_vaddr can't possibly be this. - if (!base) { - return; - } - const intptr_t base_as_uintptr_t = reinterpret_cast(base); - // Fake VDSO has low bit set. - const bool fake_vdso = ((base_as_uintptr_t & 1) != 0); - base = reinterpret_cast(base_as_uintptr_t & ~1); - const char *const base_as_char = reinterpret_cast(base); - if (base_as_char[EI_MAG0] != ELFMAG0 || base_as_char[EI_MAG1] != ELFMAG1 || - base_as_char[EI_MAG2] != ELFMAG2 || base_as_char[EI_MAG3] != ELFMAG3) { - RAW_DCHECK(false, "no ELF magic"); // at %p", base); - return; - } - int elf_class = base_as_char[EI_CLASS]; - if (elf_class != CurrentElfClass::kElfClass) { - DCHECK_EQ(elf_class, CurrentElfClass::kElfClass); - return; - } - switch (base_as_char[EI_DATA]) { - case ELFDATA2LSB: { - if (__LITTLE_ENDIAN != __BYTE_ORDER) { - DCHECK_EQ(__LITTLE_ENDIAN, __BYTE_ORDER); // << ": wrong byte order"; - return; - } - break; - } - case ELFDATA2MSB: { - if (__BIG_ENDIAN != __BYTE_ORDER) { - DCHECK_EQ(__BIG_ENDIAN, __BYTE_ORDER); // << ": wrong byte order"; - return; - } - break; - } - default: { - RAW_DCHECK(false, "unexpected data encoding"); // << base_as_char[EI_DATA]; - return; - } - } - - ehdr_ = reinterpret_cast(base); - const ElfW(Phdr) *dynamic_program_header = NULL; - for (int i = 0; i < ehdr_->e_phnum; ++i) { - const ElfW(Phdr) *const program_header = GetPhdr(i); - switch (program_header->p_type) { - case PT_LOAD: - if (link_base_ == ~0L) { - link_base_ = program_header->p_vaddr; - } - break; - case PT_DYNAMIC: - dynamic_program_header = program_header; - break; - } - } - if (link_base_ == ~0L || !dynamic_program_header) { - RAW_DCHECK(~0L != link_base_, "no PT_LOADs in VDSO"); - RAW_DCHECK(dynamic_program_header, "no PT_DYNAMIC in VDSO"); - // Mark this image as not present. Can not recur infinitely. - Init(0); - return; - } - ptrdiff_t relocation = - base_as_char - reinterpret_cast(link_base_); - ElfW(Dyn) *dynamic_entry = - reinterpret_cast(dynamic_program_header->p_vaddr + - relocation); - for (; dynamic_entry->d_tag != DT_NULL; ++dynamic_entry) { - ElfW(Xword) value = dynamic_entry->d_un.d_val; - if (fake_vdso) { - // A complication: in the real VDSO, dynamic entries are not relocated - // (it wasn't loaded by a dynamic loader). But when testing with a - // "fake" dlopen()ed vdso library, the loader relocates some (but - // not all!) of them before we get here. - if (dynamic_entry->d_tag == DT_VERDEF) { - // The only dynamic entry (of the ones we care about) libc-2.3.6 - // loader doesn't relocate. - value += relocation; - } - } else { - // Real VDSO. Everything needs to be relocated. - value += relocation; - } - switch (dynamic_entry->d_tag) { - case DT_HASH: - hash_ = reinterpret_cast(value); - break; - case DT_SYMTAB: - dynsym_ = reinterpret_cast(value); - break; - case DT_STRTAB: - dynstr_ = reinterpret_cast(value); - break; - case DT_VERSYM: - versym_ = reinterpret_cast(value); - break; - case DT_VERDEF: - verdef_ = reinterpret_cast(value); - break; - case DT_VERDEFNUM: - verdefnum_ = dynamic_entry->d_un.d_val; - break; - case DT_STRSZ: - strsize_ = dynamic_entry->d_un.d_val; - break; - default: - // Unrecognized entries explicitly ignored. - break; - } - } - if (!hash_ || !dynsym_ || !dynstr_ || !versym_ || - !verdef_ || !verdefnum_ || !strsize_) { - RAW_DCHECK(hash_, "invalid VDSO (no DT_HASH)"); - RAW_DCHECK(dynsym_, "invalid VDSO (no DT_SYMTAB)"); - RAW_DCHECK(dynstr_, "invalid VDSO (no DT_STRTAB)"); - RAW_DCHECK(versym_, "invalid VDSO (no DT_VERSYM)"); - RAW_DCHECK(verdef_, "invalid VDSO (no DT_VERDEF)"); - RAW_DCHECK(verdefnum_, "invalid VDSO (no DT_VERDEFNUM)"); - RAW_DCHECK(strsize_, "invalid VDSO (no DT_STRSZ)"); - // Mark this image as not present. Can not recur infinitely. - Init(0); - return; - } -} - -bool ElfMemImage::LookupSymbol(const char *name, - const char *version, - int type, - SymbolInfo *info) const { - for (SymbolIterator it = begin(); it != end(); ++it) { - if (strcmp(it->name, name) == 0 && strcmp(it->version, version) == 0 && - CurrentElfClass::ElfType(it->symbol) == type) { - if (info) { - *info = *it; - } - return true; - } - } - return false; -} - -bool ElfMemImage::LookupSymbolByAddress(const void *address, - SymbolInfo *info_out) const { - for (SymbolIterator it = begin(); it != end(); ++it) { - const char *const symbol_start = - reinterpret_cast(it->address); - const char *const symbol_end = symbol_start + it->symbol->st_size; - if (symbol_start <= address && address < symbol_end) { - if (info_out) { - // Client wants to know details for that symbol (the usual case). - if (CurrentElfClass::ElfBind(it->symbol) == STB_GLOBAL) { - // Strong symbol; just return it. - *info_out = *it; - return true; - } else { - // Weak or local. Record it, but keep looking for a strong one. - *info_out = *it; - } - } else { - // Client only cares if there is an overlapping symbol. - return true; - } - } - } - return false; -} - -ElfMemImage::SymbolIterator::SymbolIterator(const void *const image, int index) - : index_(index), image_(image) { -} - -const ElfMemImage::SymbolInfo *ElfMemImage::SymbolIterator::operator->() const { - return &info_; -} - -const ElfMemImage::SymbolInfo& ElfMemImage::SymbolIterator::operator*() const { - return info_; -} - -bool ElfMemImage::SymbolIterator::operator==(const SymbolIterator &rhs) const { - return this->image_ == rhs.image_ && this->index_ == rhs.index_; -} - -bool ElfMemImage::SymbolIterator::operator!=(const SymbolIterator &rhs) const { - return !(*this == rhs); -} - -ElfMemImage::SymbolIterator &ElfMemImage::SymbolIterator::operator++() { - this->Update(1); - return *this; -} - -ElfMemImage::SymbolIterator ElfMemImage::begin() const { - SymbolIterator it(this, 0); - it.Update(0); - return it; -} - -ElfMemImage::SymbolIterator ElfMemImage::end() const { - return SymbolIterator(this, GetNumSymbols()); -} - -void ElfMemImage::SymbolIterator::Update(int increment) { - const ElfMemImage *image = reinterpret_cast(image_); - CHECK(image->IsPresent() || increment == 0); - if (!image->IsPresent()) { - return; - } - index_ += increment; - if (index_ >= image->GetNumSymbols()) { - index_ = image->GetNumSymbols(); - return; - } - const ElfW(Sym) *symbol = image->GetDynsym(index_); - const ElfW(Versym) *version_symbol = image->GetVersym(index_); - CHECK(symbol && version_symbol); - const char *const symbol_name = image->GetDynstr(symbol->st_name); - const ElfW(Versym) version_index = version_symbol[0] & VERSYM_VERSION; - const ElfW(Verdef) *version_definition = NULL; - const char *version_name = ""; - if (symbol->st_shndx == SHN_UNDEF) { - // Undefined symbols reference DT_VERNEED, not DT_VERDEF, and - // version_index could well be greater than verdefnum_, so calling - // GetVerdef(version_index) may trigger assertion. - } else { - version_definition = image->GetVerdef(version_index); - } - if (version_definition) { - // I am expecting 1 or 2 auxiliary entries: 1 for the version itself, - // optional 2nd if the version has a parent. - CHECK_LE(1, version_definition->vd_cnt); - CHECK_LE(version_definition->vd_cnt, 2); - const ElfW(Verdaux) *version_aux = image->GetVerdefAux(version_definition); - version_name = image->GetVerstr(version_aux->vda_name); - } - info_.name = symbol_name; - info_.version = version_name; - info_.address = image->GetSymAddr(symbol); - info_.symbol = symbol; -} - -} // namespace base - -#if __clang__ -#pragma clang diagnostic pop -#endif - -#endif // HAVE_ELF_MEM_IMAGE diff --git a/contrib/libtcmalloc/src/base/elf_mem_image.h b/contrib/libtcmalloc/src/base/elf_mem_image.h deleted file mode 100644 index df63cf8b4da..00000000000 --- a/contrib/libtcmalloc/src/base/elf_mem_image.h +++ /dev/null @@ -1,135 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Paul Pluzhnikov -// -// Allow dynamic symbol lookup for in-memory Elf images. - -#ifndef BASE_ELF_MEM_IMAGE_H_ -#define BASE_ELF_MEM_IMAGE_H_ - -#include "../config.h" -#ifdef HAVE_FEATURES_H -#include // for __GLIBC__ -#endif - -// Maybe one day we can rewrite this file not to require the elf -// symbol extensions in glibc, but for right now we need them. -#if defined(__ELF__) && defined(__GLIBC__) && !defined(__native_client__) - -#define HAVE_ELF_MEM_IMAGE 1 - -#include -#include // for ElfW - -namespace base { - -// An in-memory ELF image (may not exist on disk). -class ElfMemImage { - public: - // Sentinel: there could never be an elf image at this address. - static const void *const kInvalidBase; - - // Information about a single vdso symbol. - // All pointers are into .dynsym, .dynstr, or .text of the VDSO. - // Do not free() them or modify through them. - struct SymbolInfo { - const char *name; // E.g. "__vdso_getcpu" - const char *version; // E.g. "LINUX_2.6", could be "" - // for unversioned symbol. - const void *address; // Relocated symbol address. - const ElfW(Sym) *symbol; // Symbol in the dynamic symbol table. - }; - - // Supports iteration over all dynamic symbols. - class SymbolIterator { - public: - friend class ElfMemImage; - const SymbolInfo *operator->() const; - const SymbolInfo &operator*() const; - SymbolIterator& operator++(); - bool operator!=(const SymbolIterator &rhs) const; - bool operator==(const SymbolIterator &rhs) const; - private: - SymbolIterator(const void *const image, int index); - void Update(int incr); - SymbolInfo info_; - int index_; - const void *const image_; - }; - - - explicit ElfMemImage(const void *base); - void Init(const void *base); - bool IsPresent() const { return ehdr_ != NULL; } - const ElfW(Phdr)* GetPhdr(int index) const; - const ElfW(Sym)* GetDynsym(int index) const; - const ElfW(Versym)* GetVersym(int index) const; - const ElfW(Verdef)* GetVerdef(int index) const; - const ElfW(Verdaux)* GetVerdefAux(const ElfW(Verdef) *verdef) const; - const char* GetDynstr(ElfW(Word) offset) const; - const void* GetSymAddr(const ElfW(Sym) *sym) const; - const char* GetVerstr(ElfW(Word) offset) const; - int GetNumSymbols() const; - - SymbolIterator begin() const; - SymbolIterator end() const; - - // Look up versioned dynamic symbol in the image. - // Returns false if image is not present, or doesn't contain given - // symbol/version/type combination. - // If info_out != NULL, additional details are filled in. - bool LookupSymbol(const char *name, const char *version, - int symbol_type, SymbolInfo *info_out) const; - - // Find info about symbol (if any) which overlaps given address. - // Returns true if symbol was found; false if image isn't present - // or doesn't have a symbol overlapping given address. - // If info_out != NULL, additional details are filled in. - bool LookupSymbolByAddress(const void *address, SymbolInfo *info_out) const; - - private: - const ElfW(Ehdr) *ehdr_; - const ElfW(Sym) *dynsym_; - const ElfW(Versym) *versym_; - const ElfW(Verdef) *verdef_; - const ElfW(Word) *hash_; - const char *dynstr_; - size_t strsize_; - size_t verdefnum_; - ElfW(Addr) link_base_; // Link-time base (p_vaddr of first PT_LOAD). -}; - -} // namespace base - -#endif // __ELF__ and __GLIBC__ and !__native_client__ - -#endif // BASE_ELF_MEM_IMAGE_H_ diff --git a/contrib/libtcmalloc/src/base/elfcore.h b/contrib/libtcmalloc/src/base/elfcore.h deleted file mode 100644 index 98fd23b6738..00000000000 --- a/contrib/libtcmalloc/src/base/elfcore.h +++ /dev/null @@ -1,401 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2005-2008, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Markus Gutschke, Carl Crous - */ - -#ifndef _ELFCORE_H -#define _ELFCORE_H -#ifdef __cplusplus -extern "C" { -#endif - -/* We currently only support x86-32, x86-64, ARM, MIPS, PPC on Linux. - * Porting to other related platforms should not be difficult. - */ -#if (defined(__i386__) || defined(__x86_64__) || defined(__arm__) || \ - defined(__mips__) || defined(__PPC__)) && defined(__linux) - -#include -#include -#include -#include "../config.h" - - -/* Define the DUMPER symbol to make sure that there is exactly one - * core dumper built into the library. - */ -#define DUMPER "ELF" - -/* By the time that we get a chance to read CPU registers in the - * calling thread, they are already in a not particularly useful - * state. Besides, there will be multiple frames on the stack that are - * just making the core file confusing. To fix this problem, we take a - * snapshot of the frame pointer, stack pointer, and instruction - * pointer at an earlier time, and then insert these values into the - * core file. - */ - -#if defined(__i386__) || defined(__x86_64__) - typedef struct i386_regs { /* Normal (non-FPU) CPU registers */ - #ifdef __x86_64__ - #define BP rbp - #define SP rsp - #define IP rip - uint64_t r15,r14,r13,r12,rbp,rbx,r11,r10; - uint64_t r9,r8,rax,rcx,rdx,rsi,rdi,orig_rax; - uint64_t rip,cs,eflags; - uint64_t rsp,ss; - uint64_t fs_base, gs_base; - uint64_t ds,es,fs,gs; - #else - #define BP ebp - #define SP esp - #define IP eip - uint32_t ebx, ecx, edx, esi, edi, ebp, eax; - uint16_t ds, __ds, es, __es; - uint16_t fs, __fs, gs, __gs; - uint32_t orig_eax, eip; - uint16_t cs, __cs; - uint32_t eflags, esp; - uint16_t ss, __ss; - #endif - } i386_regs; -#elif defined(__arm__) - typedef struct arm_regs { /* General purpose registers */ - #define BP uregs[11] /* Frame pointer */ - #define SP uregs[13] /* Stack pointer */ - #define IP uregs[15] /* Program counter */ - #define LR uregs[14] /* Link register */ - long uregs[18]; - } arm_regs; -#elif defined(__mips__) - typedef struct mips_regs { - unsigned long pad[6]; /* Unused padding to match kernel structures */ - unsigned long uregs[32]; /* General purpose registers. */ - unsigned long hi; /* Used for multiplication and division. */ - unsigned long lo; - unsigned long cp0_epc; /* Program counter. */ - unsigned long cp0_badvaddr; - unsigned long cp0_status; - unsigned long cp0_cause; - unsigned long unused; - } mips_regs; -#elif defined (__PPC__) - typedef struct ppc_regs { - #define SP uregs[1] /* Stack pointer */ - #define IP rip /* Program counter */ - #define LR lr /* Link register */ - unsigned long uregs[32]; /* General Purpose Registers - r0-r31. */ - double fpr[32]; /* Floating-Point Registers - f0-f31. */ - unsigned long rip; /* Program counter. */ - unsigned long msr; - unsigned long ccr; - unsigned long lr; - unsigned long ctr; - unsigned long xeq; - unsigned long mq; - } ppc_regs; -#endif - -#if defined(__i386__) && defined(__GNUC__) - /* On x86 we provide an optimized version of the FRAME() macro, if the - * compiler supports a GCC-style asm() directive. This results in somewhat - * more accurate values for CPU registers. - */ - typedef struct Frame { - struct i386_regs uregs; - int errno_; - pid_t tid; - } Frame; - #define FRAME(f) Frame f; \ - do { \ - f.errno_ = errno; \ - f.tid = sys_gettid(); \ - __asm__ volatile ( \ - "push %%ebp\n" \ - "push %%ebx\n" \ - "mov %%ebx,0(%%eax)\n" \ - "mov %%ecx,4(%%eax)\n" \ - "mov %%edx,8(%%eax)\n" \ - "mov %%esi,12(%%eax)\n" \ - "mov %%edi,16(%%eax)\n" \ - "mov %%ebp,20(%%eax)\n" \ - "mov %%eax,24(%%eax)\n" \ - "mov %%ds,%%ebx\n" \ - "mov %%ebx,28(%%eax)\n" \ - "mov %%es,%%ebx\n" \ - "mov %%ebx,32(%%eax)\n" \ - "mov %%fs,%%ebx\n" \ - "mov %%ebx,36(%%eax)\n" \ - "mov %%gs,%%ebx\n" \ - "mov %%ebx, 40(%%eax)\n" \ - "call 0f\n" \ - "0:pop %%ebx\n" \ - "add $1f-0b,%%ebx\n" \ - "mov %%ebx,48(%%eax)\n" \ - "mov %%cs,%%ebx\n" \ - "mov %%ebx,52(%%eax)\n" \ - "pushf\n" \ - "pop %%ebx\n" \ - "mov %%ebx,56(%%eax)\n" \ - "mov %%esp,%%ebx\n" \ - "add $8,%%ebx\n" \ - "mov %%ebx,60(%%eax)\n" \ - "mov %%ss,%%ebx\n" \ - "mov %%ebx,64(%%eax)\n" \ - "pop %%ebx\n" \ - "pop %%ebp\n" \ - "1:" \ - : : "a" (&f) : "memory"); \ - } while (0) - #define SET_FRAME(f,r) \ - do { \ - errno = (f).errno_; \ - (r) = (f).uregs; \ - } while (0) -#elif defined(__x86_64__) && defined(__GNUC__) - /* The FRAME and SET_FRAME macros for x86_64. */ - typedef struct Frame { - struct i386_regs uregs; - int errno_; - pid_t tid; - } Frame; - #define FRAME(f) Frame f; \ - do { \ - f.errno_ = errno; \ - f.tid = sys_gettid(); \ - __asm__ volatile ( \ - "push %%rbp\n" \ - "push %%rbx\n" \ - "mov %%r15,0(%%rax)\n" \ - "mov %%r14,8(%%rax)\n" \ - "mov %%r13,16(%%rax)\n" \ - "mov %%r12,24(%%rax)\n" \ - "mov %%rbp,32(%%rax)\n" \ - "mov %%rbx,40(%%rax)\n" \ - "mov %%r11,48(%%rax)\n" \ - "mov %%r10,56(%%rax)\n" \ - "mov %%r9,64(%%rax)\n" \ - "mov %%r8,72(%%rax)\n" \ - "mov %%rax,80(%%rax)\n" \ - "mov %%rcx,88(%%rax)\n" \ - "mov %%rdx,96(%%rax)\n" \ - "mov %%rsi,104(%%rax)\n" \ - "mov %%rdi,112(%%rax)\n" \ - "mov %%ds,%%rbx\n" \ - "mov %%rbx,184(%%rax)\n" \ - "mov %%es,%%rbx\n" \ - "mov %%rbx,192(%%rax)\n" \ - "mov %%fs,%%rbx\n" \ - "mov %%rbx,200(%%rax)\n" \ - "mov %%gs,%%rbx\n" \ - "mov %%rbx,208(%%rax)\n" \ - "call 0f\n" \ - "0:pop %%rbx\n" \ - "add $1f-0b,%%rbx\n" \ - "mov %%rbx,128(%%rax)\n" \ - "mov %%cs,%%rbx\n" \ - "mov %%rbx,136(%%rax)\n" \ - "pushf\n" \ - "pop %%rbx\n" \ - "mov %%rbx,144(%%rax)\n" \ - "mov %%rsp,%%rbx\n" \ - "add $16,%%ebx\n" \ - "mov %%rbx,152(%%rax)\n" \ - "mov %%ss,%%rbx\n" \ - "mov %%rbx,160(%%rax)\n" \ - "pop %%rbx\n" \ - "pop %%rbp\n" \ - "1:" \ - : : "a" (&f) : "memory"); \ - } while (0) - #define SET_FRAME(f,r) \ - do { \ - errno = (f).errno_; \ - (f).uregs.fs_base = (r).fs_base; \ - (f).uregs.gs_base = (r).gs_base; \ - (r) = (f).uregs; \ - } while (0) -#elif defined(__arm__) && defined(__GNUC__) - /* ARM calling conventions are a little more tricky. A little assembly - * helps in obtaining an accurate snapshot of all registers. - */ - typedef struct Frame { - struct arm_regs arm; - int errno_; - pid_t tid; - } Frame; - #define FRAME(f) Frame f; \ - do { \ - long cpsr; \ - f.errno_ = errno; \ - f.tid = sys_gettid(); \ - __asm__ volatile( \ - "stmia %0, {r0-r15}\n" /* All integer regs */\ - : : "r"(&f.arm) : "memory"); \ - f.arm.uregs[16] = 0; \ - __asm__ volatile( \ - "mrs %0, cpsr\n" /* Condition code reg */\ - : "=r"(cpsr)); \ - f.arm.uregs[17] = cpsr; \ - } while (0) - #define SET_FRAME(f,r) \ - do { \ - /* Don't override the FPU status register. */\ - /* Use the value obtained from ptrace(). This*/\ - /* works, because our code does not perform */\ - /* any FPU operations, itself. */\ - long fps = (f).arm.uregs[16]; \ - errno = (f).errno_; \ - (r) = (f).arm; \ - (r).uregs[16] = fps; \ - } while (0) -#elif defined(__mips__) && defined(__GNUC__) - typedef struct Frame { - struct mips_regs mips_regs; - int errno_; - pid_t tid; - } Frame; - #define MIPSREG(n) ({ register unsigned long r __asm__("$"#n); r; }) - #define FRAME(f) Frame f = { 0 }; \ - do { \ - unsigned long hi, lo; \ - register unsigned long pc __asm__("$31"); \ - f.mips_regs.uregs[ 0] = MIPSREG( 0); \ - f.mips_regs.uregs[ 1] = MIPSREG( 1); \ - f.mips_regs.uregs[ 2] = MIPSREG( 2); \ - f.mips_regs.uregs[ 3] = MIPSREG( 3); \ - f.mips_regs.uregs[ 4] = MIPSREG( 4); \ - f.mips_regs.uregs[ 5] = MIPSREG( 5); \ - f.mips_regs.uregs[ 6] = MIPSREG( 6); \ - f.mips_regs.uregs[ 7] = MIPSREG( 7); \ - f.mips_regs.uregs[ 8] = MIPSREG( 8); \ - f.mips_regs.uregs[ 9] = MIPSREG( 9); \ - f.mips_regs.uregs[10] = MIPSREG(10); \ - f.mips_regs.uregs[11] = MIPSREG(11); \ - f.mips_regs.uregs[12] = MIPSREG(12); \ - f.mips_regs.uregs[13] = MIPSREG(13); \ - f.mips_regs.uregs[14] = MIPSREG(14); \ - f.mips_regs.uregs[15] = MIPSREG(15); \ - f.mips_regs.uregs[16] = MIPSREG(16); \ - f.mips_regs.uregs[17] = MIPSREG(17); \ - f.mips_regs.uregs[18] = MIPSREG(18); \ - f.mips_regs.uregs[19] = MIPSREG(19); \ - f.mips_regs.uregs[20] = MIPSREG(20); \ - f.mips_regs.uregs[21] = MIPSREG(21); \ - f.mips_regs.uregs[22] = MIPSREG(22); \ - f.mips_regs.uregs[23] = MIPSREG(23); \ - f.mips_regs.uregs[24] = MIPSREG(24); \ - f.mips_regs.uregs[25] = MIPSREG(25); \ - f.mips_regs.uregs[26] = MIPSREG(26); \ - f.mips_regs.uregs[27] = MIPSREG(27); \ - f.mips_regs.uregs[28] = MIPSREG(28); \ - f.mips_regs.uregs[29] = MIPSREG(29); \ - f.mips_regs.uregs[30] = MIPSREG(30); \ - f.mips_regs.uregs[31] = MIPSREG(31); \ - __asm__ volatile ("mfhi %0" : "=r"(hi)); \ - __asm__ volatile ("mflo %0" : "=r"(lo)); \ - __asm__ volatile ("jal 1f; 1:nop" : "=r"(pc)); \ - f.mips_regs.hi = hi; \ - f.mips_regs.lo = lo; \ - f.mips_regs.cp0_epc = pc; \ - f.errno_ = errno; \ - f.tid = sys_gettid(); \ - } while (0) - #define SET_FRAME(f,r) \ - do { \ - errno = (f).errno_; \ - memcpy((r).uregs, (f).mips_regs.uregs, \ - 32*sizeof(unsigned long)); \ - (r).hi = (f).mips_regs.hi; \ - (r).lo = (f).mips_regs.lo; \ - (r).cp0_epc = (f).mips_regs.cp0_epc; \ - } while (0) -#else - /* If we do not have a hand-optimized assembly version of the FRAME() - * macro, we cannot reliably unroll the stack. So, we show a few additional - * stack frames for the coredumper. - */ - typedef struct Frame { - pid_t tid; - } Frame; - #define FRAME(f) Frame f; do { f.tid = sys_gettid(); } while (0) - #define SET_FRAME(f,r) do { } while (0) -#endif - - -/* Internal function for generating a core file. This API can change without - * notice and is only supposed to be used internally by the core dumper. - * - * This function works for both single- and multi-threaded core - * dumps. If called as - * - * FRAME(frame); - * InternalGetCoreDump(&frame, 0, NULL, ap); - * - * it creates a core file that only contains information about the - * calling thread. - * - * Optionally, the caller can provide information about other threads - * by passing their process ids in "thread_pids". The process id of - * the caller should not be included in this array. All of the threads - * must have been attached to with ptrace(), prior to calling this - * function. They will be detached when "InternalGetCoreDump()" returns. - * - * This function either returns a file handle that can be read for obtaining - * a core dump, or "-1" in case of an error. In the latter case, "errno" - * will be set appropriately. - * - * While "InternalGetCoreDump()" is not technically async signal safe, you - * might be tempted to invoke it from a signal handler. The code goes to - * great lengths to make a best effort that this will actually work. But in - * any case, you must make sure that you preserve the value of "errno" - * yourself. It is guaranteed to be clobbered otherwise. - * - * Also, "InternalGetCoreDump" is not strictly speaking re-entrant. Again, - * it makes a best effort to behave reasonably when called in a multi- - * threaded environment, but it is ultimately the caller's responsibility - * to provide locking. - */ -int InternalGetCoreDump(void *frame, int num_threads, pid_t *thread_pids, - va_list ap - /* const struct CoreDumpParameters *params, - const char *file_name, - const char *PATH - */); - -#endif - -#ifdef __cplusplus -} -#endif -#endif /* _ELFCORE_H */ diff --git a/contrib/libtcmalloc/src/base/googleinit.h b/contrib/libtcmalloc/src/base/googleinit.h deleted file mode 100644 index 3ea411a325a..00000000000 --- a/contrib/libtcmalloc/src/base/googleinit.h +++ /dev/null @@ -1,74 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Jacob Hoffman-Andrews - -#ifndef _GOOGLEINIT_H -#define _GOOGLEINIT_H - -#include "base/logging.h" - -class GoogleInitializer { - public: - typedef void (*VoidFunction)(void); - GoogleInitializer(const char* name, VoidFunction ctor, VoidFunction dtor) - : name_(name), destructor_(dtor) { - RAW_VLOG(10, " constructing: %s\n", name_); - if (ctor) - ctor(); - } - ~GoogleInitializer() { - RAW_VLOG(10, " destroying: %s\n", name_); - if (destructor_) - destructor_(); - } - - private: - const char* const name_; - const VoidFunction destructor_; -}; - -#define REGISTER_MODULE_INITIALIZER(name, body) \ - namespace { \ - static void google_init_module_##name () { body; } \ - GoogleInitializer google_initializer_module_##name(#name, \ - google_init_module_##name, NULL); \ - } - -#define REGISTER_MODULE_DESTRUCTOR(name, body) \ - namespace { \ - static void google_destruct_module_##name () { body; } \ - GoogleInitializer google_destructor_module_##name(#name, \ - NULL, google_destruct_module_##name); \ - } - - -#endif /* _GOOGLEINIT_H */ diff --git a/contrib/libtcmalloc/src/base/linux_syscall_support.h b/contrib/libtcmalloc/src/base/linux_syscall_support.h deleted file mode 100644 index 6a94dc3fc72..00000000000 --- a/contrib/libtcmalloc/src/base/linux_syscall_support.h +++ /dev/null @@ -1,2880 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2005-2008, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Markus Gutschke - */ - -/* This file includes Linux-specific support functions common to the - * coredumper and the thread lister; primarily, this is a collection - * of direct system calls, and a couple of symbols missing from - * standard header files. - * There are a few options that the including file can set to control - * the behavior of this file: - * - * SYS_CPLUSPLUS: - * The entire header file will normally be wrapped in 'extern "C" { }", - * making it suitable for compilation as both C and C++ source. If you - * do not want to do this, you can set the SYS_CPLUSPLUS macro to inhibit - * the wrapping. N.B. doing so will suppress inclusion of all prerequisite - * system header files, too. It is the caller's responsibility to provide - * the necessary definitions. - * - * SYS_ERRNO: - * All system calls will update "errno" unless overriden by setting the - * SYS_ERRNO macro prior to including this file. SYS_ERRNO should be - * an l-value. - * - * SYS_INLINE: - * New symbols will be defined "static inline", unless overridden by - * the SYS_INLINE macro. - * - * SYS_LINUX_SYSCALL_SUPPORT_H - * This macro is used to avoid multiple inclusions of this header file. - * If you need to include this file more than once, make sure to - * unset SYS_LINUX_SYSCALL_SUPPORT_H before each inclusion. - * - * SYS_PREFIX: - * New system calls will have a prefix of "sys_" unless overridden by - * the SYS_PREFIX macro. Valid values for this macro are [0..9] which - * results in prefixes "sys[0..9]_". It is also possible to set this - * macro to -1, which avoids all prefixes. - * - * This file defines a few internal symbols that all start with "LSS_". - * Do not access these symbols from outside this file. They are not part - * of the supported API. - * - * NOTE: This is a stripped down version of the official opensource - * version of linux_syscall_support.h, which lives at - * http://code.google.com/p/linux-syscall-support/ - * It includes only the syscalls that are used in perftools, plus a - * few extra. Here's the breakdown: - * 1) Perftools uses these: grep -rho 'sys_[a-z0-9_A-Z]* *(' src | sort -u - * sys__exit( - * sys_clone( - * sys_close( - * sys_fcntl( - * sys_fstat( - * sys_futex( - * sys_getcpu( - * sys_getdents64( - * sys_getppid( - * sys_gettid( - * sys_lseek( - * sys_mmap( - * sys_mremap( - * sys_munmap( - * sys_open( - * sys_pipe( - * sys_prctl( - * sys_ptrace( - * sys_ptrace_detach( - * sys_read( - * sys_sched_yield( - * sys_sigaction( - * sys_sigaltstack( - * sys_sigdelset( - * sys_sigfillset( - * sys_sigprocmask( - * sys_socket( - * sys_stat( - * sys_waitpid( - * 2) These are used as subroutines of the above: - * sys_getpid -- gettid - * sys_kill -- ptrace_detach - * sys_restore -- sigaction - * sys_restore_rt -- sigaction - * sys_socketcall -- socket - * sys_wait4 -- waitpid - * 3) I left these in even though they're not used. They either - * complement the above (write vs read) or are variants (rt_sigaction): - * sys_fstat64 - * sys_llseek - * sys_mmap2 - * sys_openat - * sys_getdents - * sys_rt_sigaction - * sys_rt_sigprocmask - * sys_sigaddset - * sys_sigemptyset - * sys_stat64 - * sys_write - */ -#ifndef SYS_LINUX_SYSCALL_SUPPORT_H -#define SYS_LINUX_SYSCALL_SUPPORT_H - -/* We currently only support x86-32, x86-64, ARM, MIPS, PPC/PPC64, Aarch64, s390 and s390x - * on Linux. - * Porting to other related platforms should not be difficult. - */ -#if (defined(__i386__) || defined(__x86_64__) || defined(__arm__) || \ - defined(__mips__) || defined(__PPC__) || \ - defined(__aarch64__) || defined(__s390__)) \ - && (defined(__linux)) - -#ifndef SYS_CPLUSPLUS -#ifdef __cplusplus -/* Some system header files in older versions of gcc neglect to properly - * handle being included from C++. As it appears to be harmless to have - * multiple nested 'extern "C"' blocks, just add another one here. - */ -extern "C" { -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __mips__ -/* Include definitions of the ABI currently in use. */ -#include -#endif - -#endif - -/* As glibc often provides subtly incompatible data structures (and implicit - * wrapper functions that convert them), we provide our own kernel data - * structures for use by the system calls. - * These structures have been developed by using Linux 2.6.23 headers for - * reference. Note though, we do not care about exact API compatibility - * with the kernel, and in fact the kernel often does not have a single - * API that works across architectures. Instead, we try to mimic the glibc - * API where reasonable, and only guarantee ABI compatibility with the - * kernel headers. - * Most notably, here are a few changes that were made to the structures - * defined by kernel headers: - * - * - we only define structures, but not symbolic names for kernel data - * types. For the latter, we directly use the native C datatype - * (i.e. "unsigned" instead of "mode_t"). - * - in a few cases, it is possible to define identical structures for - * both 32bit (e.g. i386) and 64bit (e.g. x86-64) platforms by - * standardizing on the 64bit version of the data types. In particular, - * this means that we use "unsigned" where the 32bit headers say - * "unsigned long". - * - overall, we try to minimize the number of cases where we need to - * conditionally define different structures. - * - the "struct kernel_sigaction" class of structures have been - * modified to more closely mimic glibc's API by introducing an - * anonymous union for the function pointer. - * - a small number of field names had to have an underscore appended to - * them, because glibc defines a global macro by the same name. - */ - -/* include/linux/dirent.h */ -struct kernel_dirent64 { - unsigned long long d_ino; - long long d_off; - unsigned short d_reclen; - unsigned char d_type; - char d_name[256]; -}; - -/* include/linux/dirent.h */ -struct kernel_dirent { - long d_ino; - long d_off; - unsigned short d_reclen; - char d_name[256]; -}; - -/* include/linux/time.h */ -struct kernel_timespec { - long tv_sec; - long tv_nsec; -}; - -/* include/linux/time.h */ -struct kernel_timeval { - long tv_sec; - long tv_usec; -}; - -/* include/linux/resource.h */ -struct kernel_rusage { - struct kernel_timeval ru_utime; - struct kernel_timeval ru_stime; - long ru_maxrss; - long ru_ixrss; - long ru_idrss; - long ru_isrss; - long ru_minflt; - long ru_majflt; - long ru_nswap; - long ru_inblock; - long ru_oublock; - long ru_msgsnd; - long ru_msgrcv; - long ru_nsignals; - long ru_nvcsw; - long ru_nivcsw; -}; - -#if defined(__i386__) || defined(__arm__) \ - || defined(__PPC__) || (defined(__s390__) && !defined(__s390x__)) - -/* include/asm-{arm,i386,mips,ppc}/signal.h */ -struct kernel_old_sigaction { - union { - void (*sa_handler_)(int); - void (*sa_sigaction_)(int, siginfo_t *, void *); - }; - unsigned long sa_mask; - unsigned long sa_flags; - void (*sa_restorer)(void); -} __attribute__((packed,aligned(4))); -#elif (defined(__mips__) && _MIPS_SIM == _MIPS_SIM_ABI32) - #define kernel_old_sigaction kernel_sigaction -#elif defined(__aarch64__) - // No kernel_old_sigaction defined for arm64. -#endif - -/* Some kernel functions (e.g. sigaction() in 2.6.23) require that the - * exactly match the size of the signal set, even though the API was - * intended to be extensible. We define our own KERNEL_NSIG to deal with - * this. - * Please note that glibc provides signals [1.._NSIG-1], whereas the - * kernel (and this header) provides the range [1..KERNEL_NSIG]. The - * actual number of signals is obviously the same, but the constants - * differ by one. - */ -#ifdef __mips__ -#define KERNEL_NSIG 128 -#else -#define KERNEL_NSIG 64 -#endif - -/* include/asm-{arm,i386,mips,x86_64}/signal.h */ -struct kernel_sigset_t { - unsigned long sig[(KERNEL_NSIG + 8*sizeof(unsigned long) - 1)/ - (8*sizeof(unsigned long))]; -}; - -/* include/asm-{arm,generic,i386,mips,x86_64,ppc}/signal.h */ -struct kernel_sigaction { -#ifdef __mips__ - unsigned long sa_flags; - union { - void (*sa_handler_)(int); - void (*sa_sigaction_)(int, siginfo_t *, void *); - }; - struct kernel_sigset_t sa_mask; -#else - union { - void (*sa_handler_)(int); - void (*sa_sigaction_)(int, siginfo_t *, void *); - }; - unsigned long sa_flags; - void (*sa_restorer)(void); - struct kernel_sigset_t sa_mask; -#endif -}; - -/* include/asm-{arm,i386,mips,ppc,s390}/stat.h */ -#ifdef __mips__ -#if _MIPS_SIM == _MIPS_SIM_ABI64 -struct kernel_stat { -#else -struct kernel_stat64 { -#endif - unsigned st_dev; - unsigned __pad0[3]; - unsigned long long st_ino; - unsigned st_mode; - unsigned st_nlink; - unsigned st_uid; - unsigned st_gid; - unsigned st_rdev; - unsigned __pad1[3]; - long long st_size; - unsigned st_atime_; - unsigned st_atime_nsec_; - unsigned st_mtime_; - unsigned st_mtime_nsec_; - unsigned st_ctime_; - unsigned st_ctime_nsec_; - unsigned st_blksize; - unsigned __pad2; - unsigned long long st_blocks; -}; -#elif defined __PPC__ -struct kernel_stat64 { - unsigned long long st_dev; - unsigned long long st_ino; - unsigned st_nlink; - unsigned st_mode; - unsigned st_uid; - unsigned st_gid; - int __pad2; - unsigned long long st_rdev; - long long st_size; - long long st_blksize; - long long st_blocks; - kernel_timespec st_atim; - kernel_timespec st_mtim; - kernel_timespec st_ctim; - unsigned long __unused4; - unsigned long __unused5; - unsigned long __unused6; -}; -#else -struct kernel_stat64 { - unsigned long long st_dev; - unsigned char __pad0[4]; - unsigned __st_ino; - unsigned st_mode; - unsigned st_nlink; - unsigned st_uid; - unsigned st_gid; - unsigned long long st_rdev; - unsigned char __pad3[4]; - long long st_size; - unsigned st_blksize; - unsigned long long st_blocks; - unsigned st_atime_; - unsigned st_atime_nsec_; - unsigned st_mtime_; - unsigned st_mtime_nsec_; - unsigned st_ctime_; - unsigned st_ctime_nsec_; - unsigned long long st_ino; -}; -#endif - -/* include/asm-{arm,generic,i386,mips,x86_64,ppc,s390}/stat.h */ -#if defined(__i386__) || defined(__arm__) -struct kernel_stat { - /* The kernel headers suggest that st_dev and st_rdev should be 32bit - * quantities encoding 12bit major and 20bit minor numbers in an interleaved - * format. In reality, we do not see useful data in the top bits. So, - * we'll leave the padding in here, until we find a better solution. - */ - unsigned short st_dev; - short pad1; - unsigned st_ino; - unsigned short st_mode; - unsigned short st_nlink; - unsigned short st_uid; - unsigned short st_gid; - unsigned short st_rdev; - short pad2; - unsigned st_size; - unsigned st_blksize; - unsigned st_blocks; - unsigned st_atime_; - unsigned st_atime_nsec_; - unsigned st_mtime_; - unsigned st_mtime_nsec_; - unsigned st_ctime_; - unsigned st_ctime_nsec_; - unsigned __unused4; - unsigned __unused5; -}; -#elif defined(__x86_64__) -struct kernel_stat { - uint64_t st_dev; - uint64_t st_ino; - uint64_t st_nlink; - unsigned st_mode; - unsigned st_uid; - unsigned st_gid; - unsigned __pad0; - uint64_t st_rdev; - int64_t st_size; - int64_t st_blksize; - int64_t st_blocks; - uint64_t st_atime_; - uint64_t st_atime_nsec_; - uint64_t st_mtime_; - uint64_t st_mtime_nsec_; - uint64_t st_ctime_; - uint64_t st_ctime_nsec_; - int64_t __unused[3]; -}; -#elif defined(__PPC__) -struct kernel_stat { - unsigned long long st_dev; - unsigned long st_ino; - unsigned long st_nlink; - unsigned long st_mode; - unsigned st_uid; - unsigned st_gid; - int __pad2; - unsigned long long st_rdev; - long st_size; - unsigned long st_blksize; - unsigned long st_blocks; - kernel_timespec st_atim; - kernel_timespec st_mtim; - kernel_timespec st_ctim; - unsigned long __unused4; - unsigned long __unused5; - unsigned long __unused6; -}; -#elif (defined(__mips__) && _MIPS_SIM != _MIPS_SIM_ABI64) -struct kernel_stat { - unsigned st_dev; - int st_pad1[3]; - unsigned st_ino; - unsigned st_mode; - unsigned st_nlink; - unsigned st_uid; - unsigned st_gid; - unsigned st_rdev; - int st_pad2[2]; - long st_size; - int st_pad3; - long st_atime_; - long st_atime_nsec_; - long st_mtime_; - long st_mtime_nsec_; - long st_ctime_; - long st_ctime_nsec_; - int st_blksize; - int st_blocks; - int st_pad4[14]; -}; -#elif defined(__aarch64__) -struct kernel_stat { - unsigned long st_dev; - unsigned long st_ino; - unsigned int st_mode; - unsigned int st_nlink; - unsigned int st_uid; - unsigned int st_gid; - unsigned long st_rdev; - unsigned long __pad1; - long st_size; - int st_blksize; - int __pad2; - long st_blocks; - long st_atime_; - unsigned long st_atime_nsec_; - long st_mtime_; - unsigned long st_mtime_nsec_; - long st_ctime_; - unsigned long st_ctime_nsec_; - unsigned int __unused4; - unsigned int __unused5; -}; -#elif defined(__s390x__) -struct kernel_stat { - unsigned long st_dev; - unsigned long st_ino; - unsigned long st_nlink; - unsigned int st_mode; - unsigned int st_uid; - unsigned int st_gid; - unsigned int __pad1; - unsigned long st_rdev; - unsigned long st_size; - unsigned long st_atime_; - unsigned long st_atime_nsec_; - unsigned long st_mtime_; - unsigned long st_mtime_nsec_; - unsigned long st_ctime_; - unsigned long st_ctime_nsec_; - unsigned long st_blksize; - long st_blocks; - unsigned long __unused[3]; -}; -#elif defined(__s390__) -struct kernel_stat { - unsigned short st_dev; - unsigned short __pad1; - unsigned long st_ino; - unsigned short st_mode; - unsigned short st_nlink; - unsigned short st_uid; - unsigned short st_gid; - unsigned short st_rdev; - unsigned short __pad2; - unsigned long st_size; - unsigned long st_blksize; - unsigned long st_blocks; - unsigned long st_atime_; - unsigned long st_atime_nsec_; - unsigned long st_mtime_; - unsigned long st_mtime_nsec_; - unsigned long st_ctime_; - unsigned long st_ctime_nsec_; - unsigned long __unused4; - unsigned long __unused5; -}; -#endif - - -/* Definitions missing from the standard header files */ -#ifndef O_DIRECTORY -#if defined(__arm__) -#define O_DIRECTORY 0040000 -#else -#define O_DIRECTORY 0200000 -#endif -#endif -#ifndef PR_GET_DUMPABLE -#define PR_GET_DUMPABLE 3 -#endif -#ifndef PR_SET_DUMPABLE -#define PR_SET_DUMPABLE 4 -#endif -#ifndef AT_FDCWD -#define AT_FDCWD (-100) -#endif -#ifndef AT_SYMLINK_NOFOLLOW -#define AT_SYMLINK_NOFOLLOW 0x100 -#endif -#ifndef AT_REMOVEDIR -#define AT_REMOVEDIR 0x200 -#endif -#ifndef MREMAP_FIXED -#define MREMAP_FIXED 2 -#endif -#ifndef SA_RESTORER -#define SA_RESTORER 0x04000000 -#endif - -#if defined(__i386__) -#ifndef __NR_rt_sigaction -#define __NR_rt_sigaction 174 -#define __NR_rt_sigprocmask 175 -#endif -#ifndef __NR_stat64 -#define __NR_stat64 195 -#endif -#ifndef __NR_fstat64 -#define __NR_fstat64 197 -#endif -#ifndef __NR_getdents64 -#define __NR_getdents64 220 -#endif -#ifndef __NR_gettid -#define __NR_gettid 224 -#endif -#ifndef __NR_futex -#define __NR_futex 240 -#endif -#ifndef __NR_openat -#define __NR_openat 295 -#endif -#ifndef __NR_getcpu -#define __NR_getcpu 318 -#endif -/* End of i386 definitions */ -#elif defined(__arm__) -#ifndef __syscall -#if defined(__thumb__) || defined(__ARM_EABI__) -#define __SYS_REG(name) register long __sysreg __asm__("r6") = __NR_##name; -#define __SYS_REG_LIST(regs...) [sysreg] "r" (__sysreg) , ##regs -#define __syscall(name) "swi\t0" -#define __syscall_safe(name) \ - "push {r7}\n" \ - "mov r7,%[sysreg]\n" \ - __syscall(name)"\n" \ - "pop {r7}" -#else -#define __SYS_REG(name) -#define __SYS_REG_LIST(regs...) regs -#define __syscall(name) "swi\t" __sys1(__NR_##name) "" -#define __syscall_safe(name) __syscall(name) -#endif -#endif -#ifndef __NR_rt_sigaction -#define __NR_rt_sigaction (__NR_SYSCALL_BASE + 174) -#define __NR_rt_sigprocmask (__NR_SYSCALL_BASE + 175) -#endif -#ifndef __NR_stat64 -#define __NR_stat64 (__NR_SYSCALL_BASE + 195) -#endif -#ifndef __NR_fstat64 -#define __NR_fstat64 (__NR_SYSCALL_BASE + 197) -#endif -#ifndef __NR_getdents64 -#define __NR_getdents64 (__NR_SYSCALL_BASE + 217) -#endif -#ifndef __NR_gettid -#define __NR_gettid (__NR_SYSCALL_BASE + 224) -#endif -#ifndef __NR_futex -#define __NR_futex (__NR_SYSCALL_BASE + 240) -#endif -/* End of ARM definitions */ -#elif defined(__x86_64__) -#ifndef __NR_gettid -#define __NR_gettid 186 -#endif -#ifndef __NR_futex -#define __NR_futex 202 -#endif -#ifndef __NR_getdents64 -#define __NR_getdents64 217 -#endif -#ifndef __NR_openat -#define __NR_openat 257 -#endif -/* End of x86-64 definitions */ -#elif defined(__mips__) -#if _MIPS_SIM == _MIPS_SIM_ABI32 -#ifndef __NR_rt_sigaction -#define __NR_rt_sigaction (__NR_Linux + 194) -#define __NR_rt_sigprocmask (__NR_Linux + 195) -#endif -#ifndef __NR_stat64 -#define __NR_stat64 (__NR_Linux + 213) -#endif -#ifndef __NR_fstat64 -#define __NR_fstat64 (__NR_Linux + 215) -#endif -#ifndef __NR_getdents64 -#define __NR_getdents64 (__NR_Linux + 219) -#endif -#ifndef __NR_gettid -#define __NR_gettid (__NR_Linux + 222) -#endif -#ifndef __NR_futex -#define __NR_futex (__NR_Linux + 238) -#endif -#ifndef __NR_openat -#define __NR_openat (__NR_Linux + 288) -#endif -#ifndef __NR_fstatat -#define __NR_fstatat (__NR_Linux + 293) -#endif -#ifndef __NR_getcpu -#define __NR_getcpu (__NR_Linux + 312) -#endif -/* End of MIPS (old 32bit API) definitions */ -#elif _MIPS_SIM == _MIPS_SIM_ABI64 -#ifndef __NR_gettid -#define __NR_gettid (__NR_Linux + 178) -#endif -#ifndef __NR_futex -#define __NR_futex (__NR_Linux + 194) -#endif -#ifndef __NR_openat -#define __NR_openat (__NR_Linux + 247) -#endif -#ifndef __NR_fstatat -#define __NR_fstatat (__NR_Linux + 252) -#endif -#ifndef __NR_getcpu -#define __NR_getcpu (__NR_Linux + 271) -#endif -/* End of MIPS (64bit API) definitions */ -#else -#ifndef __NR_gettid -#define __NR_gettid (__NR_Linux + 178) -#endif -#ifndef __NR_futex -#define __NR_futex (__NR_Linux + 194) -#endif -#ifndef __NR_openat -#define __NR_openat (__NR_Linux + 251) -#endif -#ifndef __NR_fstatat -#define __NR_fstatat (__NR_Linux + 256) -#endif -#ifndef __NR_getcpu -#define __NR_getcpu (__NR_Linux + 275) -#endif -/* End of MIPS (new 32bit API) definitions */ -#endif -/* End of MIPS definitions */ -#elif defined(__PPC__) -#ifndef __NR_rt_sigaction -#define __NR_rt_sigaction 173 -#define __NR_rt_sigprocmask 174 -#endif -#ifndef __NR_stat64 -#define __NR_stat64 195 -#endif -#ifndef __NR_fstat64 -#define __NR_fstat64 197 -#endif -#ifndef __NR_socket -#define __NR_socket 198 -#endif -#ifndef __NR_getdents64 -#define __NR_getdents64 202 -#endif -#ifndef __NR_gettid -#define __NR_gettid 207 -#endif -#ifndef __NR_futex -#define __NR_futex 221 -#endif -#ifndef __NR_openat -#define __NR_openat 286 -#endif -#ifndef __NR_getcpu -#define __NR_getcpu 302 -#endif -/* End of powerpc defininitions */ -#elif defined(__aarch64__) -#ifndef __NR_fstatat -#define __NR_fstatat 79 -#endif -/* End of aarch64 defininitions */ -#elif defined(__s390__) -#ifndef __NR_quotactl -#define __NR_quotactl 131 -#endif -#ifndef __NR_rt_sigreturn -#define __NR_rt_sigreturn 173 -#endif -#ifndef __NR_rt_sigaction -#define __NR_rt_sigaction 174 -#endif -#ifndef __NR_rt_sigprocmask -#define __NR_rt_sigprocmask 175 -#endif -#ifndef __NR_rt_sigpending -#define __NR_rt_sigpending 176 -#endif -#ifndef __NR_rt_sigsuspend -#define __NR_rt_sigsuspend 179 -#endif -#ifndef __NR_pread64 -#define __NR_pread64 180 -#endif -#ifndef __NR_pwrite64 -#define __NR_pwrite64 181 -#endif -#ifndef __NR_getdents64 -#define __NR_getdents64 220 -#endif -#ifndef __NR_readahead -#define __NR_readahead 222 -#endif -#ifndef __NR_setxattr -#define __NR_setxattr 224 -#endif -#ifndef __NR_lsetxattr -#define __NR_lsetxattr 225 -#endif -#ifndef __NR_getxattr -#define __NR_getxattr 227 -#endif -#ifndef __NR_lgetxattr -#define __NR_lgetxattr 228 -#endif -#ifndef __NR_listxattr -#define __NR_listxattr 230 -#endif -#ifndef __NR_llistxattr -#define __NR_llistxattr 231 -#endif -#ifndef __NR_gettid -#define __NR_gettid 236 -#endif -#ifndef __NR_tkill -#define __NR_tkill 237 -#endif -#ifndef __NR_futex -#define __NR_futex 238 -#endif -#ifndef __NR_sched_setaffinity -#define __NR_sched_setaffinity 239 -#endif -#ifndef __NR_sched_getaffinity -#define __NR_sched_getaffinity 240 -#endif -#ifndef __NR_set_tid_address -#define __NR_set_tid_address 252 -#endif -#ifndef __NR_clock_gettime -#define __NR_clock_gettime 260 -#endif -#ifndef __NR_clock_getres -#define __NR_clock_getres 261 -#endif -#ifndef __NR_statfs64 -#define __NR_statfs64 265 -#endif -#ifndef __NR_fstatfs64 -#define __NR_fstatfs64 266 -#endif -#ifndef __NR_ioprio_set -#define __NR_ioprio_set 282 -#endif -#ifndef __NR_ioprio_get -#define __NR_ioprio_get 283 -#endif -#ifndef __NR_openat -#define __NR_openat 288 -#endif -#ifndef __NR_unlinkat -#define __NR_unlinkat 294 -#endif -#ifndef __NR_move_pages -#define __NR_move_pages 310 -#endif -#ifndef __NR_getcpu -#define __NR_getcpu 311 -#endif -#ifndef __NR_fallocate -#define __NR_fallocate 314 -#endif -/* Some syscalls are named/numbered differently between s390 and s390x. */ -#ifdef __s390x__ -# ifndef __NR_getrlimit -# define __NR_getrlimit 191 -# endif -# ifndef __NR_setresuid -# define __NR_setresuid 208 -# endif -# ifndef __NR_getresuid -# define __NR_getresuid 209 -# endif -# ifndef __NR_setresgid -# define __NR_setresgid 210 -# endif -# ifndef __NR_getresgid -# define __NR_getresgid 211 -# endif -# ifndef __NR_setfsuid -# define __NR_setfsuid 215 -# endif -# ifndef __NR_setfsgid -# define __NR_setfsgid 216 -# endif -# ifndef __NR_fadvise64 -# define __NR_fadvise64 253 -# endif -# ifndef __NR_newfstatat -# define __NR_newfstatat 293 -# endif -#else /* __s390x__ */ -# ifndef __NR_getrlimit -# define __NR_getrlimit 76 -# endif -# ifndef __NR_setfsuid -# define __NR_setfsuid 138 -# endif -# ifndef __NR_setfsgid -# define __NR_setfsgid 139 -# endif -# ifndef __NR_setresuid -# define __NR_setresuid 164 -# endif -# ifndef __NR_getresuid -# define __NR_getresuid 165 -# endif -# ifndef __NR_setresgid -# define __NR_setresgid 170 -# endif -# ifndef __NR_getresgid -# define __NR_getresgid 171 -# endif -# ifndef __NR_ugetrlimit -# define __NR_ugetrlimit 191 -# endif -# ifndef __NR_mmap2 -# define __NR_mmap2 192 -# endif -# ifndef __NR_setresuid32 -# define __NR_setresuid32 208 -# endif -# ifndef __NR_getresuid32 -# define __NR_getresuid32 209 -# endif -# ifndef __NR_setresgid32 -# define __NR_setresgid32 210 -# endif -# ifndef __NR_getresgid32 -# define __NR_getresgid32 211 -# endif -# ifndef __NR_setfsuid32 -# define __NR_setfsuid32 215 -# endif -# ifndef __NR_setfsgid32 -# define __NR_setfsgid32 216 -# endif -# ifndef __NR_fadvise64_64 -# define __NR_fadvise64_64 264 -# endif -# ifndef __NR_fstatat64 -# define __NR_fstatat64 293 -# endif -#endif /* __s390__ */ -/* End of s390/s390x definitions */ -#endif - - -/* After forking, we must make sure to only call system calls. */ -#if __BOUNDED_POINTERS__ - #error "Need to port invocations of syscalls for bounded ptrs" -#else - /* The core dumper and the thread lister get executed after threads - * have been suspended. As a consequence, we cannot call any functions - * that acquire locks. Unfortunately, libc wraps most system calls - * (e.g. in order to implement pthread_atfork, and to make calls - * cancellable), which means we cannot call these functions. Instead, - * we have to call syscall() directly. - */ - #undef LSS_ERRNO - #ifdef SYS_ERRNO - /* Allow the including file to override the location of errno. This can - * be useful when using clone() with the CLONE_VM option. - */ - #define LSS_ERRNO SYS_ERRNO - #else - #define LSS_ERRNO errno - #endif - - #undef LSS_INLINE - #ifdef SYS_INLINE - #define LSS_INLINE SYS_INLINE - #else - #define LSS_INLINE static inline - #endif - - /* Allow the including file to override the prefix used for all new - * system calls. By default, it will be set to "sys_". - */ - #undef LSS_NAME - #ifndef SYS_PREFIX - #define LSS_NAME(name) sys_##name - #elif SYS_PREFIX < 0 - #define LSS_NAME(name) name - #elif SYS_PREFIX == 0 - #define LSS_NAME(name) sys0_##name - #elif SYS_PREFIX == 1 - #define LSS_NAME(name) sys1_##name - #elif SYS_PREFIX == 2 - #define LSS_NAME(name) sys2_##name - #elif SYS_PREFIX == 3 - #define LSS_NAME(name) sys3_##name - #elif SYS_PREFIX == 4 - #define LSS_NAME(name) sys4_##name - #elif SYS_PREFIX == 5 - #define LSS_NAME(name) sys5_##name - #elif SYS_PREFIX == 6 - #define LSS_NAME(name) sys6_##name - #elif SYS_PREFIX == 7 - #define LSS_NAME(name) sys7_##name - #elif SYS_PREFIX == 8 - #define LSS_NAME(name) sys8_##name - #elif SYS_PREFIX == 9 - #define LSS_NAME(name) sys9_##name - #endif - - #undef LSS_RETURN - #if (defined(__i386__) || defined(__x86_64__) || defined(__arm__) || \ - defined(__aarch64__) || defined(__s390__)) - /* Failing system calls return a negative result in the range of - * -1..-4095. These are "errno" values with the sign inverted. - */ - #define LSS_RETURN(type, res) \ - do { \ - if ((unsigned long)(res) >= (unsigned long)(-4095)) { \ - LSS_ERRNO = -(res); \ - res = -1; \ - } \ - return (type) (res); \ - } while (0) - #elif defined(__mips__) - /* On MIPS, failing system calls return -1, and set errno in a - * separate CPU register. - */ - #define LSS_RETURN(type, res, err) \ - do { \ - if (err) { \ - LSS_ERRNO = (res); \ - res = -1; \ - } \ - return (type) (res); \ - } while (0) - #elif defined(__PPC__) - /* On PPC, failing system calls return -1, and set errno in a - * separate CPU register. See linux/unistd.h. - */ - #define LSS_RETURN(type, res, err) \ - do { \ - if (err & 0x10000000 ) { \ - LSS_ERRNO = (res); \ - res = -1; \ - } \ - return (type) (res); \ - } while (0) - #endif - #if defined(__i386__) - #if defined(NO_FRAME_POINTER) && (100 * __GNUC__ + __GNUC_MINOR__ >= 404) - /* This only works for GCC-4.4 and above -- the first version to use - .cfi directives for dwarf unwind info. */ - #define CFI_ADJUST_CFA_OFFSET(adjust) \ - ".cfi_adjust_cfa_offset " #adjust "\n" - #else - #define CFI_ADJUST_CFA_OFFSET(adjust) /**/ - #endif - - /* In PIC mode (e.g. when building shared libraries), gcc for i386 - * reserves ebx. Unfortunately, most distribution ship with implementations - * of _syscallX() which clobber ebx. - * Also, most definitions of _syscallX() neglect to mark "memory" as being - * clobbered. This causes problems with compilers, that do a better job - * at optimizing across __asm__ calls. - * So, we just have to redefine all of the _syscallX() macros. - */ - #undef LSS_BODY - #define LSS_BODY(type,args...) \ - long __res; \ - __asm__ __volatile__("push %%ebx\n" \ - CFI_ADJUST_CFA_OFFSET(4) \ - "movl %2,%%ebx\n" \ - "int $0x80\n" \ - "pop %%ebx\n" \ - CFI_ADJUST_CFA_OFFSET(-4) \ - args \ - : "esp", "memory"); \ - LSS_RETURN(type,__res) - #undef _syscall0 - #define _syscall0(type,name) \ - type LSS_NAME(name)(void) { \ - long __res; \ - __asm__ volatile("int $0x80" \ - : "=a" (__res) \ - : "0" (__NR_##name) \ - : "memory"); \ - LSS_RETURN(type,__res); \ - } - #undef _syscall1 - #define _syscall1(type,name,type1,arg1) \ - type LSS_NAME(name)(type1 arg1) { \ - LSS_BODY(type, \ - : "=a" (__res) \ - : "0" (__NR_##name), "ri" ((long)(arg1))); \ - } - #undef _syscall2 - #define _syscall2(type,name,type1,arg1,type2,arg2) \ - type LSS_NAME(name)(type1 arg1,type2 arg2) { \ - LSS_BODY(type, \ - : "=a" (__res) \ - : "0" (__NR_##name),"ri" ((long)(arg1)), "c" ((long)(arg2))); \ - } - #undef _syscall3 - #define _syscall3(type,name,type1,arg1,type2,arg2,type3,arg3) \ - type LSS_NAME(name)(type1 arg1,type2 arg2,type3 arg3) { \ - LSS_BODY(type, \ - : "=a" (__res) \ - : "0" (__NR_##name), "ri" ((long)(arg1)), "c" ((long)(arg2)), \ - "d" ((long)(arg3))); \ - } - #undef _syscall4 - #define _syscall4(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4) { \ - LSS_BODY(type, \ - : "=a" (__res) \ - : "0" (__NR_##name), "ri" ((long)(arg1)), "c" ((long)(arg2)), \ - "d" ((long)(arg3)),"S" ((long)(arg4))); \ - } - #undef _syscall5 - #define _syscall5(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5) { \ - long __res; \ - __asm__ __volatile__("push %%ebx\n" \ - "movl %2,%%ebx\n" \ - "movl %1,%%eax\n" \ - "int $0x80\n" \ - "pop %%ebx" \ - : "=a" (__res) \ - : "i" (__NR_##name), "ri" ((long)(arg1)), \ - "c" ((long)(arg2)), "d" ((long)(arg3)), \ - "S" ((long)(arg4)), "D" ((long)(arg5)) \ - : "esp", "memory"); \ - LSS_RETURN(type,__res); \ - } - #undef _syscall6 - #define _syscall6(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5,type6,arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5, type6 arg6) { \ - long __res; \ - struct { long __a1; long __a6; } __s = { (long)arg1, (long) arg6 }; \ - __asm__ __volatile__("push %%ebp\n" \ - "push %%ebx\n" \ - "movl 4(%2),%%ebp\n" \ - "movl 0(%2), %%ebx\n" \ - "movl %1,%%eax\n" \ - "int $0x80\n" \ - "pop %%ebx\n" \ - "pop %%ebp" \ - : "=a" (__res) \ - : "i" (__NR_##name), "0" ((long)(&__s)), \ - "c" ((long)(arg2)), "d" ((long)(arg3)), \ - "S" ((long)(arg4)), "D" ((long)(arg5)) \ - : "esp", "memory"); \ - LSS_RETURN(type,__res); \ - } - LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, - int flags, void *arg, int *parent_tidptr, - void *newtls, int *child_tidptr) { - long __res; - __asm__ __volatile__(/* if (fn == NULL) - * return -EINVAL; - */ - "movl %3,%%ecx\n" - "jecxz 1f\n" - - /* if (child_stack == NULL) - * return -EINVAL; - */ - "movl %4,%%ecx\n" - "jecxz 1f\n" - - /* Set up alignment of the child stack: - * child_stack = (child_stack & ~0xF) - 20; - */ - "andl $-16,%%ecx\n" - "subl $20,%%ecx\n" - - /* Push "arg" and "fn" onto the stack that will be - * used by the child. - */ - "movl %6,%%eax\n" - "movl %%eax,4(%%ecx)\n" - "movl %3,%%eax\n" - "movl %%eax,(%%ecx)\n" - - /* %eax = syscall(%eax = __NR_clone, - * %ebx = flags, - * %ecx = child_stack, - * %edx = parent_tidptr, - * %esi = newtls, - * %edi = child_tidptr) - * Also, make sure that %ebx gets preserved as it is - * used in PIC mode. - */ - "movl %8,%%esi\n" - "movl %7,%%edx\n" - "movl %5,%%eax\n" - "movl %9,%%edi\n" - "pushl %%ebx\n" - "movl %%eax,%%ebx\n" - "movl %2,%%eax\n" - "int $0x80\n" - - /* In the parent: restore %ebx - * In the child: move "fn" into %ebx - */ - "popl %%ebx\n" - - /* if (%eax != 0) - * return %eax; - */ - "test %%eax,%%eax\n" - "jnz 1f\n" - - /* In the child, now. Terminate frame pointer chain. - */ - "movl $0,%%ebp\n" - - /* Call "fn". "arg" is already on the stack. - */ - "call *%%ebx\n" - - /* Call _exit(%ebx). Unfortunately older versions - * of gcc restrict the number of arguments that can - * be passed to asm(). So, we need to hard-code the - * system call number. - */ - "movl %%eax,%%ebx\n" - "movl $1,%%eax\n" - "int $0x80\n" - - /* Return to parent. - */ - "1:\n" - : "=a" (__res) - : "0"(-EINVAL), "i"(__NR_clone), - "m"(fn), "m"(child_stack), "m"(flags), "m"(arg), - "m"(parent_tidptr), "m"(newtls), "m"(child_tidptr) - : "esp", "memory", "ecx", "edx", "esi", "edi"); - LSS_RETURN(int, __res); - } - - LSS_INLINE void (*LSS_NAME(restore_rt)(void))(void) { - /* On i386, the kernel does not know how to return from a signal - * handler. Instead, it relies on user space to provide a - * restorer function that calls the {rt_,}sigreturn() system call. - * Unfortunately, we cannot just reference the glibc version of this - * function, as glibc goes out of its way to make it inaccessible. - */ - void (*res)(void); - __asm__ __volatile__("call 2f\n" - "0:.align 16\n" - "1:movl %1,%%eax\n" - "int $0x80\n" - "2:popl %0\n" - "addl $(1b-0b),%0\n" - : "=a" (res) - : "i" (__NR_rt_sigreturn)); - return res; - } - LSS_INLINE void (*LSS_NAME(restore)(void))(void) { - /* On i386, the kernel does not know how to return from a signal - * handler. Instead, it relies on user space to provide a - * restorer function that calls the {rt_,}sigreturn() system call. - * Unfortunately, we cannot just reference the glibc version of this - * function, as glibc goes out of its way to make it inaccessible. - */ - void (*res)(void); - __asm__ __volatile__("call 2f\n" - "0:.align 16\n" - "1:pop %%eax\n" - "movl %1,%%eax\n" - "int $0x80\n" - "2:popl %0\n" - "addl $(1b-0b),%0\n" - : "=a" (res) - : "i" (__NR_sigreturn)); - return res; - } - #elif defined(__x86_64__) - /* There are no known problems with any of the _syscallX() macros - * currently shipping for x86_64, but we still need to be able to define - * our own version so that we can override the location of the errno - * location (e.g. when using the clone() system call with the CLONE_VM - * option). - */ - #undef LSS_ENTRYPOINT - #define LSS_ENTRYPOINT "syscall\n" - - /* The x32 ABI has 32 bit longs, but the syscall interface is 64 bit. - * We need to explicitly cast to an unsigned 64 bit type to avoid implicit - * sign extension. We can't cast pointers directly because those are - * 32 bits, and gcc will dump ugly warnings about casting from a pointer - * to an integer of a different size. - */ - #undef LSS_SYSCALL_ARG - #define LSS_SYSCALL_ARG(a) ((uint64_t)(uintptr_t)(a)) - #undef _LSS_RETURN - #define _LSS_RETURN(type, res, cast) \ - do { \ - if ((uint64_t)(res) >= (uint64_t)(-4095)) { \ - LSS_ERRNO = -(res); \ - res = -1; \ - } \ - return (type)(cast)(res); \ - } while (0) - #undef LSS_RETURN - #define LSS_RETURN(type, res) _LSS_RETURN(type, res, uintptr_t) - - #undef _LSS_BODY - #define _LSS_BODY(nr, type, name, cast, ...) \ - long long __res; \ - __asm__ __volatile__(LSS_BODY_ASM##nr LSS_ENTRYPOINT \ - : "=a" (__res) \ - : "0" (__NR_##name) LSS_BODY_ARG##nr(__VA_ARGS__) \ - : LSS_BODY_CLOBBER##nr "r11", "rcx", "memory"); \ - _LSS_RETURN(type, __res, cast) - #undef LSS_BODY - #define LSS_BODY(nr, type, name, args...) \ - _LSS_BODY(nr, type, name, uintptr_t, ## args) - - #undef LSS_BODY_ASM0 - #undef LSS_BODY_ASM1 - #undef LSS_BODY_ASM2 - #undef LSS_BODY_ASM3 - #undef LSS_BODY_ASM4 - #undef LSS_BODY_ASM5 - #undef LSS_BODY_ASM6 - #define LSS_BODY_ASM0 - #define LSS_BODY_ASM1 LSS_BODY_ASM0 - #define LSS_BODY_ASM2 LSS_BODY_ASM1 - #define LSS_BODY_ASM3 LSS_BODY_ASM2 - #define LSS_BODY_ASM4 LSS_BODY_ASM3 "movq %5,%%r10;" - #define LSS_BODY_ASM5 LSS_BODY_ASM4 "movq %6,%%r8;" - #define LSS_BODY_ASM6 LSS_BODY_ASM5 "movq %7,%%r9;" - - #undef LSS_BODY_CLOBBER0 - #undef LSS_BODY_CLOBBER1 - #undef LSS_BODY_CLOBBER2 - #undef LSS_BODY_CLOBBER3 - #undef LSS_BODY_CLOBBER4 - #undef LSS_BODY_CLOBBER5 - #undef LSS_BODY_CLOBBER6 - #define LSS_BODY_CLOBBER0 - #define LSS_BODY_CLOBBER1 LSS_BODY_CLOBBER0 - #define LSS_BODY_CLOBBER2 LSS_BODY_CLOBBER1 - #define LSS_BODY_CLOBBER3 LSS_BODY_CLOBBER2 - #define LSS_BODY_CLOBBER4 LSS_BODY_CLOBBER3 "r10", - #define LSS_BODY_CLOBBER5 LSS_BODY_CLOBBER4 "r8", - #define LSS_BODY_CLOBBER6 LSS_BODY_CLOBBER5 "r9", - - #undef LSS_BODY_ARG0 - #undef LSS_BODY_ARG1 - #undef LSS_BODY_ARG2 - #undef LSS_BODY_ARG3 - #undef LSS_BODY_ARG4 - #undef LSS_BODY_ARG5 - #undef LSS_BODY_ARG6 - #define LSS_BODY_ARG0() - #define LSS_BODY_ARG1(arg1) \ - LSS_BODY_ARG0(), "D" (arg1) - #define LSS_BODY_ARG2(arg1, arg2) \ - LSS_BODY_ARG1(arg1), "S" (arg2) - #define LSS_BODY_ARG3(arg1, arg2, arg3) \ - LSS_BODY_ARG2(arg1, arg2), "d" (arg3) - #define LSS_BODY_ARG4(arg1, arg2, arg3, arg4) \ - LSS_BODY_ARG3(arg1, arg2, arg3), "r" (arg4) - #define LSS_BODY_ARG5(arg1, arg2, arg3, arg4, arg5) \ - LSS_BODY_ARG4(arg1, arg2, arg3, arg4), "r" (arg5) - #define LSS_BODY_ARG6(arg1, arg2, arg3, arg4, arg5, arg6) \ - LSS_BODY_ARG5(arg1, arg2, arg3, arg4, arg5), "r" (arg6) - - #undef _syscall0 - #define _syscall0(type,name) \ - type LSS_NAME(name)() { \ - LSS_BODY(0, type, name); \ - } - #undef _syscall1 - #define _syscall1(type,name,type1,arg1) \ - type LSS_NAME(name)(type1 arg1) { \ - LSS_BODY(1, type, name, LSS_SYSCALL_ARG(arg1)); \ - } - #undef _syscall2 - #define _syscall2(type,name,type1,arg1,type2,arg2) \ - type LSS_NAME(name)(type1 arg1, type2 arg2) { \ - LSS_BODY(2, type, name, LSS_SYSCALL_ARG(arg1), LSS_SYSCALL_ARG(arg2));\ - } - #undef _syscall3 - #define _syscall3(type,name,type1,arg1,type2,arg2,type3,arg3) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3) { \ - LSS_BODY(3, type, name, LSS_SYSCALL_ARG(arg1), LSS_SYSCALL_ARG(arg2), \ - LSS_SYSCALL_ARG(arg3)); \ - } - #undef _syscall4 - #define _syscall4(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4) { \ - LSS_BODY(4, type, name, LSS_SYSCALL_ARG(arg1), LSS_SYSCALL_ARG(arg2), \ - LSS_SYSCALL_ARG(arg3), LSS_SYSCALL_ARG(arg4));\ - } - #undef _syscall5 - #define _syscall5(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5) { \ - LSS_BODY(5, type, name, LSS_SYSCALL_ARG(arg1), LSS_SYSCALL_ARG(arg2), \ - LSS_SYSCALL_ARG(arg3), LSS_SYSCALL_ARG(arg4), \ - LSS_SYSCALL_ARG(arg5)); \ - } - #undef _syscall6 - #define _syscall6(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5,type6,arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5, type6 arg6) { \ - LSS_BODY(6, type, name, LSS_SYSCALL_ARG(arg1), LSS_SYSCALL_ARG(arg2), \ - LSS_SYSCALL_ARG(arg3), LSS_SYSCALL_ARG(arg4), \ - LSS_SYSCALL_ARG(arg5), LSS_SYSCALL_ARG(arg6));\ - } - LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, - int flags, void *arg, int *parent_tidptr, - void *newtls, int *child_tidptr) { - long long __res; - { - __asm__ __volatile__(/* if (fn == NULL) - * return -EINVAL; - */ - "testq %4,%4\n" - "jz 1f\n" - - /* if (child_stack == NULL) - * return -EINVAL; - */ - "testq %5,%5\n" - "jz 1f\n" - - /* Set up alignment of the child stack: - * child_stack = (child_stack & ~0xF) - 16; - */ - "andq $-16,%5\n" - "subq $16,%5\n" - - /* Push "arg" and "fn" onto the stack that will be - * used by the child. - */ - "movq %7,8(%5)\n" - "movq %4,0(%5)\n" - - /* %rax = syscall(%rax = __NR_clone, - * %rdi = flags, - * %rsi = child_stack, - * %rdx = parent_tidptr, - * %r8 = new_tls, - * %r10 = child_tidptr) - */ - "movq %2,%%rax\n" - "movq %9,%%r8\n" - "movq %10,%%r10\n" - "syscall\n" - - /* if (%rax != 0) - * return; - */ - "testq %%rax,%%rax\n" - "jnz 1f\n" - - /* In the child. Terminate frame pointer chain. - */ - "xorq %%rbp,%%rbp\n" - - /* Call "fn(arg)". - */ - "popq %%rax\n" - "popq %%rdi\n" - "call *%%rax\n" - - /* Call _exit(%ebx). - */ - "movq %%rax,%%rdi\n" - "movq %3,%%rax\n" - "syscall\n" - - /* Return to parent. - */ - "1:\n" - : "=a" (__res) - : "0"(-EINVAL), "i"(__NR_clone), "i"(__NR_exit), - "r"(LSS_SYSCALL_ARG(fn)), - "S"(LSS_SYSCALL_ARG(child_stack)), - "D"(LSS_SYSCALL_ARG(flags)), - "r"(LSS_SYSCALL_ARG(arg)), - "d"(LSS_SYSCALL_ARG(parent_tidptr)), - "r"(LSS_SYSCALL_ARG(newtls)), - "r"(LSS_SYSCALL_ARG(child_tidptr)) - : "rsp", "memory", "r8", "r10", "r11", "rcx"); - } - LSS_RETURN(int, __res); - } - - LSS_INLINE void (*LSS_NAME(restore_rt)(void))(void) { - /* On x86-64, the kernel does not know how to return from - * a signal handler. Instead, it relies on user space to provide a - * restorer function that calls the rt_sigreturn() system call. - * Unfortunately, we cannot just reference the glibc version of this - * function, as glibc goes out of its way to make it inaccessible. - */ - long long res; - __asm__ __volatile__("call 2f\n" - "0:.align 16\n" - "1:movq %1,%%rax\n" - "syscall\n" - "2:popq %0\n" - "addq $(1b-0b),%0\n" - : "=a" (res) - : "i" (__NR_rt_sigreturn)); - return (void (*)(void))(uintptr_t)res; - } - #elif defined(__arm__) - /* Most definitions of _syscallX() neglect to mark "memory" as being - * clobbered. This causes problems with compilers, that do a better job - * at optimizing across __asm__ calls. - * So, we just have to redefine all fo the _syscallX() macros. - */ - #undef LSS_REG - #define LSS_REG(r,a) register long __r##r __asm__("r"#r) = (long)a - - /* r0..r3 are scratch registers and not preserved across function - * calls. We need to first evaluate the first 4 syscall arguments - * and store them on stack. They must be loaded into r0..r3 after - * all function calls to avoid r0..r3 being clobbered. - */ - #undef LSS_SAVE_ARG - #define LSS_SAVE_ARG(r,a) long __tmp##r = (long)a - #undef LSS_LOAD_ARG - #define LSS_LOAD_ARG(r) register long __r##r __asm__("r"#r) = __tmp##r - - #undef LSS_BODY - #define LSS_BODY(type, name, args...) \ - register long __res_r0 __asm__("r0"); \ - long __res; \ - __SYS_REG(name) \ - __asm__ __volatile__ (__syscall_safe(name) \ - : "=r"(__res_r0) \ - : __SYS_REG_LIST(args) \ - : "lr", "memory"); \ - __res = __res_r0; \ - LSS_RETURN(type, __res) - #undef _syscall0 - #define _syscall0(type, name) \ - type LSS_NAME(name)() { \ - LSS_BODY(type, name); \ - } - #undef _syscall1 - #define _syscall1(type, name, type1, arg1) \ - type LSS_NAME(name)(type1 arg1) { \ - /* There is no need for using a volatile temp. */ \ - LSS_REG(0, arg1); \ - LSS_BODY(type, name, "r"(__r0)); \ - } - #undef _syscall2 - #define _syscall2(type, name, type1, arg1, type2, arg2) \ - type LSS_NAME(name)(type1 arg1, type2 arg2) { \ - LSS_SAVE_ARG(0, arg1); \ - LSS_SAVE_ARG(1, arg2); \ - LSS_LOAD_ARG(0); \ - LSS_LOAD_ARG(1); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1)); \ - } - #undef _syscall3 - #define _syscall3(type, name, type1, arg1, type2, arg2, type3, arg3) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3) { \ - LSS_SAVE_ARG(0, arg1); \ - LSS_SAVE_ARG(1, arg2); \ - LSS_SAVE_ARG(2, arg3); \ - LSS_LOAD_ARG(0); \ - LSS_LOAD_ARG(1); \ - LSS_LOAD_ARG(2); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2)); \ - } - #undef _syscall4 - #define _syscall4(type, name, type1, arg1, type2, arg2, type3, arg3, \ - type4, arg4) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4) { \ - LSS_SAVE_ARG(0, arg1); \ - LSS_SAVE_ARG(1, arg2); \ - LSS_SAVE_ARG(2, arg3); \ - LSS_SAVE_ARG(3, arg4); \ - LSS_LOAD_ARG(0); \ - LSS_LOAD_ARG(1); \ - LSS_LOAD_ARG(2); \ - LSS_LOAD_ARG(3); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3)); \ - } - #undef _syscall5 - #define _syscall5(type, name, type1, arg1, type2, arg2, type3, arg3, \ - type4, arg4, type5, arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5) { \ - LSS_SAVE_ARG(0, arg1); \ - LSS_SAVE_ARG(1, arg2); \ - LSS_SAVE_ARG(2, arg3); \ - LSS_SAVE_ARG(3, arg4); \ - LSS_REG(4, arg5); \ - LSS_LOAD_ARG(0); \ - LSS_LOAD_ARG(1); \ - LSS_LOAD_ARG(2); \ - LSS_LOAD_ARG(3); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3), \ - "r"(__r4)); \ - } - #undef _syscall6 - #define _syscall6(type, name, type1, arg1, type2, arg2, type3, arg3, \ - type4, arg4, type5, arg5, type6, arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5, type6 arg6) { \ - LSS_SAVE_ARG(0, arg1); \ - LSS_SAVE_ARG(1, arg2); \ - LSS_SAVE_ARG(2, arg3); \ - LSS_SAVE_ARG(3, arg4); \ - LSS_REG(4, arg5); \ - LSS_REG(5, arg6); \ - LSS_LOAD_ARG(0); \ - LSS_LOAD_ARG(1); \ - LSS_LOAD_ARG(2); \ - LSS_LOAD_ARG(3); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3), \ - "r"(__r4), "r"(__r5)); \ - } - LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, - int flags, void *arg, int *parent_tidptr, - void *newtls, int *child_tidptr) { - register long __res __asm__("r5"); - { - if (fn == NULL || child_stack == NULL) { - __res = -EINVAL; - goto clone_exit; - } - - /* stash first 4 arguments on stack first because we can only load - * them after all function calls. - */ - int tmp_flags = flags; - int * tmp_stack = (int*) child_stack; - void * tmp_ptid = parent_tidptr; - void * tmp_tls = newtls; - - register int *__ctid __asm__("r4") = child_tidptr; - - /* Push "arg" and "fn" onto the stack that will be - * used by the child. - */ - *(--tmp_stack) = (int) arg; - *(--tmp_stack) = (int) fn; - - /* We must load r0..r3 last after all possible function calls. */ - register int __flags __asm__("r0") = tmp_flags; - register void *__stack __asm__("r1") = tmp_stack; - register void *__ptid __asm__("r2") = tmp_ptid; - register void *__tls __asm__("r3") = tmp_tls; - - /* %r0 = syscall(%r0 = flags, - * %r1 = child_stack, - * %r2 = parent_tidptr, - * %r3 = newtls, - * %r4 = child_tidptr) - */ - __SYS_REG(clone) - __asm__ __volatile__(/* %r0 = syscall(%r0 = flags, - * %r1 = child_stack, - * %r2 = parent_tidptr, - * %r3 = newtls, - * %r4 = child_tidptr) - */ - "push {r7}\n" - "mov r7,%1\n" - __syscall(clone)"\n" - - /* if (%r0 != 0) - * return %r0; - */ - "movs %0,r0\n" - "bne 1f\n" - - /* In the child, now. Call "fn(arg)". - */ - "ldr r0,[sp, #4]\n" - "mov lr,pc\n" - "ldr pc,[sp]\n" - - /* Call _exit(%r0), which never returns. We only - * need to set r7 for EABI syscall ABI but we do - * this always to simplify code sharing between - * old and new syscall ABIs. - */ - "mov r7,%2\n" - __syscall(exit)"\n" - - /* Pop r7 from the stack only in the parent. - */ - "1: pop {r7}\n" - : "=r" (__res) - : "r"(__sysreg), - "i"(__NR_exit), "r"(__stack), "r"(__flags), - "r"(__ptid), "r"(__tls), "r"(__ctid) - : "cc", "lr", "memory"); - } - clone_exit: - LSS_RETURN(int, __res); - } - #elif defined(__mips__) - #undef LSS_REG - #define LSS_REG(r,a) register unsigned long __r##r __asm__("$"#r) = \ - (unsigned long)(a) - - #if _MIPS_SIM == _MIPS_SIM_ABI32 - // See http://sources.redhat.com/ml/libc-alpha/2004-10/msg00050.html - // or http://www.linux-mips.org/archives/linux-mips/2004-10/msg00142.html - #define MIPS_SYSCALL_CLOBBERS "$1", "$3", "$8", "$9", "$10", "$11", "$12",\ - "$13", "$14", "$15", "$24", "$25", "memory" - #else - #define MIPS_SYSCALL_CLOBBERS "$1", "$3", "$10", "$11", "$12", "$13", \ - "$14", "$15", "$24", "$25", "memory" - #endif - - #undef LSS_BODY - #define LSS_BODY(type,name,r7,...) \ - register unsigned long __v0 __asm__("$2") = __NR_##name; \ - __asm__ __volatile__ ("syscall\n" \ - : "=&r"(__v0), r7 (__r7) \ - : "0"(__v0), ##__VA_ARGS__ \ - : MIPS_SYSCALL_CLOBBERS); \ - LSS_RETURN(type, __v0, __r7) - #undef _syscall0 - #define _syscall0(type, name) \ - type LSS_NAME(name)() { \ - register unsigned long __r7 __asm__("$7"); \ - LSS_BODY(type, name, "=r"); \ - } - #undef _syscall1 - #define _syscall1(type, name, type1, arg1) \ - type LSS_NAME(name)(type1 arg1) { \ - register unsigned long __r7 __asm__("$7"); \ - LSS_REG(4, arg1); LSS_BODY(type, name, "=r", "r"(__r4)); \ - } - #undef _syscall2 - #define _syscall2(type, name, type1, arg1, type2, arg2) \ - type LSS_NAME(name)(type1 arg1, type2 arg2) { \ - register unsigned long __r7 __asm__("$7"); \ - LSS_REG(4, arg1); LSS_REG(5, arg2); \ - LSS_BODY(type, name, "=r", "r"(__r4), "r"(__r5)); \ - } - #undef _syscall3 - #define _syscall3(type, name, type1, arg1, type2, arg2, type3, arg3) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3) { \ - register unsigned long __r7 __asm__("$7"); \ - LSS_REG(4, arg1); LSS_REG(5, arg2); LSS_REG(6, arg3); \ - LSS_BODY(type, name, "=r", "r"(__r4), "r"(__r5), "r"(__r6)); \ - } - #undef _syscall4 - #define _syscall4(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4) { \ - LSS_REG(4, arg1); LSS_REG(5, arg2); LSS_REG(6, arg3); \ - LSS_REG(7, arg4); \ - LSS_BODY(type, name, "+r", "r"(__r4), "r"(__r5), "r"(__r6)); \ - } - #undef _syscall5 - #if _MIPS_SIM == _MIPS_SIM_ABI32 - /* The old 32bit MIPS system call API passes the fifth and sixth argument - * on the stack, whereas the new APIs use registers "r8" and "r9". - */ - #define _syscall5(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5) { \ - LSS_REG(4, arg1); LSS_REG(5, arg2); LSS_REG(6, arg3); \ - LSS_REG(7, arg4); \ - register unsigned long __v0 __asm__("$2"); \ - __asm__ __volatile__ (".set noreorder\n" \ - "lw $2, %6\n" \ - "subu $29, 32\n" \ - "sw $2, 16($29)\n" \ - "li $2, %2\n" \ - "syscall\n" \ - "addiu $29, 32\n" \ - ".set reorder\n" \ - : "=&r"(__v0), "+r" (__r7) \ - : "i" (__NR_##name), "r"(__r4), "r"(__r5), \ - "r"(__r6), "m" ((unsigned long)arg5) \ - : MIPS_SYSCALL_CLOBBERS); \ - LSS_RETURN(type, __v0, __r7); \ - } - #else - #define _syscall5(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5) { \ - LSS_REG(4, arg1); LSS_REG(5, arg2); LSS_REG(6, arg3); \ - LSS_REG(7, arg4); LSS_REG(8, arg5); \ - LSS_BODY(type, name, "+r", "r"(__r4), "r"(__r5), "r"(__r6), \ - "r"(__r8)); \ - } - #endif - #undef _syscall6 - #if _MIPS_SIM == _MIPS_SIM_ABI32 - /* The old 32bit MIPS system call API passes the fifth and sixth argument - * on the stack, whereas the new APIs use registers "r8" and "r9". - */ - #define _syscall6(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5,type6,arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5, type6 arg6) { \ - LSS_REG(4, arg1); LSS_REG(5, arg2); LSS_REG(6, arg3); \ - LSS_REG(7, arg4); \ - register unsigned long __v0 __asm__("$2"); \ - __asm__ __volatile__ (".set noreorder\n" \ - "lw $2, %6\n" \ - "lw $8, %7\n" \ - "subu $29, 32\n" \ - "sw $2, 16($29)\n" \ - "sw $8, 20($29)\n" \ - "li $2, %2\n" \ - "syscall\n" \ - "addiu $29, 32\n" \ - ".set reorder\n" \ - : "=&r"(__v0), "+r" (__r7) \ - : "i" (__NR_##name), "r"(__r4), "r"(__r5), \ - "r"(__r6), "m" ((unsigned long)arg5), \ - "m" ((unsigned long)arg6) \ - : MIPS_SYSCALL_CLOBBERS); \ - LSS_RETURN(type, __v0, __r7); \ - } - #else - #define _syscall6(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5,type6,arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5,type6 arg6) { \ - LSS_REG(4, arg1); LSS_REG(5, arg2); LSS_REG(6, arg3); \ - LSS_REG(7, arg4); LSS_REG(8, arg5); LSS_REG(9, arg6); \ - LSS_BODY(type, name, "+r", "r"(__r4), "r"(__r5), "r"(__r6), \ - "r"(__r8), "r"(__r9)); \ - } - #endif - LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, - int flags, void *arg, int *parent_tidptr, - void *newtls, int *child_tidptr) { - register unsigned long __v0 __asm__("$2"); - register unsigned long __r7 __asm__("$7") = (unsigned long)newtls; - { - register int __flags __asm__("$4") = flags; - register void *__stack __asm__("$5") = child_stack; - register void *__ptid __asm__("$6") = parent_tidptr; - register int *__ctid __asm__("$8") = child_tidptr; - __asm__ __volatile__( - #if _MIPS_SIM == _MIPS_SIM_ABI32 && _MIPS_SZPTR == 32 - "subu $29,24\n" - #elif _MIPS_SIM == _MIPS_SIM_NABI32 - "sub $29,16\n" - #else - "dsubu $29,16\n" - #endif - - /* if (fn == NULL || child_stack == NULL) - * return -EINVAL; - */ - "li %0,%2\n" - "beqz %5,1f\n" - "beqz %6,1f\n" - - /* Push "arg" and "fn" onto the stack that will be - * used by the child. - */ - #if _MIPS_SIM == _MIPS_SIM_ABI32 && _MIPS_SZPTR == 32 - "subu %6,32\n" - "sw %5,0(%6)\n" - "sw %8,4(%6)\n" - #elif _MIPS_SIM == _MIPS_SIM_NABI32 - "sub %6,32\n" - "sw %5,0(%6)\n" - "sw %8,8(%6)\n" - #else - "dsubu %6,32\n" - "sd %5,0(%6)\n" - "sd %8,8(%6)\n" - #endif - - /* $7 = syscall($4 = flags, - * $5 = child_stack, - * $6 = parent_tidptr, - * $7 = newtls, - * $8 = child_tidptr) - */ - "li $2,%3\n" - "syscall\n" - - /* if ($7 != 0) - * return $2; - */ - "bnez $7,1f\n" - "bnez $2,1f\n" - - /* In the child, now. Call "fn(arg)". - */ - #if _MIPS_SIM == _MIPS_SIM_ABI32 && _MIPS_SZPTR == 32 - "lw $25,0($29)\n" - "lw $4,4($29)\n" - #elif _MIPS_SIM == _MIPS_SIM_NABI32 - "lw $25,0($29)\n" - "lw $4,8($29)\n" - #else - "ld $25,0($29)\n" - "ld $4,8($29)\n" - #endif - "jalr $25\n" - - /* Call _exit($2) - */ - "move $4,$2\n" - "li $2,%4\n" - "syscall\n" - - "1:\n" - #if _MIPS_SIM == _MIPS_SIM_ABI32 && _MIPS_SZPTR == 32 - "addu $29, 24\n" - #elif _MIPS_SIM == _MIPS_SIM_NABI32 - "add $29, 16\n" - #else - "daddu $29,16\n" - #endif - : "=&r" (__v0), "=r" (__r7) - : "i"(-EINVAL), "i"(__NR_clone), "i"(__NR_exit), - "r"(fn), "r"(__stack), "r"(__flags), "r"(arg), - "r"(__ptid), "r"(__r7), "r"(__ctid) - : "$9", "$10", "$11", "$12", "$13", "$14", "$15", - "$24", "memory"); - } - LSS_RETURN(int, __v0, __r7); - } - #elif defined (__PPC__) - #undef LSS_LOADARGS_0 - #define LSS_LOADARGS_0(name, dummy...) \ - __sc_0 = __NR_##name - #undef LSS_LOADARGS_1 - #define LSS_LOADARGS_1(name, arg1) \ - LSS_LOADARGS_0(name); \ - __sc_3 = (unsigned long) (arg1) - #undef LSS_LOADARGS_2 - #define LSS_LOADARGS_2(name, arg1, arg2) \ - LSS_LOADARGS_1(name, arg1); \ - __sc_4 = (unsigned long) (arg2) - #undef LSS_LOADARGS_3 - #define LSS_LOADARGS_3(name, arg1, arg2, arg3) \ - LSS_LOADARGS_2(name, arg1, arg2); \ - __sc_5 = (unsigned long) (arg3) - #undef LSS_LOADARGS_4 - #define LSS_LOADARGS_4(name, arg1, arg2, arg3, arg4) \ - LSS_LOADARGS_3(name, arg1, arg2, arg3); \ - __sc_6 = (unsigned long) (arg4) - #undef LSS_LOADARGS_5 - #define LSS_LOADARGS_5(name, arg1, arg2, arg3, arg4, arg5) \ - LSS_LOADARGS_4(name, arg1, arg2, arg3, arg4); \ - __sc_7 = (unsigned long) (arg5) - #undef LSS_LOADARGS_6 - #define LSS_LOADARGS_6(name, arg1, arg2, arg3, arg4, arg5, arg6) \ - LSS_LOADARGS_5(name, arg1, arg2, arg3, arg4, arg5); \ - __sc_8 = (unsigned long) (arg6) - #undef LSS_ASMINPUT_0 - #define LSS_ASMINPUT_0 "0" (__sc_0) - #undef LSS_ASMINPUT_1 - #define LSS_ASMINPUT_1 LSS_ASMINPUT_0, "1" (__sc_3) - #undef LSS_ASMINPUT_2 - #define LSS_ASMINPUT_2 LSS_ASMINPUT_1, "2" (__sc_4) - #undef LSS_ASMINPUT_3 - #define LSS_ASMINPUT_3 LSS_ASMINPUT_2, "3" (__sc_5) - #undef LSS_ASMINPUT_4 - #define LSS_ASMINPUT_4 LSS_ASMINPUT_3, "4" (__sc_6) - #undef LSS_ASMINPUT_5 - #define LSS_ASMINPUT_5 LSS_ASMINPUT_4, "5" (__sc_7) - #undef LSS_ASMINPUT_6 - #define LSS_ASMINPUT_6 LSS_ASMINPUT_5, "6" (__sc_8) - #undef LSS_BODY - #define LSS_BODY(nr, type, name, args...) \ - long __sc_ret, __sc_err; \ - { \ - register unsigned long __sc_0 __asm__ ("r0"); \ - register unsigned long __sc_3 __asm__ ("r3"); \ - register unsigned long __sc_4 __asm__ ("r4"); \ - register unsigned long __sc_5 __asm__ ("r5"); \ - register unsigned long __sc_6 __asm__ ("r6"); \ - register unsigned long __sc_7 __asm__ ("r7"); \ - register unsigned long __sc_8 __asm__ ("r8"); \ - \ - LSS_LOADARGS_##nr(name, args); \ - __asm__ __volatile__ \ - ("sc\n\t" \ - "mfcr %0" \ - : "=&r" (__sc_0), \ - "=&r" (__sc_3), "=&r" (__sc_4), \ - "=&r" (__sc_5), "=&r" (__sc_6), \ - "=&r" (__sc_7), "=&r" (__sc_8) \ - : LSS_ASMINPUT_##nr \ - : "cr0", "ctr", "memory", \ - "r9", "r10", "r11", "r12"); \ - __sc_ret = __sc_3; \ - __sc_err = __sc_0; \ - } \ - LSS_RETURN(type, __sc_ret, __sc_err) - #undef _syscall0 - #define _syscall0(type, name) \ - type LSS_NAME(name)(void) { \ - LSS_BODY(0, type, name); \ - } - #undef _syscall1 - #define _syscall1(type, name, type1, arg1) \ - type LSS_NAME(name)(type1 arg1) { \ - LSS_BODY(1, type, name, arg1); \ - } - #undef _syscall2 - #define _syscall2(type, name, type1, arg1, type2, arg2) \ - type LSS_NAME(name)(type1 arg1, type2 arg2) { \ - LSS_BODY(2, type, name, arg1, arg2); \ - } - #undef _syscall3 - #define _syscall3(type, name, type1, arg1, type2, arg2, type3, arg3) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3) { \ - LSS_BODY(3, type, name, arg1, arg2, arg3); \ - } - #undef _syscall4 - #define _syscall4(type, name, type1, arg1, type2, arg2, type3, arg3, \ - type4, arg4) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4) { \ - LSS_BODY(4, type, name, arg1, arg2, arg3, arg4); \ - } - #undef _syscall5 - #define _syscall5(type, name, type1, arg1, type2, arg2, type3, arg3, \ - type4, arg4, type5, arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5) { \ - LSS_BODY(5, type, name, arg1, arg2, arg3, arg4, arg5); \ - } - #undef _syscall6 - #define _syscall6(type, name, type1, arg1, type2, arg2, type3, arg3, \ - type4, arg4, type5, arg5, type6, arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5, type6 arg6) { \ - LSS_BODY(6, type, name, arg1, arg2, arg3, arg4, arg5, arg6); \ - } - /* clone function adapted from glibc 2.18 clone.S */ - LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, - int flags, void *arg, int *parent_tidptr, - void *newtls, int *child_tidptr) { - long __ret, __err; - { -#if defined(__PPC64__) - -/* Stack frame offsets. */ -#if _CALL_ELF != 2 -#define FRAME_MIN_SIZE 112 -#define FRAME_TOC_SAVE 40 -#else -#define FRAME_MIN_SIZE 32 -#define FRAME_TOC_SAVE 24 -#endif - - - register int (*__fn)(void *) __asm__ ("r3") = fn; - register void *__cstack __asm__ ("r4") = child_stack; - register int __flags __asm__ ("r5") = flags; - register void * __arg __asm__ ("r6") = arg; - register int * __ptidptr __asm__ ("r7") = parent_tidptr; - register void * __newtls __asm__ ("r8") = newtls; - register int * __ctidptr __asm__ ("r9") = child_tidptr; - __asm__ __volatile__( - /* check for fn == NULL - * and child_stack == NULL - */ - "cmpdi cr0, %6, 0\n\t" - "cmpdi cr1, %7, 0\n\t" - "cror cr0*4+eq, cr1*4+eq, cr0*4+eq\n\t" - "beq- cr0, 1f\n\t" - - /* set up stack frame for child */ - "clrrdi %7, %7, 4\n\t" - "li 0, 0\n\t" - "stdu 0, -%13(%7)\n\t" - - /* fn, arg, child_stack are saved acrVoss the syscall */ - "mr 28, %6\n\t" - "mr 29, %7\n\t" - "mr 27, %9\n\t" - - /* syscall - r3 == flags - r4 == child_stack - r5 == parent_tidptr - r6 == newtls - r7 == child_tidptr */ - "mr 3, %8\n\t" - "mr 5, %10\n\t" - "mr 6, %11\n\t" - "mr 7, %12\n\t" - "li 0, %4\n\t" - "sc\n\t" - - /* Test if syscall was successful */ - "cmpdi cr1, 3, 0\n\t" - "crandc cr1*4+eq, cr1*4+eq, cr0*4+so\n\t" - "bne- cr1, 1f\n\t" - - /* Do the function call */ - "std 2, %14(1)\n\t" -#if _CALL_ELF != 2 - "ld 0, 0(28)\n\t" - "ld 2, 8(28)\n\t" - "mtctr 0\n\t" -#else - "mr 12, 28\n\t" - "mtctr 12\n\t" -#endif - "mr 3, 27\n\t" - "bctrl\n\t" - "ld 2, %14(1)\n\t" - - /* Call _exit(r3) */ - "li 0, %5\n\t" - "sc\n\t" - - /* Return to parent */ - "1:\n\t" - "mr %0, 3\n\t" - : "=r" (__ret), "=r" (__err) - : "0" (-1), "i" (EINVAL), - "i" (__NR_clone), "i" (__NR_exit), - "r" (__fn), "r" (__cstack), "r" (__flags), - "r" (__arg), "r" (__ptidptr), "r" (__newtls), - "r" (__ctidptr), "i" (FRAME_MIN_SIZE), "i" (FRAME_TOC_SAVE) - : "cr0", "cr1", "memory", "ctr", - "r0", "r29", "r27", "r28"); -#else - register int (*__fn)(void *) __asm__ ("r8") = fn; - register void *__cstack __asm__ ("r4") = child_stack; - register int __flags __asm__ ("r3") = flags; - register void * __arg __asm__ ("r9") = arg; - register int * __ptidptr __asm__ ("r5") = parent_tidptr; - register void * __newtls __asm__ ("r6") = newtls; - register int * __ctidptr __asm__ ("r7") = child_tidptr; - __asm__ __volatile__( - /* check for fn == NULL - * and child_stack == NULL - */ - "cmpwi cr0, %6, 0\n\t" - "cmpwi cr1, %7, 0\n\t" - "cror cr0*4+eq, cr1*4+eq, cr0*4+eq\n\t" - "beq- cr0, 1f\n\t" - - /* set up stack frame for child */ - "clrrwi %7, %7, 4\n\t" - "li 0, 0\n\t" - "stwu 0, -16(%7)\n\t" - - /* fn, arg, child_stack are saved across the syscall: r28-30 */ - "mr 28, %6\n\t" - "mr 29, %7\n\t" - "mr 27, %9\n\t" - - /* syscall */ - "li 0, %4\n\t" - /* flags already in r3 - * child_stack already in r4 - * ptidptr already in r5 - * newtls already in r6 - * ctidptr already in r7 - */ - "sc\n\t" - - /* Test if syscall was successful */ - "cmpwi cr1, 3, 0\n\t" - "crandc cr1*4+eq, cr1*4+eq, cr0*4+so\n\t" - "bne- cr1, 1f\n\t" - - /* Do the function call */ - "mtctr 28\n\t" - "mr 3, 27\n\t" - "bctrl\n\t" - - /* Call _exit(r3) */ - "li 0, %5\n\t" - "sc\n\t" - - /* Return to parent */ - "1:\n" - "mfcr %1\n\t" - "mr %0, 3\n\t" - : "=r" (__ret), "=r" (__err) - : "0" (-1), "1" (EINVAL), - "i" (__NR_clone), "i" (__NR_exit), - "r" (__fn), "r" (__cstack), "r" (__flags), - "r" (__arg), "r" (__ptidptr), "r" (__newtls), - "r" (__ctidptr) - : "cr0", "cr1", "memory", "ctr", - "r0", "r29", "r27", "r28"); - -#endif - } - LSS_RETURN(int, __ret, __err); - } - #elif defined(__aarch64__) - #undef LSS_REG - #define LSS_REG(r,a) register long __x##r __asm__("x"#r) = (long)a - #undef LSS_BODY - #define LSS_BODY(type,name,args...) \ - register long __res_x0 __asm__("x0"); \ - long __res; \ - __asm__ __volatile__ ("mov x8, %1\n" \ - "svc 0x0\n" \ - : "=r"(__res_x0) \ - : "i"(__NR_##name) , ## args \ - : "memory"); \ - __res = __res_x0; \ - LSS_RETURN(type, __res) - #undef _syscall0 - #define _syscall0(type, name) \ - type LSS_NAME(name)(void) { \ - LSS_BODY(type, name); \ - } - #undef _syscall1 - #define _syscall1(type, name, type1, arg1) \ - type LSS_NAME(name)(type1 arg1) { \ - LSS_REG(0, arg1); LSS_BODY(type, name, "r"(__x0)); \ - } - #undef _syscall2 - #define _syscall2(type, name, type1, arg1, type2, arg2) \ - type LSS_NAME(name)(type1 arg1, type2 arg2) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); \ - LSS_BODY(type, name, "r"(__x0), "r"(__x1)); \ - } - #undef _syscall3 - #define _syscall3(type, name, type1, arg1, type2, arg2, type3, arg3) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_BODY(type, name, "r"(__x0), "r"(__x1), "r"(__x2)); \ - } - #undef _syscall4 - #define _syscall4(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); \ - LSS_BODY(type, name, "r"(__x0), "r"(__x1), "r"(__x2), "r"(__x3)); \ - } - #undef _syscall5 - #define _syscall5(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); LSS_REG(4, arg5); \ - LSS_BODY(type, name, "r"(__x0), "r"(__x1), "r"(__x2), "r"(__x3), \ - "r"(__x4)); \ - } - #undef _syscall6 - #define _syscall6(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5,type6,arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5, type6 arg6) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); LSS_REG(4, arg5); LSS_REG(5, arg6); \ - LSS_BODY(type, name, "r"(__x0), "r"(__x1), "x"(__x2), "r"(__x3), \ - "r"(__x4), "r"(__x5)); \ - } - /* clone function adapted from glibc 2.18 clone.S */ - LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, - int flags, void *arg, int *parent_tidptr, - void *newtls, int *child_tidptr) { - long __res; - { - register int (*__fn)(void *) __asm__("x0") = fn; - register void *__stack __asm__("x1") = child_stack; - register int __flags __asm__("x2") = flags; - register void *__arg __asm__("x3") = arg; - register int *__ptid __asm__("x4") = parent_tidptr; - register void *__tls __asm__("x5") = newtls; - register int *__ctid __asm__("x6") = child_tidptr; - __asm__ __volatile__(/* if (fn == NULL || child_stack == NULL) - * return -EINVAL; - */ - "cbz x0,1f\n" - "cbz x1,1f\n" - - /* Push "arg" and "fn" onto the stack that will be - * used by the child. - */ - "stp x0,x3, [x1, #-16]!\n" - - "mov x0,x2\n" /* flags */ - "mov x2,x4\n" /* ptid */ - "mov x3,x5\n" /* tls */ - "mov x4,x6\n" /* ctid */ - "mov x8,%9\n" /* clone */ - - "svc 0x0\n" - - /* if (%r0 != 0) - * return %r0; - */ - "cmp x0, #0\n" - "bne 2f\n" - - /* In the child, now. Call "fn(arg)". - */ - "ldp x1, x0, [sp], #16\n" - "blr x1\n" - - /* Call _exit(%r0). - */ - "mov x8, %10\n" - "svc 0x0\n" - "1:\n" - "mov x8, %1\n" - "2:\n" - : "=r" (__res) - : "i"(-EINVAL), - "r"(__fn), "r"(__stack), "r"(__flags), "r"(__arg), - "r"(__ptid), "r"(__tls), "r"(__ctid), - "i"(__NR_clone), "i"(__NR_exit) - : "x30", "memory"); - } - LSS_RETURN(int, __res); - } - #elif defined(__s390__) - #undef LSS_REG - #define LSS_REG(r, a) register unsigned long __r##r __asm__("r"#r) = (unsigned long) a - #undef LSS_BODY - #define LSS_BODY(type, name, args...) \ - register unsigned long __nr __asm__("r1") \ - = (unsigned long)(__NR_##name); \ - register long __res_r2 __asm__("r2"); \ - long __res; \ - __asm__ __volatile__ \ - ("svc 0\n\t" \ - : "=d"(__res_r2) \ - : "d"(__nr), ## args \ - : "memory"); \ - __res = __res_r2; \ - LSS_RETURN(type, __res) - #undef _syscall0 - #define _syscall0(type, name) \ - type LSS_NAME(name)(void) { \ - LSS_BODY(type, name); \ - } - #undef _syscall1 - #define _syscall1(type, name, type1, arg1) \ - type LSS_NAME(name)(type1 arg1) { \ - LSS_REG(2, arg1); \ - LSS_BODY(type, name, "0"(__r2)); \ - } - #undef _syscall2 - #define _syscall2(type, name, type1, arg1, type2, arg2) \ - type LSS_NAME(name)(type1 arg1, type2 arg2) { \ - LSS_REG(2, arg1); LSS_REG(3, arg2); \ - LSS_BODY(type, name, "0"(__r2), "d"(__r3)); \ - } - #undef _syscall3 - #define _syscall3(type, name, type1, arg1, type2, arg2, type3, arg3) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3) { \ - LSS_REG(2, arg1); LSS_REG(3, arg2); LSS_REG(4, arg3); \ - LSS_BODY(type, name, "0"(__r2), "d"(__r3), "d"(__r4)); \ - } - #undef _syscall4 - #define _syscall4(type, name, type1, arg1, type2, arg2, type3, arg3, \ - type4, arg4) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, \ - type4 arg4) { \ - LSS_REG(2, arg1); LSS_REG(3, arg2); LSS_REG(4, arg3); \ - LSS_REG(5, arg4); \ - LSS_BODY(type, name, "0"(__r2), "d"(__r3), "d"(__r4), \ - "d"(__r5)); \ - } - #undef _syscall5 - #define _syscall5(type, name, type1, arg1, type2, arg2, type3, arg3, \ - type4, arg4, type5, arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, \ - type4 arg4, type5 arg5) { \ - LSS_REG(2, arg1); LSS_REG(3, arg2); LSS_REG(4, arg3); \ - LSS_REG(5, arg4); LSS_REG(6, arg5); \ - LSS_BODY(type, name, "0"(__r2), "d"(__r3), "d"(__r4), \ - "d"(__r5), "d"(__r6)); \ - } - #undef _syscall6 - #define _syscall6(type, name, type1, arg1, type2, arg2, type3, arg3, \ - type4, arg4, type5, arg5, type6, arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, \ - type4 arg4, type5 arg5, type6 arg6) { \ - LSS_REG(2, arg1); LSS_REG(3, arg2); LSS_REG(4, arg3); \ - LSS_REG(5, arg4); LSS_REG(6, arg5); LSS_REG(7, arg6); \ - LSS_BODY(type, name, "0"(__r2), "d"(__r3), "d"(__r4), \ - "d"(__r5), "d"(__r6), "d"(__r7)); \ - } - LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, - int flags, void *arg, int *parent_tidptr, - void *newtls, int *child_tidptr) { - long __ret; - { - register int (*__fn)(void *) __asm__ ("r1") = fn; - register void *__cstack __asm__ ("r2") = child_stack; - register int __flags __asm__ ("r3") = flags; - register void *__arg __asm__ ("r0") = arg; - register int *__ptidptr __asm__ ("r4") = parent_tidptr; - register void *__newtls __asm__ ("r6") = newtls; - register int *__ctidptr __asm__ ("r5") = child_tidptr; - __asm__ __volatile__ ( - #ifndef __s390x__ - /* arg already in r0 */ - "ltr %4, %4\n\t" /* check fn, which is already in r1 */ - "jz 1f\n\t" /* NULL function pointer, return -EINVAL */ - "ltr %5, %5\n\t" /* check child_stack, which is already in r2 */ - "jz 1f\n\t" /* NULL stack pointer, return -EINVAL */ - /* flags already in r3 */ - /* parent_tidptr already in r4 */ - /* child_tidptr already in r5 */ - /* newtls already in r6 */ - "svc %2\n\t" /* invoke clone syscall */ - "ltr %0,%%r2\n\t" /* load return code into __ret and test */ - "jnz 1f\n\t" /* return to parent if non-zero */ - /* start child thread */ - "lr %%r2, %7\n\t" /* set first parameter to void *arg */ - "ahi %%r15, -96\n\t" /* make room on the stack for the save area */ - "xc 0(4,%%r15), 0(%%r15)\n\t" - "basr %%r14, %4\n\t" /* jump to fn */ - "svc %3\n" /* invoke exit syscall */ - "1:\n" - #else - /* arg already in r0 */ - "ltgr %4, %4\n\t" /* check fn, which is already in r1 */ - "jz 1f\n\t" /* NULL function pointer, return -EINVAL */ - "ltgr %5, %5\n\t" /* check child_stack, which is already in r2 */ - "jz 1f\n\t" /* NULL stack pointer, return -EINVAL */ - /* flags already in r3 */ - /* parent_tidptr already in r4 */ - /* child_tidptr already in r5 */ - /* newtls already in r6 */ - "svc %2\n\t" /* invoke clone syscall */ - "ltgr %0, %%r2\n\t" /* load return code into __ret and test */ - "jnz 1f\n\t" /* return to parent if non-zero */ - /* start child thread */ - "lgr %%r2, %7\n\t" /* set first parameter to void *arg */ - "aghi %%r15, -160\n\t" /* make room on the stack for the save area */ - "xc 0(8,%%r15), 0(%%r15)\n\t" - "basr %%r14, %4\n\t" /* jump to fn */ - "svc %3\n" /* invoke exit syscall */ - "1:\n" - #endif - : "=r" (__ret) - : "0" (-EINVAL), "i" (__NR_clone), "i" (__NR_exit), - "d" (__fn), "d" (__cstack), "d" (__flags), "d" (__arg), - "d" (__ptidptr), "d" (__newtls), "d" (__ctidptr) - : "cc", "r14", "memory" - ); - } - LSS_RETURN(int, __ret); - } - #endif - #define __NR__exit __NR_exit - #define __NR__gettid __NR_gettid - #define __NR__mremap __NR_mremap - LSS_INLINE _syscall1(int, close, int, f) - LSS_INLINE _syscall1(int, _exit, int, e) - LSS_INLINE _syscall3(int, fcntl, int, f, - int, c, long, a) - LSS_INLINE _syscall2(int, fstat, int, f, - struct kernel_stat*, b) - LSS_INLINE _syscall6(int, futex, int*, a, - int, o, int, v, - struct kernel_timespec*, t, - int*, a2, - int, v3) -#ifdef __NR_getdents64 - LSS_INLINE _syscall3(int, getdents64, int, f, - struct kernel_dirent64*, d, int, c) -#define KERNEL_DIRENT kernel_dirent64 -#define GETDENTS sys_getdents64 -#else - LSS_INLINE _syscall3(int, getdents, int, f, - struct kernel_dirent*, d, int, c) -#define KERNEL_DIRENT kernel_dirent -#define GETDENTS sys_getdents -#endif - LSS_INLINE _syscall0(pid_t, getpid) - LSS_INLINE _syscall0(pid_t, getppid) - LSS_INLINE _syscall0(pid_t, _gettid) - LSS_INLINE _syscall2(int, kill, pid_t, p, - int, s) - #if defined(__x86_64__) - /* Need to make sure off_t isn't truncated to 32-bits under x32. */ - LSS_INLINE off_t LSS_NAME(lseek)(int f, off_t o, int w) { - _LSS_BODY(3, off_t, lseek, off_t, LSS_SYSCALL_ARG(f), (uint64_t)(o), - LSS_SYSCALL_ARG(w)); - } - #else - LSS_INLINE _syscall3(off_t, lseek, int, f, - off_t, o, int, w) - #endif - LSS_INLINE _syscall2(int, munmap, void*, s, - size_t, l) - LSS_INLINE _syscall5(void*, _mremap, void*, o, - size_t, os, size_t, ns, - unsigned long, f, void *, a) - LSS_INLINE _syscall2(int, prctl, int, o, - long, a) - LSS_INLINE _syscall4(long, ptrace, int, r, - pid_t, p, void *, a, void *, d) - LSS_INLINE _syscall3(ssize_t, read, int, f, - void *, b, size_t, c) - LSS_INLINE _syscall4(int, rt_sigaction, int, s, - const struct kernel_sigaction*, a, - struct kernel_sigaction*, o, size_t, c) - LSS_INLINE _syscall4(int, rt_sigprocmask, int, h, - const struct kernel_sigset_t*, s, - struct kernel_sigset_t*, o, size_t, c); - LSS_INLINE _syscall0(int, sched_yield) - LSS_INLINE _syscall2(int, sigaltstack, const stack_t*, s, - const stack_t*, o) - #if defined(__NR_fstatat) - LSS_INLINE _syscall4(int, fstatat, int, d, const char *, p, - struct kernel_stat*, b, int, flags) - LSS_INLINE int LSS_NAME(stat)(const char* p, struct kernel_stat* b) { - return LSS_NAME(fstatat)(AT_FDCWD,p,b,0); - } - #else - LSS_INLINE _syscall2(int, stat, const char*, f, - struct kernel_stat*, b) - #endif - LSS_INLINE _syscall3(ssize_t, write, int, f, - const void *, b, size_t, c) - #if defined(__NR_getcpu) - LSS_INLINE _syscall3(long, getcpu, unsigned *, cpu, - unsigned *, node, void *, unused); - #endif - #if defined(__x86_64__) || defined(__aarch64__) || \ - (defined(__mips__) && _MIPS_SIM != _MIPS_SIM_ABI32) - LSS_INLINE _syscall3(int, socket, int, d, - int, t, int, p) - #endif - #if defined(__x86_64__) || defined(__s390x__) - LSS_INLINE int LSS_NAME(sigaction)(int signum, - const struct kernel_sigaction *act, - struct kernel_sigaction *oldact) { - #if defined(__x86_64__) - /* On x86_64, the kernel requires us to always set our own - * SA_RESTORER in order to be able to return from a signal handler. - * This function must have a "magic" signature that the "gdb" - * (and maybe the kernel?) can recognize. - */ - if (act != NULL && !(act->sa_flags & SA_RESTORER)) { - struct kernel_sigaction a = *act; - a.sa_flags |= SA_RESTORER; - a.sa_restorer = LSS_NAME(restore_rt)(); - return LSS_NAME(rt_sigaction)(signum, &a, oldact, - (KERNEL_NSIG+7)/8); - } else - #endif - return LSS_NAME(rt_sigaction)(signum, act, oldact, - (KERNEL_NSIG+7)/8); - } - - LSS_INLINE int LSS_NAME(sigprocmask)(int how, - const struct kernel_sigset_t *set, - struct kernel_sigset_t *oldset) { - return LSS_NAME(rt_sigprocmask)(how, set, oldset, (KERNEL_NSIG+7)/8); - } - #endif - #if (defined(__aarch64__)) || \ - (defined(__mips__) && (_MIPS_ISA == _MIPS_ISA_MIPS64)) - LSS_INLINE int LSS_NAME(sigaction)(int signum, - const struct kernel_sigaction *act, - struct kernel_sigaction *oldact) { - return LSS_NAME(rt_sigaction)(signum, act, oldact, (KERNEL_NSIG+7)/8); - - } - LSS_INLINE int LSS_NAME(sigprocmask)(int how, - const struct kernel_sigset_t *set, - struct kernel_sigset_t *oldset) { - return LSS_NAME(rt_sigprocmask)(how, set, oldset, (KERNEL_NSIG+7)/8); - } - #endif - #ifdef __NR_wait4 - LSS_INLINE _syscall4(pid_t, wait4, pid_t, p, - int*, s, int, o, - struct kernel_rusage*, r) - LSS_INLINE pid_t LSS_NAME(waitpid)(pid_t pid, int *status, int options){ - return LSS_NAME(wait4)(pid, status, options, 0); - } - #else - LSS_INLINE _syscall3(pid_t, waitpid, pid_t, p, - int*, s, int, o) - #endif - #ifdef __NR_openat - LSS_INLINE _syscall4(int, openat, int, d, const char *, p, int, f, int, m) - LSS_INLINE int LSS_NAME(open)(const char* p, int f, int m) { - return LSS_NAME(openat)(AT_FDCWD,p,f,m ); - } - #else - LSS_INLINE _syscall3(int, open, const char*, p, - int, f, int, m) - #endif - LSS_INLINE int LSS_NAME(sigemptyset)(struct kernel_sigset_t *set) { - memset(&set->sig, 0, sizeof(set->sig)); - return 0; - } - - LSS_INLINE int LSS_NAME(sigfillset)(struct kernel_sigset_t *set) { - memset(&set->sig, -1, sizeof(set->sig)); - return 0; - } - - LSS_INLINE int LSS_NAME(sigaddset)(struct kernel_sigset_t *set, - int signum) { - if (signum < 1 || signum > (int)(8*sizeof(set->sig))) { - LSS_ERRNO = EINVAL; - return -1; - } else { - set->sig[(signum - 1)/(8*sizeof(set->sig[0]))] - |= 1UL << ((signum - 1) % (8*sizeof(set->sig[0]))); - return 0; - } - } - - LSS_INLINE int LSS_NAME(sigdelset)(struct kernel_sigset_t *set, - int signum) { - if (signum < 1 || signum > (int)(8*sizeof(set->sig))) { - LSS_ERRNO = EINVAL; - return -1; - } else { - set->sig[(signum - 1)/(8*sizeof(set->sig[0]))] - &= ~(1UL << ((signum - 1) % (8*sizeof(set->sig[0])))); - return 0; - } - } - - #if defined(__i386__) || \ - defined(__arm__) || \ - (defined(__mips__) && _MIPS_SIM == _MIPS_SIM_ABI32) || \ - defined(__PPC__) || \ - (defined(__s390__) && !defined(__s390x__)) - #define __NR__sigaction __NR_sigaction - #define __NR__sigprocmask __NR_sigprocmask - LSS_INLINE _syscall2(int, fstat64, int, f, - struct kernel_stat64 *, b) - LSS_INLINE _syscall5(int, _llseek, uint, fd, ulong, hi, ulong, lo, - loff_t *, res, uint, wh) -#if defined(__s390__) && !defined(__s390x__) - /* On s390, mmap2() arguments are passed in memory. */ - LSS_INLINE void* LSS_NAME(_mmap2)(void *s, size_t l, int p, int f, int d, - off_t o) { - unsigned long buf[6] = { (unsigned long) s, (unsigned long) l, - (unsigned long) p, (unsigned long) f, - (unsigned long) d, (unsigned long) o }; - LSS_REG(2, buf); - LSS_BODY(void*, mmap2, "0"(__r2)); - } -#elif !defined(__PPC64__) - #define __NR__mmap2 __NR_mmap2 - LSS_INLINE _syscall6(void*, _mmap2, void*, s, - size_t, l, int, p, - int, f, int, d, - off_t, o) -#endif - LSS_INLINE _syscall3(int, _sigaction, int, s, - const struct kernel_old_sigaction*, a, - struct kernel_old_sigaction*, o) - LSS_INLINE _syscall3(int, _sigprocmask, int, h, - const unsigned long*, s, - unsigned long*, o) - LSS_INLINE _syscall2(int, stat64, const char *, p, - struct kernel_stat64 *, b) - - LSS_INLINE int LSS_NAME(sigaction)(int signum, - const struct kernel_sigaction *act, - struct kernel_sigaction *oldact) { - int old_errno = LSS_ERRNO; - int rc; - struct kernel_sigaction a; - if (act != NULL) { - a = *act; - #ifdef __i386__ - /* On i386, the kernel requires us to always set our own - * SA_RESTORER when using realtime signals. Otherwise, it does not - * know how to return from a signal handler. This function must have - * a "magic" signature that the "gdb" (and maybe the kernel?) can - * recognize. - * Apparently, a SA_RESTORER is implicitly set by the kernel, when - * using non-realtime signals. - * - * TODO: Test whether ARM needs a restorer - */ - if (!(a.sa_flags & SA_RESTORER)) { - a.sa_flags |= SA_RESTORER; - a.sa_restorer = (a.sa_flags & SA_SIGINFO) - ? LSS_NAME(restore_rt)() : LSS_NAME(restore)(); - } - #endif - } - rc = LSS_NAME(rt_sigaction)(signum, act ? &a : act, oldact, - (KERNEL_NSIG+7)/8); - if (rc < 0 && LSS_ERRNO == ENOSYS) { - struct kernel_old_sigaction oa, ooa, *ptr_a = &oa, *ptr_oa = &ooa; - if (!act) { - ptr_a = NULL; - } else { - oa.sa_handler_ = act->sa_handler_; - memcpy(&oa.sa_mask, &act->sa_mask, sizeof(oa.sa_mask)); - #ifndef __mips__ - oa.sa_restorer = act->sa_restorer; - #endif - oa.sa_flags = act->sa_flags; - } - if (!oldact) { - ptr_oa = NULL; - } - LSS_ERRNO = old_errno; - rc = LSS_NAME(_sigaction)(signum, ptr_a, ptr_oa); - if (rc == 0 && oldact) { - if (act) { - memcpy(oldact, act, sizeof(*act)); - } else { - memset(oldact, 0, sizeof(*oldact)); - } - oldact->sa_handler_ = ptr_oa->sa_handler_; - oldact->sa_flags = ptr_oa->sa_flags; - memcpy(&oldact->sa_mask, &ptr_oa->sa_mask, sizeof(ptr_oa->sa_mask)); - #ifndef __mips__ - oldact->sa_restorer = ptr_oa->sa_restorer; - #endif - } - } - return rc; - } - - LSS_INLINE int LSS_NAME(sigprocmask)(int how, - const struct kernel_sigset_t *set, - struct kernel_sigset_t *oldset) { - int olderrno = LSS_ERRNO; - int rc = LSS_NAME(rt_sigprocmask)(how, set, oldset, (KERNEL_NSIG+7)/8); - if (rc < 0 && LSS_ERRNO == ENOSYS) { - LSS_ERRNO = olderrno; - if (oldset) { - LSS_NAME(sigemptyset)(oldset); - } - rc = LSS_NAME(_sigprocmask)(how, - set ? &set->sig[0] : NULL, - oldset ? &oldset->sig[0] : NULL); - } - return rc; - } - #endif - #if defined(__i386__) || \ - defined(__ARM_ARCH_3__) || defined(__ARM_EABI__) || \ - (defined(__mips__) && _MIPS_SIM == _MIPS_SIM_ABI32) || \ - (defined(__PPC__) && !defined(__PPC64__)) || \ - (defined(__s390__) && !defined(__s390x__)) - /* On these architectures, implement mmap() with mmap2(). */ - LSS_INLINE void* LSS_NAME(mmap)(void *s, size_t l, int p, int f, int d, - int64_t o) { - if (o % 4096) { - LSS_ERRNO = EINVAL; - return (void *) -1; - } - return LSS_NAME(_mmap2)(s, l, p, f, d, (o / 4096)); - } - #elif defined(__s390x__) - /* On s390x, mmap() arguments are passed in memory. */ - LSS_INLINE void* LSS_NAME(mmap)(void *s, size_t l, int p, int f, int d, - int64_t o) { - unsigned long buf[6] = { (unsigned long) s, (unsigned long) l, - (unsigned long) p, (unsigned long) f, - (unsigned long) d, (unsigned long) o }; - LSS_REG(2, buf); - LSS_BODY(void*, mmap, "0"(__r2)); - } - #elif defined(__x86_64__) - /* Need to make sure __off64_t isn't truncated to 32-bits under x32. */ - LSS_INLINE void* LSS_NAME(mmap)(void *s, size_t l, int p, int f, int d, - int64_t o) { - LSS_BODY(6, void*, mmap, LSS_SYSCALL_ARG(s), LSS_SYSCALL_ARG(l), - LSS_SYSCALL_ARG(p), LSS_SYSCALL_ARG(f), - LSS_SYSCALL_ARG(d), (uint64_t)(o)); - } - #else - /* Remaining 64-bit architectures. */ - LSS_INLINE _syscall6(void*, mmap, void*, addr, size_t, length, int, prot, - int, flags, int, fd, int64_t, offset) - #endif - #if defined(__i386__) || \ - defined(__PPC__) || \ - (defined(__arm__) && !defined(__ARM_EABI__)) || \ - (defined(__mips__) && _MIPS_SIM == _MIPS_SIM_ABI32) || \ - defined(__s390__) - - /* See sys_socketcall in net/socket.c in kernel source. - * It de-multiplexes on its first arg and unpacks the arglist - * array in its second arg. - */ - LSS_INLINE _syscall2(int, socketcall, int, c, unsigned long*, a) - - LSS_INLINE int LSS_NAME(socket)(int domain, int type, int protocol) { - unsigned long args[3] = { - (unsigned long) domain, - (unsigned long) type, - (unsigned long) protocol - }; - return LSS_NAME(socketcall)(1, args); - } - #elif defined(__ARM_EABI__) - LSS_INLINE _syscall3(int, socket, int, d, - int, t, int, p) - #endif - #if defined(__mips__) - /* sys_pipe() on MIPS has non-standard calling conventions, as it returns - * both file handles through CPU registers. - */ - LSS_INLINE int LSS_NAME(pipe)(int *p) { - register unsigned long __v0 __asm__("$2") = __NR_pipe; - register unsigned long __v1 __asm__("$3"); - register unsigned long __r7 __asm__("$7"); - __asm__ __volatile__ ("syscall\n" - : "=&r"(__v0), "=&r"(__v1), "+r" (__r7) - : "0"(__v0) - : "$8", "$9", "$10", "$11", "$12", - "$13", "$14", "$15", "$24", "memory"); - if (__r7) { - LSS_ERRNO = __v0; - return -1; - } else { - p[0] = __v0; - p[1] = __v1; - return 0; - } - } - #elif defined(__NR_pipe2) - LSS_INLINE _syscall2(int, pipe2, int *, p, - int, f ) - LSS_INLINE int LSS_NAME(pipe)( int * p) { - return LSS_NAME(pipe2)(p, 0); - } - #else - LSS_INLINE _syscall1(int, pipe, int *, p) - #endif - - LSS_INLINE pid_t LSS_NAME(gettid)() { - pid_t tid = LSS_NAME(_gettid)(); - if (tid != -1) { - return tid; - } - return LSS_NAME(getpid)(); - } - - LSS_INLINE void *LSS_NAME(mremap)(void *old_address, size_t old_size, - size_t new_size, int flags, ...) { - va_list ap; - void *new_address, *rc; - va_start(ap, flags); - new_address = va_arg(ap, void *); - rc = LSS_NAME(_mremap)(old_address, old_size, new_size, - flags, new_address); - va_end(ap); - return rc; - } - - LSS_INLINE int LSS_NAME(ptrace_detach)(pid_t pid) { - /* PTRACE_DETACH can sometimes forget to wake up the tracee and it - * then sends job control signals to the real parent, rather than to - * the tracer. We reduce the risk of this happening by starting a - * whole new time slice, and then quickly sending a SIGCONT signal - * right after detaching from the tracee. - */ - int rc, err; - LSS_NAME(sched_yield)(); - rc = LSS_NAME(ptrace)(PTRACE_DETACH, pid, (void *)0, (void *)0); - err = LSS_ERRNO; - LSS_NAME(kill)(pid, SIGCONT); - LSS_ERRNO = err; - return rc; - } -#endif - -#if defined(__cplusplus) && !defined(SYS_CPLUSPLUS) -} -#endif - -#endif -#endif diff --git a/contrib/libtcmalloc/src/base/linuxthreads.cc b/contrib/libtcmalloc/src/base/linuxthreads.cc deleted file mode 100644 index 891e70c88c4..00000000000 --- a/contrib/libtcmalloc/src/base/linuxthreads.cc +++ /dev/null @@ -1,707 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2005-2007, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Markus Gutschke - */ - -#include "base/linuxthreads.h" - -#ifdef THREADS -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "base/linux_syscall_support.h" -#include "base/thread_lister.h" - -#ifndef CLONE_UNTRACED -#define CLONE_UNTRACED 0x00800000 -#endif - - -/* Synchronous signals that should not be blocked while in the lister thread. - */ -static const int sync_signals[] = { SIGABRT, SIGILL, SIGFPE, SIGSEGV, SIGBUS, - SIGXCPU, SIGXFSZ }; - -/* itoa() is not a standard function, and we cannot safely call printf() - * after suspending threads. So, we just implement our own copy. A - * recursive approach is the easiest here. - */ -static char *local_itoa(char *buf, int i) { - if (i < 0) { - *buf++ = '-'; - return local_itoa(buf, -i); - } else { - if (i >= 10) - buf = local_itoa(buf, i/10); - *buf++ = (i%10) + '0'; - *buf = '\000'; - return buf; - } -} - - -/* Wrapper around clone() that runs "fn" on the same stack as the - * caller! Unlike fork(), the cloned thread shares the same address space. - * The caller must be careful to use only minimal amounts of stack until - * the cloned thread has returned. - * There is a good chance that the cloned thread and the caller will share - * the same copy of errno! - */ -#ifdef __GNUC__ -#if __GNUC__ == 3 && __GNUC_MINOR__ >= 1 || __GNUC__ > 3 -/* Try to force this function into a separate stack frame, and make sure - * that arguments are passed on the stack. - */ -static int local_clone (int (*fn)(void *), void *arg, ...) - __attribute__ ((noinline)); -#endif -#endif - -/* To avoid the gap cross page boundaries, increase by the large parge - * size mostly PowerPC system uses. */ -#ifdef __PPC64__ -#define CLONE_STACK_SIZE 65536 -#else -#define CLONE_STACK_SIZE 4096 -#endif - -static int local_clone (int (*fn)(void *), void *arg, ...) { - /* Leave 4kB of gap between the callers stack and the new clone. This - * should be more than sufficient for the caller to call waitpid() until - * the cloned thread terminates. - * - * It is important that we set the CLONE_UNTRACED flag, because newer - * versions of "gdb" otherwise attempt to attach to our thread, and will - * attempt to reap its status codes. This subsequently results in the - * caller hanging indefinitely in waitpid(), waiting for a change in - * status that will never happen. By setting the CLONE_UNTRACED flag, we - * prevent "gdb" from stealing events, but we still expect the thread - * lister to fail, because it cannot PTRACE_ATTACH to the process that - * is being debugged. This is OK and the error code will be reported - * correctly. - */ - return sys_clone(fn, (char *)&arg - CLONE_STACK_SIZE, - CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_UNTRACED, arg, 0, 0, 0); -} - - -/* Local substitute for the atoi() function, which is not necessarily safe - * to call once threads are suspended (depending on whether libc looks up - * locale information, when executing atoi()). - */ -static int local_atoi(const char *s) { - int n = 0; - int neg = *s == '-'; - if (neg) - s++; - while (*s >= '0' && *s <= '9') - n = 10*n + (*s++ - '0'); - return neg ? -n : n; -} - - -/* Re-runs fn until it doesn't cause EINTR - */ -#define NO_INTR(fn) do {} while ((fn) < 0 && errno == EINTR) - - -/* Wrap a class around system calls, in order to give us access to - * a private copy of errno. This only works in C++, but it has the - * advantage of not needing nested functions, which are a non-standard - * language extension. - */ -#ifdef __cplusplus -namespace { - class SysCalls { - public: - #define SYS_CPLUSPLUS - #define SYS_ERRNO my_errno - #define SYS_INLINE inline - #define SYS_PREFIX -1 - #undef SYS_LINUX_SYSCALL_SUPPORT_H - #include "linux_syscall_support.h" - SysCalls() : my_errno(0) { } - int my_errno; - }; -} -#define ERRNO sys.my_errno -#else -#define ERRNO my_errno -#endif - - -/* Wrapper for open() which is guaranteed to never return EINTR. - */ -static int c_open(const char *fname, int flags, int mode) { - ssize_t rc; - NO_INTR(rc = sys_open(fname, flags, mode)); - return rc; -} - - -/* abort() is not safely reentrant, and changes it's behavior each time - * it is called. This means, if the main application ever called abort() - * we cannot safely call it again. This would happen if we were called - * from a SIGABRT signal handler in the main application. So, document - * that calling SIGABRT from the thread lister makes it not signal safe - * (and vice-versa). - * Also, since we share address space with the main application, we - * cannot call abort() from the callback and expect the main application - * to behave correctly afterwards. In fact, the only thing we can do, is - * to terminate the main application with extreme prejudice (aka - * PTRACE_KILL). - * We set up our own SIGABRT handler to do this. - * In order to find the main application from the signal handler, we - * need to store information about it in global variables. This is - * safe, because the main application should be suspended at this - * time. If the callback ever called TCMalloc_ResumeAllProcessThreads(), then - * we are running a higher risk, though. So, try to avoid calling - * abort() after calling TCMalloc_ResumeAllProcessThreads. - */ -static volatile int *sig_pids, sig_num_threads, sig_proc, sig_marker; - - -/* Signal handler to help us recover from dying while we are attached to - * other threads. - */ -static void SignalHandler(int signum, siginfo_t *si, void *data) { - if (sig_pids != NULL) { - if (signum == SIGABRT) { - while (sig_num_threads-- > 0) { - /* Not sure if sched_yield is really necessary here, but it does not */ - /* hurt, and it might be necessary for the same reasons that we have */ - /* to do so in sys_ptrace_detach(). */ - sys_sched_yield(); - sys_ptrace(PTRACE_KILL, sig_pids[sig_num_threads], 0, 0); - } - } else if (sig_num_threads > 0) { - TCMalloc_ResumeAllProcessThreads(sig_num_threads, (int *)sig_pids); - } - } - sig_pids = NULL; - if (sig_marker >= 0) - NO_INTR(sys_close(sig_marker)); - sig_marker = -1; - if (sig_proc >= 0) - NO_INTR(sys_close(sig_proc)); - sig_proc = -1; - - sys__exit(signum == SIGABRT ? 1 : 2); -} - - -/* Try to dirty the stack, and hope that the compiler is not smart enough - * to optimize this function away. Or worse, the compiler could inline the - * function and permanently allocate the data on the stack. - */ -static void DirtyStack(size_t amount) { - char buf[amount]; - memset(buf, 0, amount); - sys_read(-1, buf, amount); -} - - -/* Data structure for passing arguments to the lister thread. - */ -#define ALT_STACKSIZE (MINSIGSTKSZ + 4096) - -struct ListerParams { - int result, err; - char *altstack_mem; - ListAllProcessThreadsCallBack callback; - void *parameter; - va_list ap; - sem_t *lock; -}; - - -static void ListerThread(struct ListerParams *args) { - int found_parent = 0; - pid_t clone_pid = sys_gettid(), ppid = sys_getppid(); - char proc_self_task[80], marker_name[48], *marker_path; - const char *proc_paths[3]; - const char *const *proc_path = proc_paths; - int proc = -1, marker = -1, num_threads = 0; - int max_threads = 0, sig; - struct kernel_stat marker_sb, proc_sb; - stack_t altstack; - - /* Wait for parent thread to set appropriate permissions - * to allow ptrace activity - */ - if (sem_wait(args->lock) < 0) { - goto failure; - } - - /* Create "marker" that we can use to detect threads sharing the same - * address space and the same file handles. By setting the FD_CLOEXEC flag - * we minimize the risk of misidentifying child processes as threads; - * and since there is still a race condition, we will filter those out - * later, anyway. - */ - if ((marker = sys_socket(PF_LOCAL, SOCK_DGRAM, 0)) < 0 || - sys_fcntl(marker, F_SETFD, FD_CLOEXEC) < 0) { - failure: - args->result = -1; - args->err = errno; - if (marker >= 0) - NO_INTR(sys_close(marker)); - sig_marker = marker = -1; - if (proc >= 0) - NO_INTR(sys_close(proc)); - sig_proc = proc = -1; - sys__exit(1); - } - - /* Compute search paths for finding thread directories in /proc */ - local_itoa(strrchr(strcpy(proc_self_task, "/proc/"), '\000'), ppid); - strcpy(marker_name, proc_self_task); - marker_path = marker_name + strlen(marker_name); - strcat(proc_self_task, "/task/"); - proc_paths[0] = proc_self_task; /* /proc/$$/task/ */ - proc_paths[1] = "/proc/"; /* /proc/ */ - proc_paths[2] = NULL; - - /* Compute path for marker socket in /proc */ - local_itoa(strcpy(marker_path, "/fd/") + 4, marker); - if (sys_stat(marker_name, &marker_sb) < 0) { - goto failure; - } - - /* Catch signals on an alternate pre-allocated stack. This way, we can - * safely execute the signal handler even if we ran out of memory. - */ - memset(&altstack, 0, sizeof(altstack)); - altstack.ss_sp = args->altstack_mem; - altstack.ss_flags = 0; - altstack.ss_size = ALT_STACKSIZE; - sys_sigaltstack(&altstack, (const stack_t *)NULL); - - /* Some kernels forget to wake up traced processes, when the - * tracer dies. So, intercept synchronous signals and make sure - * that we wake up our tracees before dying. It is the caller's - * responsibility to ensure that asynchronous signals do not - * interfere with this function. - */ - sig_marker = marker; - sig_proc = -1; - for (sig = 0; sig < sizeof(sync_signals)/sizeof(*sync_signals); sig++) { - struct kernel_sigaction sa; - memset(&sa, 0, sizeof(sa)); - sa.sa_sigaction_ = SignalHandler; - sys_sigfillset(&sa.sa_mask); - sa.sa_flags = SA_ONSTACK|SA_SIGINFO|SA_RESETHAND; - sys_sigaction(sync_signals[sig], &sa, (struct kernel_sigaction *)NULL); - } - - /* Read process directories in /proc/... */ - for (;;) { - /* Some kernels know about threads, and hide them in "/proc" - * (although they are still there, if you know the process - * id). Threads are moved into a separate "task" directory. We - * check there first, and then fall back on the older naming - * convention if necessary. - */ - if ((sig_proc = proc = c_open(*proc_path, O_RDONLY|O_DIRECTORY, 0)) < 0) { - if (*++proc_path != NULL) - continue; - goto failure; - } - if (sys_fstat(proc, &proc_sb) < 0) - goto failure; - - /* Since we are suspending threads, we cannot call any libc - * functions that might acquire locks. Most notably, we cannot - * call malloc(). So, we have to allocate memory on the stack, - * instead. Since we do not know how much memory we need, we - * make a best guess. And if we guessed incorrectly we retry on - * a second iteration (by jumping to "detach_threads"). - * - * Unless the number of threads is increasing very rapidly, we - * should never need to do so, though, as our guestimate is very - * conservative. - */ - if (max_threads < proc_sb.st_nlink + 100) - max_threads = proc_sb.st_nlink + 100; - - /* scope */ { - pid_t pids[max_threads]; - int added_entries = 0; - sig_num_threads = num_threads; - sig_pids = pids; - for (;;) { - struct KERNEL_DIRENT *entry; - char buf[4096]; - ssize_t nbytes = GETDENTS(proc, (struct KERNEL_DIRENT *)buf, - sizeof(buf)); - if (nbytes < 0) - goto failure; - else if (nbytes == 0) { - if (added_entries) { - /* Need to keep iterating over "/proc" in multiple - * passes until we no longer find any more threads. This - * algorithm eventually completes, when all threads have - * been suspended. - */ - added_entries = 0; - sys_lseek(proc, 0, SEEK_SET); - continue; - } - break; - } - for (entry = (struct KERNEL_DIRENT *)buf; - entry < (struct KERNEL_DIRENT *)&buf[nbytes]; - entry = (struct KERNEL_DIRENT *)((char *)entry+entry->d_reclen)) { - if (entry->d_ino != 0) { - const char *ptr = entry->d_name; - pid_t pid; - - /* Some kernels hide threads by preceding the pid with a '.' */ - if (*ptr == '.') - ptr++; - - /* If the directory is not numeric, it cannot be a - * process/thread - */ - if (*ptr < '0' || *ptr > '9') - continue; - pid = local_atoi(ptr); - - /* Attach (and suspend) all threads */ - if (pid && pid != clone_pid) { - struct kernel_stat tmp_sb; - char fname[entry->d_reclen + 48]; - strcat(strcat(strcpy(fname, "/proc/"), - entry->d_name), marker_path); - - /* Check if the marker is identical to the one we created */ - if (sys_stat(fname, &tmp_sb) >= 0 && - marker_sb.st_ino == tmp_sb.st_ino) { - long i, j; - - /* Found one of our threads, make sure it is no duplicate */ - for (i = 0; i < num_threads; i++) { - /* Linear search is slow, but should not matter much for - * the typically small number of threads. - */ - if (pids[i] == pid) { - /* Found a duplicate; most likely on second pass */ - goto next_entry; - } - } - - /* Check whether data structure needs growing */ - if (num_threads >= max_threads) { - /* Back to square one, this time with more memory */ - NO_INTR(sys_close(proc)); - goto detach_threads; - } - - /* Attaching to thread suspends it */ - pids[num_threads++] = pid; - sig_num_threads = num_threads; - if (sys_ptrace(PTRACE_ATTACH, pid, (void *)0, - (void *)0) < 0) { - /* If operation failed, ignore thread. Maybe it - * just died? There might also be a race - * condition with a concurrent core dumper or - * with a debugger. In that case, we will just - * make a best effort, rather than failing - * entirely. - */ - num_threads--; - sig_num_threads = num_threads; - goto next_entry; - } - while (sys_waitpid(pid, (int *)0, __WALL) < 0) { - if (errno != EINTR) { - sys_ptrace_detach(pid); - num_threads--; - sig_num_threads = num_threads; - goto next_entry; - } - } - - if (sys_ptrace(PTRACE_PEEKDATA, pid, &i, &j) || i++ != j || - sys_ptrace(PTRACE_PEEKDATA, pid, &i, &j) || i != j) { - /* Address spaces are distinct, even though both - * processes show the "marker". This is probably - * a forked child process rather than a thread. - */ - sys_ptrace_detach(pid); - num_threads--; - sig_num_threads = num_threads; - } else { - found_parent |= pid == ppid; - added_entries++; - } - } - } - } - next_entry:; - } - } - NO_INTR(sys_close(proc)); - sig_proc = proc = -1; - - /* If we failed to find any threads, try looking somewhere else in - * /proc. Maybe, threads are reported differently on this system. - */ - if (num_threads > 1 || !*++proc_path) { - NO_INTR(sys_close(marker)); - sig_marker = marker = -1; - - /* If we never found the parent process, something is very wrong. - * Most likely, we are running in debugger. Any attempt to operate - * on the threads would be very incomplete. Let's just report an - * error to the caller. - */ - if (!found_parent) { - TCMalloc_ResumeAllProcessThreads(num_threads, pids); - sys__exit(3); - } - - /* Now we are ready to call the callback, - * which takes care of resuming the threads for us. - */ - args->result = args->callback(args->parameter, num_threads, - pids, args->ap); - args->err = errno; - - /* Callback should have resumed threads, but better safe than sorry */ - if (TCMalloc_ResumeAllProcessThreads(num_threads, pids)) { - /* Callback forgot to resume at least one thread, report error */ - args->err = EINVAL; - args->result = -1; - } - - sys__exit(0); - } - detach_threads: - /* Resume all threads prior to retrying the operation */ - TCMalloc_ResumeAllProcessThreads(num_threads, pids); - sig_pids = NULL; - num_threads = 0; - sig_num_threads = num_threads; - max_threads += 100; - } - } -} - - -/* This function gets the list of all linux threads of the current process - * passes them to the 'callback' along with the 'parameter' pointer; at the - * call back call time all the threads are paused via - * PTRACE_ATTACH. - * The callback is executed from a separate thread which shares only the - * address space, the filesystem, and the filehandles with the caller. Most - * notably, it does not share the same pid and ppid; and if it terminates, - * the rest of the application is still there. 'callback' is supposed to do - * or arrange for TCMalloc_ResumeAllProcessThreads. This happens automatically, if - * the thread raises a synchronous signal (e.g. SIGSEGV); asynchronous - * signals are blocked. If the 'callback' decides to unblock them, it must - * ensure that they cannot terminate the application, or that - * TCMalloc_ResumeAllProcessThreads will get called. - * It is an error for the 'callback' to make any library calls that could - * acquire locks. Most notably, this means that most system calls have to - * avoid going through libc. Also, this means that it is not legal to call - * exit() or abort(). - * We return -1 on error and the return value of 'callback' on success. - */ -int TCMalloc_ListAllProcessThreads(void *parameter, - ListAllProcessThreadsCallBack callback, ...) { - char altstack_mem[ALT_STACKSIZE]; - struct ListerParams args; - pid_t clone_pid; - int dumpable = 1, sig; - struct kernel_sigset_t sig_blocked, sig_old; - sem_t lock; - - va_start(args.ap, callback); - - /* If we are short on virtual memory, initializing the alternate stack - * might trigger a SIGSEGV. Let's do this early, before it could get us - * into more trouble (i.e. before signal handlers try to use the alternate - * stack, and before we attach to other threads). - */ - memset(altstack_mem, 0, sizeof(altstack_mem)); - - /* Some of our cleanup functions could conceivable use more stack space. - * Try to touch the stack right now. This could be defeated by the compiler - * being too smart for it's own good, so try really hard. - */ - DirtyStack(32768); - - /* Make this process "dumpable". This is necessary in order to ptrace() - * after having called setuid(). - */ - dumpable = sys_prctl(PR_GET_DUMPABLE, 0); - if (!dumpable) - sys_prctl(PR_SET_DUMPABLE, 1); - - /* Fill in argument block for dumper thread */ - args.result = -1; - args.err = 0; - args.altstack_mem = altstack_mem; - args.parameter = parameter; - args.callback = callback; - args.lock = &lock; - - /* Before cloning the thread lister, block all asynchronous signals, as we */ - /* are not prepared to handle them. */ - sys_sigfillset(&sig_blocked); - for (sig = 0; sig < sizeof(sync_signals)/sizeof(*sync_signals); sig++) { - sys_sigdelset(&sig_blocked, sync_signals[sig]); - } - if (sys_sigprocmask(SIG_BLOCK, &sig_blocked, &sig_old)) { - args.err = errno; - args.result = -1; - goto failed; - } - - /* scope */ { - /* After cloning, both the parent and the child share the same instance - * of errno. We must make sure that at least one of these processes - * (in our case, the parent) uses modified syscall macros that update - * a local copy of errno, instead. - */ - #ifdef __cplusplus - #define sys0_sigprocmask sys.sigprocmask - #define sys0_waitpid sys.waitpid - SysCalls sys; - #else - int my_errno; - #define SYS_ERRNO my_errno - #define SYS_INLINE inline - #define SYS_PREFIX 0 - #undef SYS_LINUX_SYSCALL_SUPPORT_H - #include "linux_syscall_support.h" - #endif - - /* Lock before clone so that parent can set - * ptrace permissions (if necessary) prior - * to ListerThread actually executing - */ - if (sem_init(&lock, 0, 0) == 0) { - - int clone_errno; - clone_pid = local_clone((int (*)(void *))ListerThread, &args); - clone_errno = errno; - - sys_sigprocmask(SIG_SETMASK, &sig_old, &sig_old); - - if (clone_pid >= 0) { -#ifdef PR_SET_PTRACER - /* In newer versions of glibc permission must explicitly - * be given to allow for ptrace. - */ - prctl(PR_SET_PTRACER, clone_pid, 0, 0, 0); -#endif - /* Releasing the lock here allows the - * ListerThread to execute and ptrace us. - */ - sem_post(&lock); - int status, rc; - while ((rc = sys0_waitpid(clone_pid, &status, __WALL)) < 0 && - ERRNO == EINTR) { - /* Keep waiting */ - } - if (rc < 0) { - args.err = ERRNO; - args.result = -1; - } else if (WIFEXITED(status)) { - switch (WEXITSTATUS(status)) { - case 0: break; /* Normal process termination */ - case 2: args.err = EFAULT; /* Some fault (e.g. SIGSEGV) detected */ - args.result = -1; - break; - case 3: args.err = EPERM; /* Process is already being traced */ - args.result = -1; - break; - default:args.err = ECHILD; /* Child died unexpectedly */ - args.result = -1; - break; - } - } else if (!WIFEXITED(status)) { - args.err = EFAULT; /* Terminated due to an unhandled signal*/ - args.result = -1; - } - sem_destroy(&lock); - } else { - args.result = -1; - args.err = clone_errno; - } - } else { - args.result = -1; - args.err = errno; - } - } - - /* Restore the "dumpable" state of the process */ -failed: - if (!dumpable) - sys_prctl(PR_SET_DUMPABLE, dumpable); - - va_end(args.ap); - - errno = args.err; - return args.result; -} - -/* This function resumes the list of all linux threads that - * TCMalloc_ListAllProcessThreads pauses before giving to its callback. - * The function returns non-zero if at least one thread was - * suspended and has now been resumed. - */ -int TCMalloc_ResumeAllProcessThreads(int num_threads, pid_t *thread_pids) { - int detached_at_least_one = 0; - while (num_threads-- > 0) { - detached_at_least_one |= sys_ptrace_detach(thread_pids[num_threads]) >= 0; - } - return detached_at_least_one; -} - -#ifdef __cplusplus -} -#endif -#endif diff --git a/contrib/libtcmalloc/src/base/linuxthreads.h b/contrib/libtcmalloc/src/base/linuxthreads.h deleted file mode 100644 index 09ce45fc13f..00000000000 --- a/contrib/libtcmalloc/src/base/linuxthreads.h +++ /dev/null @@ -1,54 +0,0 @@ -/* Copyright (c) 2005-2007, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Markus Gutschke - */ - -#ifndef _LINUXTHREADS_H -#define _LINUXTHREADS_H - -/* Include thread_lister.h to get the interface that we implement for linux. - */ - -/* We currently only support certain platforms on Linux. Porting to other - * related platforms should not be difficult. - */ -#if (defined(__i386__) || defined(__x86_64__) || defined(__arm__) || \ - defined(__mips__) || defined(__PPC__) || defined(__aarch64__) || \ - defined(__s390__)) && defined(__linux) - -/* Define the THREADS symbol to make sure that there is exactly one core dumper - * built into the library. - */ -#define THREADS "Linux /proc" - -#endif - -#endif /* _LINUXTHREADS_H */ diff --git a/contrib/libtcmalloc/src/base/logging.cc b/contrib/libtcmalloc/src/base/logging.cc deleted file mode 100644 index 2b0adcb8945..00000000000 --- a/contrib/libtcmalloc/src/base/logging.cc +++ /dev/null @@ -1,108 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2007, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// This file just provides storage for FLAGS_verbose. - -#include "../config.h" -#include "base/logging.h" -#include "base/commandlineflags.h" - -DEFINE_int32(verbose, EnvToInt("PERFTOOLS_VERBOSE", 0), - "Set to numbers >0 for more verbose output, or <0 for less. " - "--verbose == -4 means we log fatal errors only."); - - -#if defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) - -// While windows does have a POSIX-compatible API -// (_open/_write/_close), it acquires memory. Using this lower-level -// windows API is the closest we can get to being "raw". -RawFD RawOpenForWriting(const char* filename) { - // CreateFile allocates memory if file_name isn't absolute, so if - // that ever becomes a problem then we ought to compute the absolute - // path on its behalf (perhaps the ntdll/kernel function isn't aware - // of the working directory?) - RawFD fd = CreateFileA(filename, GENERIC_WRITE, 0, NULL, - CREATE_ALWAYS, 0, NULL); - if (fd != kIllegalRawFD && GetLastError() == ERROR_ALREADY_EXISTS) - SetEndOfFile(fd); // truncate the existing file - return fd; -} - -void RawWrite(RawFD handle, const char* buf, size_t len) { - while (len > 0) { - DWORD wrote; - BOOL ok = WriteFile(handle, buf, len, &wrote, NULL); - // We do not use an asynchronous file handle, so ok==false means an error - if (!ok) break; - buf += wrote; - len -= wrote; - } -} - -void RawClose(RawFD handle) { - CloseHandle(handle); -} - -#else // _WIN32 || __CYGWIN__ || __CYGWIN32__ - -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_UNISTD_H -#include -#endif -#ifdef HAVE_FCNTL_H -#include -#endif - -// Re-run fn until it doesn't cause EINTR. -#define NO_INTR(fn) do {} while ((fn) < 0 && errno == EINTR) - -RawFD RawOpenForWriting(const char* filename) { - return open(filename, O_WRONLY|O_CREAT|O_TRUNC, 0664); -} - -void RawWrite(RawFD fd, const char* buf, size_t len) { - while (len > 0) { - ssize_t r; - NO_INTR(r = write(fd, buf, len)); - if (r <= 0) break; - buf += r; - len -= r; - } -} - -void RawClose(RawFD fd) { - NO_INTR(close(fd)); -} - -#endif // _WIN32 || __CYGWIN__ || __CYGWIN32__ diff --git a/contrib/libtcmalloc/src/base/logging.h b/contrib/libtcmalloc/src/base/logging.h deleted file mode 100644 index fa22489bea3..00000000000 --- a/contrib/libtcmalloc/src/base/logging.h +++ /dev/null @@ -1,259 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// This file contains #include information about logging-related stuff. -// Pretty much everybody needs to #include this file so that they can -// log various happenings. -// -#ifndef _LOGGING_H_ -#define _LOGGING_H_ - -#include "../config.h" -#include -#include -#include -#ifdef HAVE_UNISTD_H -#include // for write() -#endif -#include // for strlen(), strcmp() -#include -#include // for errno -#include "base/commandlineflags.h" - -// On some systems (like freebsd), we can't call write() at all in a -// global constructor, perhaps because errno hasn't been set up. -// (In windows, we can't call it because it might call malloc.) -// Calling the write syscall is safer (it doesn't set errno), so we -// prefer that. Note we don't care about errno for logging: we just -// do logging on a best-effort basis. -#if defined(_MSC_VER) -#define WRITE_TO_STDERR(buf, len) WriteToStderr(buf, len); // in port.cc -#elif defined(HAVE_SYS_SYSCALL_H) -#include -#define WRITE_TO_STDERR(buf, len) syscall(SYS_write, STDERR_FILENO, buf, len) -#else -#define WRITE_TO_STDERR(buf, len) write(STDERR_FILENO, buf, len) -#endif - -// MSVC and mingw define their own, safe version of vnsprintf (the -// windows one in broken) in port.cc. Everyone else can use the -// version here. We had to give it a unique name for windows. -#ifndef _WIN32 -# define perftools_vsnprintf vsnprintf -#endif - - -// We log all messages at this log-level and below. -// INFO == -1, WARNING == -2, ERROR == -3, FATAL == -4 -DECLARE_int32(verbose); - -// CHECK dies with a fatal error if condition is not true. It is *not* -// controlled by NDEBUG, so the check will be executed regardless of -// compilation mode. Therefore, it is safe to do things like: -// CHECK(fp->Write(x) == 4) -// Note we use write instead of printf/puts to avoid the risk we'll -// call malloc(). -#define CHECK(condition) \ - do { \ - if (!(condition)) { \ - WRITE_TO_STDERR("Check failed: " #condition "\n", \ - sizeof("Check failed: " #condition "\n")-1); \ - abort(); \ - } \ - } while (0) - -// This takes a message to print. The name is historical. -#define RAW_CHECK(condition, message) \ - do { \ - if (!(condition)) { \ - WRITE_TO_STDERR("Check failed: " #condition ": " message "\n", \ - sizeof("Check failed: " #condition ": " message "\n")-1);\ - abort(); \ - } \ - } while (0) - -// This is like RAW_CHECK, but only in debug-mode -#ifdef NDEBUG -enum { DEBUG_MODE = 0 }; -#define RAW_DCHECK(condition, message) -#else -enum { DEBUG_MODE = 1 }; -#define RAW_DCHECK(condition, message) RAW_CHECK(condition, message) -#endif - -// This prints errno as well. Note we use write instead of printf/puts to -// avoid the risk we'll call malloc(). -#define PCHECK(condition) \ - do { \ - if (!(condition)) { \ - const int err_no = errno; \ - WRITE_TO_STDERR("Check failed: " #condition ": ", \ - sizeof("Check failed: " #condition ": ")-1); \ - WRITE_TO_STDERR(strerror(err_no), strlen(strerror(err_no))); \ - WRITE_TO_STDERR("\n", sizeof("\n")-1); \ - abort(); \ - } \ - } while (0) - -// Helper macro for binary operators; prints the two values on error -// Don't use this macro directly in your code, use CHECK_EQ et al below - -// WARNING: These don't compile correctly if one of the arguments is a pointer -// and the other is NULL. To work around this, simply static_cast NULL to the -// type of the desired pointer. - -// TODO(jandrews): Also print the values in case of failure. Requires some -// sort of type-sensitive ToString() function. -#define CHECK_OP(op, val1, val2) \ - do { \ - if (!((val1) op (val2))) { \ - fprintf(stderr, "Check failed: %s %s %s\n", #val1, #op, #val2); \ - abort(); \ - } \ - } while (0) - -#define CHECK_EQ(val1, val2) CHECK_OP(==, val1, val2) -#define CHECK_NE(val1, val2) CHECK_OP(!=, val1, val2) -#define CHECK_LE(val1, val2) CHECK_OP(<=, val1, val2) -#define CHECK_LT(val1, val2) CHECK_OP(< , val1, val2) -#define CHECK_GE(val1, val2) CHECK_OP(>=, val1, val2) -#define CHECK_GT(val1, val2) CHECK_OP(> , val1, val2) - -// Synonyms for CHECK_* that are used in some unittests. -#define EXPECT_EQ(val1, val2) CHECK_EQ(val1, val2) -#define EXPECT_NE(val1, val2) CHECK_NE(val1, val2) -#define EXPECT_LE(val1, val2) CHECK_LE(val1, val2) -#define EXPECT_LT(val1, val2) CHECK_LT(val1, val2) -#define EXPECT_GE(val1, val2) CHECK_GE(val1, val2) -#define EXPECT_GT(val1, val2) CHECK_GT(val1, val2) -#define ASSERT_EQ(val1, val2) EXPECT_EQ(val1, val2) -#define ASSERT_NE(val1, val2) EXPECT_NE(val1, val2) -#define ASSERT_LE(val1, val2) EXPECT_LE(val1, val2) -#define ASSERT_LT(val1, val2) EXPECT_LT(val1, val2) -#define ASSERT_GE(val1, val2) EXPECT_GE(val1, val2) -#define ASSERT_GT(val1, val2) EXPECT_GT(val1, val2) -// As are these variants. -#define EXPECT_TRUE(cond) CHECK(cond) -#define EXPECT_FALSE(cond) CHECK(!(cond)) -#define EXPECT_STREQ(a, b) CHECK(strcmp(a, b) == 0) -#define ASSERT_TRUE(cond) EXPECT_TRUE(cond) -#define ASSERT_FALSE(cond) EXPECT_FALSE(cond) -#define ASSERT_STREQ(a, b) EXPECT_STREQ(a, b) - -// Used for (libc) functions that return -1 and set errno -#define CHECK_ERR(invocation) PCHECK((invocation) != -1) - -// A few more checks that only happen in debug mode -#ifdef NDEBUG -#define DCHECK_EQ(val1, val2) -#define DCHECK_NE(val1, val2) -#define DCHECK_LE(val1, val2) -#define DCHECK_LT(val1, val2) -#define DCHECK_GE(val1, val2) -#define DCHECK_GT(val1, val2) -#else -#define DCHECK_EQ(val1, val2) CHECK_EQ(val1, val2) -#define DCHECK_NE(val1, val2) CHECK_NE(val1, val2) -#define DCHECK_LE(val1, val2) CHECK_LE(val1, val2) -#define DCHECK_LT(val1, val2) CHECK_LT(val1, val2) -#define DCHECK_GE(val1, val2) CHECK_GE(val1, val2) -#define DCHECK_GT(val1, val2) CHECK_GT(val1, val2) -#endif - - -#ifdef ERROR -#undef ERROR // may conflict with ERROR macro on windows -#endif -enum LogSeverity {INFO = -1, WARNING = -2, ERROR = -3, FATAL = -4}; - -// NOTE: we add a newline to the end of the output if it's not there already -inline void LogPrintf(int severity, const char* pat, va_list ap) { - // We write directly to the stderr file descriptor and avoid FILE - // buffering because that may invoke malloc() - char buf[600]; - perftools_vsnprintf(buf, sizeof(buf)-1, pat, ap); - if (buf[0] != '\0' && buf[strlen(buf)-1] != '\n') { - assert(strlen(buf)+1 < sizeof(buf)); - strcat(buf, "\n"); - } - WRITE_TO_STDERR(buf, strlen(buf)); - if ((severity) == FATAL) - abort(); // LOG(FATAL) indicates a big problem, so don't run atexit() calls -} - -// Note that since the order of global constructors is unspecified, -// global code that calls RAW_LOG may execute before FLAGS_verbose is set. -// Such code will run with verbosity == 0 no matter what. -#define VLOG_IS_ON(severity) (FLAGS_verbose >= severity) - -// In a better world, we'd use __VA_ARGS__, but VC++ 7 doesn't support it. -#define LOG_PRINTF(severity, pat) do { \ - if (VLOG_IS_ON(severity)) { \ - va_list ap; \ - va_start(ap, pat); \ - LogPrintf(severity, pat, ap); \ - va_end(ap); \ - } \ -} while (0) - -// RAW_LOG is the main function; some synonyms are used in unittests. -inline void RAW_LOG(int lvl, const char* pat, ...) { LOG_PRINTF(lvl, pat); } -inline void RAW_VLOG(int lvl, const char* pat, ...) { LOG_PRINTF(lvl, pat); } -inline void LOG(int lvl, const char* pat, ...) { LOG_PRINTF(lvl, pat); } -inline void VLOG(int lvl, const char* pat, ...) { LOG_PRINTF(lvl, pat); } -inline void LOG_IF(int lvl, bool cond, const char* pat, ...) { - if (cond) LOG_PRINTF(lvl, pat); -} - -// This isn't technically logging, but it's also IO and also is an -// attempt to be "raw" -- that is, to not use any higher-level libc -// routines that might allocate memory or (ideally) try to allocate -// locks. We use an opaque file handle (not necessarily an int) -// to allow even more low-level stuff in the future. -// Like other "raw" routines, these functions are best effort, and -// thus don't return error codes (except RawOpenForWriting()). -#if defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) -#ifndef NOMINMAX -#define NOMINMAX // @#!$& windows -#endif -#include -typedef HANDLE RawFD; -const RawFD kIllegalRawFD = INVALID_HANDLE_VALUE; -#else -typedef int RawFD; -const RawFD kIllegalRawFD = -1; // what open returns if it fails -#endif // defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) - -RawFD RawOpenForWriting(const char* filename); // uses default permissions -void RawWrite(RawFD fd, const char* buf, size_t len); -void RawClose(RawFD fd); - -#endif // _LOGGING_H_ diff --git a/contrib/libtcmalloc/src/base/low_level_alloc.cc b/contrib/libtcmalloc/src/base/low_level_alloc.cc deleted file mode 100644 index 6b467cff123..00000000000 --- a/contrib/libtcmalloc/src/base/low_level_alloc.cc +++ /dev/null @@ -1,582 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2006, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -// A low-level allocator that can be used by other low-level -// modules without introducing dependency cycles. -// This allocator is slow and wasteful of memory; -// it should not be used when performance is key. - -#include "base/low_level_alloc.h" -#include "base/dynamic_annotations.h" -#include "base/spinlock.h" -#include "base/logging.h" -#include "malloc_hook-inl.h" -#include -#include -#ifdef HAVE_UNISTD_H -#include -#endif -#ifdef HAVE_MMAP -#include -#endif -#include // for placement-new - -// On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old -// form of the name instead. -#ifndef MAP_ANONYMOUS -# define MAP_ANONYMOUS MAP_ANON -#endif - -// A first-fit allocator with amortized logarithmic free() time. - -LowLevelAlloc::PagesAllocator::~PagesAllocator() { -} - -// --------------------------------------------------------------------------- -static const int kMaxLevel = 30; - -// We put this class-only struct in a namespace to avoid polluting the -// global namespace with this struct name (thus risking an ODR violation). -namespace low_level_alloc_internal { - // This struct describes one allocated block, or one free block. - struct AllocList { - struct Header { - intptr_t size; // size of entire region, including this field. Must be - // first. Valid in both allocated and unallocated blocks - intptr_t magic; // kMagicAllocated or kMagicUnallocated xor this - LowLevelAlloc::Arena *arena; // pointer to parent arena - void *dummy_for_alignment; // aligns regions to 0 mod 2*sizeof(void*) - } header; - - // Next two fields: in unallocated blocks: freelist skiplist data - // in allocated blocks: overlaps with client data - int levels; // levels in skiplist used - AllocList *next[kMaxLevel]; // actually has levels elements. - // The AllocList node may not have room for - // all kMaxLevel entries. See max_fit in - // LLA_SkiplistLevels() - }; -} -using low_level_alloc_internal::AllocList; - - -// --------------------------------------------------------------------------- -// A trivial skiplist implementation. This is used to keep the freelist -// in address order while taking only logarithmic time per insert and delete. - -// An integer approximation of log2(size/base) -// Requires size >= base. -static int IntLog2(size_t size, size_t base) { - int result = 0; - for (size_t i = size; i > base; i >>= 1) { // i == floor(size/2**result) - result++; - } - // floor(size / 2**result) <= base < floor(size / 2**(result-1)) - // => log2(size/(base+1)) <= result < 1+log2(size/base) - // => result ~= log2(size/base) - return result; -} - -// Return a random integer n: p(n)=1/(2**n) if 1 <= n; p(n)=0 if n < 1. -static int Random() { - static uint32 r = 1; // no locking---it's not critical - ANNOTATE_BENIGN_RACE(&r, "benign race, not critical."); - int result = 1; - while ((((r = r*1103515245 + 12345) >> 30) & 1) == 0) { - result++; - } - return result; -} - -// Return a number of skiplist levels for a node of size bytes, where -// base is the minimum node size. Compute level=log2(size / base)+n -// where n is 1 if random is false and otherwise a random number generated with -// the standard distribution for a skiplist: See Random() above. -// Bigger nodes tend to have more skiplist levels due to the log2(size / base) -// term, so first-fit searches touch fewer nodes. "level" is clipped so -// level max_fit) level = max_fit; - if (level > kMaxLevel-1) level = kMaxLevel - 1; - RAW_CHECK(level >= 1, "block not big enough for even one level"); - return level; -} - -// Return "atleast", the first element of AllocList *head s.t. *atleast >= *e. -// For 0 <= i < head->levels, set prev[i] to "no_greater", where no_greater -// points to the last element at level i in the AllocList less than *e, or is -// head if no such element exists. -static AllocList *LLA_SkiplistSearch(AllocList *head, - AllocList *e, AllocList **prev) { - AllocList *p = head; - for (int level = head->levels - 1; level >= 0; level--) { - for (AllocList *n; (n = p->next[level]) != 0 && n < e; p = n) { - } - prev[level] = p; - } - return (head->levels == 0) ? 0 : prev[0]->next[0]; -} - -// Insert element *e into AllocList *head. Set prev[] as LLA_SkiplistSearch. -// Requires that e->levels be previously set by the caller (using -// LLA_SkiplistLevels()) -static void LLA_SkiplistInsert(AllocList *head, AllocList *e, - AllocList **prev) { - LLA_SkiplistSearch(head, e, prev); - for (; head->levels < e->levels; head->levels++) { // extend prev pointers - prev[head->levels] = head; // to all *e's levels - } - for (int i = 0; i != e->levels; i++) { // add element to list - e->next[i] = prev[i]->next[i]; - prev[i]->next[i] = e; - } -} - -// Remove element *e from AllocList *head. Set prev[] as LLA_SkiplistSearch(). -// Requires that e->levels be previous set by the caller (using -// LLA_SkiplistLevels()) -static void LLA_SkiplistDelete(AllocList *head, AllocList *e, - AllocList **prev) { - AllocList *found = LLA_SkiplistSearch(head, e, prev); - RAW_CHECK(e == found, "element not in freelist"); - for (int i = 0; i != e->levels && prev[i]->next[i] == e; i++) { - prev[i]->next[i] = e->next[i]; - } - while (head->levels > 0 && head->next[head->levels - 1] == 0) { - head->levels--; // reduce head->levels if level unused - } -} - -// --------------------------------------------------------------------------- -// Arena implementation - -struct LowLevelAlloc::Arena { - Arena() : mu(SpinLock::LINKER_INITIALIZED) {} // does nothing; for static init - explicit Arena(int) : pagesize(0) {} // set pagesize to zero explicitly - // for non-static init - - SpinLock mu; // protects freelist, allocation_count, - // pagesize, roundup, min_size - AllocList freelist; // head of free list; sorted by addr (under mu) - int32 allocation_count; // count of allocated blocks (under mu) - int32 flags; // flags passed to NewArena (ro after init) - size_t pagesize; // ==getpagesize() (init under mu, then ro) - size_t roundup; // lowest power of 2 >= max(16,sizeof (AllocList)) - // (init under mu, then ro) - size_t min_size; // smallest allocation block size - // (init under mu, then ro) - PagesAllocator *allocator; -}; - -// The default arena, which is used when 0 is passed instead of an Arena -// pointer. -static struct LowLevelAlloc::Arena default_arena; - -// Non-malloc-hooked arenas: used only to allocate metadata for arenas that -// do not want malloc hook reporting, so that for them there's no malloc hook -// reporting even during arena creation. -static struct LowLevelAlloc::Arena unhooked_arena; -static struct LowLevelAlloc::Arena unhooked_async_sig_safe_arena; - -namespace { - - class DefaultPagesAllocator : public LowLevelAlloc::PagesAllocator { - public: - virtual ~DefaultPagesAllocator() {}; - virtual void *MapPages(int32 flags, size_t size); - virtual void UnMapPages(int32 flags, void *addr, size_t size); - }; - -} - -// magic numbers to identify allocated and unallocated blocks -static const intptr_t kMagicAllocated = 0x4c833e95; -static const intptr_t kMagicUnallocated = ~kMagicAllocated; - -namespace { - class SCOPED_LOCKABLE ArenaLock { - public: - explicit ArenaLock(LowLevelAlloc::Arena *arena) - EXCLUSIVE_LOCK_FUNCTION(arena->mu) - : left_(false), mask_valid_(false), arena_(arena) { - if ((arena->flags & LowLevelAlloc::kAsyncSignalSafe) != 0) { - // We've decided not to support async-signal-safe arena use until - // there a demonstrated need. Here's how one could do it though - // (would need to be made more portable). -#if 0 - sigset_t all; - sigfillset(&all); - this->mask_valid_ = - (pthread_sigmask(SIG_BLOCK, &all, &this->mask_) == 0); -#else - RAW_CHECK(false, "We do not yet support async-signal-safe arena."); -#endif - } - this->arena_->mu.Lock(); - } - ~ArenaLock() { RAW_CHECK(this->left_, "haven't left Arena region"); } - void Leave() /*UNLOCK_FUNCTION()*/ { - this->arena_->mu.Unlock(); -#if 0 - if (this->mask_valid_) { - pthread_sigmask(SIG_SETMASK, &this->mask_, 0); - } -#endif - this->left_ = true; - } - private: - bool left_; // whether left region - bool mask_valid_; -#if 0 - sigset_t mask_; // old mask of blocked signals -#endif - LowLevelAlloc::Arena *arena_; - DISALLOW_COPY_AND_ASSIGN(ArenaLock); - }; -} // anonymous namespace - -// create an appropriate magic number for an object at "ptr" -// "magic" should be kMagicAllocated or kMagicUnallocated -inline static intptr_t Magic(intptr_t magic, AllocList::Header *ptr) { - return magic ^ reinterpret_cast(ptr); -} - -// Initialize the fields of an Arena -static void ArenaInit(LowLevelAlloc::Arena *arena) { - if (arena->pagesize == 0) { - arena->pagesize = getpagesize(); - // Round up block sizes to a power of two close to the header size. - arena->roundup = 16; - while (arena->roundup < sizeof (arena->freelist.header)) { - arena->roundup += arena->roundup; - } - // Don't allocate blocks less than twice the roundup size to avoid tiny - // free blocks. - arena->min_size = 2 * arena->roundup; - arena->freelist.header.size = 0; - arena->freelist.header.magic = - Magic(kMagicUnallocated, &arena->freelist.header); - arena->freelist.header.arena = arena; - arena->freelist.levels = 0; - memset(arena->freelist.next, 0, sizeof (arena->freelist.next)); - arena->allocation_count = 0; - if (arena == &default_arena) { - // Default arena should be hooked, e.g. for heap-checker to trace - // pointer chains through objects in the default arena. - arena->flags = LowLevelAlloc::kCallMallocHook; - } else if (arena == &unhooked_async_sig_safe_arena) { - arena->flags = LowLevelAlloc::kAsyncSignalSafe; - } else { - arena->flags = 0; // other arenas' flags may be overridden by client, - // but unhooked_arena will have 0 in 'flags'. - } - arena->allocator = LowLevelAlloc::GetDefaultPagesAllocator(); - } -} - -// L < meta_data_arena->mu -LowLevelAlloc::Arena *LowLevelAlloc::NewArena(int32 flags, - Arena *meta_data_arena) { - return NewArenaWithCustomAlloc(flags, meta_data_arena, NULL); -} - -// L < meta_data_arena->mu -LowLevelAlloc::Arena *LowLevelAlloc::NewArenaWithCustomAlloc(int32 flags, - Arena *meta_data_arena, - PagesAllocator *allocator) { - RAW_CHECK(meta_data_arena != 0, "must pass a valid arena"); - if (meta_data_arena == &default_arena) { - if ((flags & LowLevelAlloc::kAsyncSignalSafe) != 0) { - meta_data_arena = &unhooked_async_sig_safe_arena; - } else if ((flags & LowLevelAlloc::kCallMallocHook) == 0) { - meta_data_arena = &unhooked_arena; - } - } - // Arena(0) uses the constructor for non-static contexts - Arena *result = - new (AllocWithArena(sizeof (*result), meta_data_arena)) Arena(0); - ArenaInit(result); - result->flags = flags; - if (allocator) { - result->allocator = allocator; - } - return result; -} - -// L < arena->mu, L < arena->arena->mu -bool LowLevelAlloc::DeleteArena(Arena *arena) { - RAW_CHECK(arena != 0 && arena != &default_arena && arena != &unhooked_arena, - "may not delete default arena"); - ArenaLock section(arena); - bool empty = (arena->allocation_count == 0); - section.Leave(); - if (empty) { - while (arena->freelist.next[0] != 0) { - AllocList *region = arena->freelist.next[0]; - size_t size = region->header.size; - arena->freelist.next[0] = region->next[0]; - RAW_CHECK(region->header.magic == - Magic(kMagicUnallocated, ®ion->header), - "bad magic number in DeleteArena()"); - RAW_CHECK(region->header.arena == arena, - "bad arena pointer in DeleteArena()"); - RAW_CHECK(size % arena->pagesize == 0, - "empty arena has non-page-aligned block size"); - RAW_CHECK(reinterpret_cast(region) % arena->pagesize == 0, - "empty arena has non-page-aligned block"); - int munmap_result; - if ((arena->flags & LowLevelAlloc::kAsyncSignalSafe) == 0) { - munmap_result = munmap(region, size); - } else { - munmap_result = MallocHook::UnhookedMUnmap(region, size); - } - RAW_CHECK(munmap_result == 0, - "LowLevelAlloc::DeleteArena: munmap failed address"); - } - Free(arena); - } - return empty; -} - -// --------------------------------------------------------------------------- - -// Return value rounded up to next multiple of align. -// align must be a power of two. -static intptr_t RoundUp(intptr_t addr, intptr_t align) { - return (addr + align - 1) & ~(align - 1); -} - -// Equivalent to "return prev->next[i]" but with sanity checking -// that the freelist is in the correct order, that it -// consists of regions marked "unallocated", and that no two regions -// are adjacent in memory (they should have been coalesced). -// L < arena->mu -static AllocList *Next(int i, AllocList *prev, LowLevelAlloc::Arena *arena) { - RAW_CHECK(i < prev->levels, "too few levels in Next()"); - AllocList *next = prev->next[i]; - if (next != 0) { - RAW_CHECK(next->header.magic == Magic(kMagicUnallocated, &next->header), - "bad magic number in Next()"); - RAW_CHECK(next->header.arena == arena, - "bad arena pointer in Next()"); - if (prev != &arena->freelist) { - RAW_CHECK(prev < next, "unordered freelist"); - RAW_CHECK(reinterpret_cast(prev) + prev->header.size < - reinterpret_cast(next), "malformed freelist"); - } - } - return next; -} - -// Coalesce list item "a" with its successor if they are adjacent. -static void Coalesce(AllocList *a) { - AllocList *n = a->next[0]; - if (n != 0 && reinterpret_cast(a) + a->header.size == - reinterpret_cast(n)) { - LowLevelAlloc::Arena *arena = a->header.arena; - a->header.size += n->header.size; - n->header.magic = 0; - n->header.arena = 0; - AllocList *prev[kMaxLevel]; - LLA_SkiplistDelete(&arena->freelist, n, prev); - LLA_SkiplistDelete(&arena->freelist, a, prev); - a->levels = LLA_SkiplistLevels(a->header.size, arena->min_size, true); - LLA_SkiplistInsert(&arena->freelist, a, prev); - } -} - -// Adds block at location "v" to the free list -// L >= arena->mu -static void AddToFreelist(void *v, LowLevelAlloc::Arena *arena) { - AllocList *f = reinterpret_cast( - reinterpret_cast(v) - sizeof (f->header)); - RAW_CHECK(f->header.magic == Magic(kMagicAllocated, &f->header), - "bad magic number in AddToFreelist()"); - RAW_CHECK(f->header.arena == arena, - "bad arena pointer in AddToFreelist()"); - f->levels = LLA_SkiplistLevels(f->header.size, arena->min_size, true); - AllocList *prev[kMaxLevel]; - LLA_SkiplistInsert(&arena->freelist, f, prev); - f->header.magic = Magic(kMagicUnallocated, &f->header); - Coalesce(f); // maybe coalesce with successor - Coalesce(prev[0]); // maybe coalesce with predecessor -} - -// Frees storage allocated by LowLevelAlloc::Alloc(). -// L < arena->mu -void LowLevelAlloc::Free(void *v) { - if (v != 0) { - AllocList *f = reinterpret_cast( - reinterpret_cast(v) - sizeof (f->header)); - RAW_CHECK(f->header.magic == Magic(kMagicAllocated, &f->header), - "bad magic number in Free()"); - LowLevelAlloc::Arena *arena = f->header.arena; - if ((arena->flags & kCallMallocHook) != 0) { - MallocHook::InvokeDeleteHook(v); - } - ArenaLock section(arena); - AddToFreelist(v, arena); - RAW_CHECK(arena->allocation_count > 0, "nothing in arena to free"); - arena->allocation_count--; - section.Leave(); - } -} - -// allocates and returns a block of size bytes, to be freed with Free() -// L < arena->mu -static void *DoAllocWithArena(size_t request, LowLevelAlloc::Arena *arena) { - void *result = 0; - if (request != 0) { - AllocList *s; // will point to region that satisfies request - ArenaLock section(arena); - ArenaInit(arena); - // round up with header - size_t req_rnd = RoundUp(request + sizeof (s->header), arena->roundup); - for (;;) { // loop until we find a suitable region - // find the minimum levels that a block of this size must have - int i = LLA_SkiplistLevels(req_rnd, arena->min_size, false) - 1; - if (i < arena->freelist.levels) { // potential blocks exist - AllocList *before = &arena->freelist; // predecessor of s - while ((s = Next(i, before, arena)) != 0 && s->header.size < req_rnd) { - before = s; - } - if (s != 0) { // we found a region - break; - } - } - // we unlock before mmap() both because mmap() may call a callback hook, - // and because it may be slow. - arena->mu.Unlock(); - // mmap generous 64K chunks to decrease - // the chances/impact of fragmentation: - size_t new_pages_size = RoundUp(req_rnd, arena->pagesize * 16); - void *new_pages = arena->allocator->MapPages(arena->flags, new_pages_size); - arena->mu.Lock(); - s = reinterpret_cast(new_pages); - s->header.size = new_pages_size; - // Pretend the block is allocated; call AddToFreelist() to free it. - s->header.magic = Magic(kMagicAllocated, &s->header); - s->header.arena = arena; - AddToFreelist(&s->levels, arena); // insert new region into free list - } - AllocList *prev[kMaxLevel]; - LLA_SkiplistDelete(&arena->freelist, s, prev); // remove from free list - // s points to the first free region that's big enough - if (req_rnd + arena->min_size <= s->header.size) { // big enough to split - AllocList *n = reinterpret_cast - (req_rnd + reinterpret_cast(s)); - n->header.size = s->header.size - req_rnd; - n->header.magic = Magic(kMagicAllocated, &n->header); - n->header.arena = arena; - s->header.size = req_rnd; - AddToFreelist(&n->levels, arena); - } - s->header.magic = Magic(kMagicAllocated, &s->header); - RAW_CHECK(s->header.arena == arena, ""); - arena->allocation_count++; - section.Leave(); - result = &s->levels; - } - ANNOTATE_NEW_MEMORY(result, request); - return result; -} - -void *LowLevelAlloc::Alloc(size_t request) { - void *result = DoAllocWithArena(request, &default_arena); - if ((default_arena.flags & kCallMallocHook) != 0) { - // this call must be directly in the user-called allocator function - // for MallocHook::GetCallerStackTrace to work properly - MallocHook::InvokeNewHook(result, request); - } - return result; -} - -void *LowLevelAlloc::AllocWithArena(size_t request, Arena *arena) { - RAW_CHECK(arena != 0, "must pass a valid arena"); - void *result = DoAllocWithArena(request, arena); - if ((arena->flags & kCallMallocHook) != 0) { - // this call must be directly in the user-called allocator function - // for MallocHook::GetCallerStackTrace to work properly - MallocHook::InvokeNewHook(result, request); - } - return result; -} - -LowLevelAlloc::Arena *LowLevelAlloc::DefaultArena() { - return &default_arena; -} - -static DefaultPagesAllocator *default_pages_allocator; -static union { - char chars[sizeof(DefaultPagesAllocator)]; - void *ptr; -} debug_pages_allocator_space; - -LowLevelAlloc::PagesAllocator *LowLevelAlloc::GetDefaultPagesAllocator(void) { - if (default_pages_allocator) { - return default_pages_allocator; - } - default_pages_allocator = new (debug_pages_allocator_space.chars) DefaultPagesAllocator(); - return default_pages_allocator; -} - -void *DefaultPagesAllocator::MapPages(int32 flags, size_t size) { - void *new_pages; - if ((flags & LowLevelAlloc::kAsyncSignalSafe) != 0) { - new_pages = MallocHook::UnhookedMMap(0, size, - PROT_WRITE|PROT_READ, - MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - } else { - new_pages = mmap(0, size, - PROT_WRITE|PROT_READ, - MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - } - RAW_CHECK(new_pages != MAP_FAILED, "mmap error"); - - return new_pages; -} - -void DefaultPagesAllocator::UnMapPages(int32 flags, void *region, size_t size) { - int munmap_result; - if ((flags & LowLevelAlloc::kAsyncSignalSafe) == 0) { - munmap_result = munmap(region, size); - } else { - munmap_result = MallocHook::UnhookedMUnmap(region, size); - } - RAW_CHECK(munmap_result == 0, - "LowLevelAlloc::DeleteArena: munmap failed address"); -} diff --git a/contrib/libtcmalloc/src/base/low_level_alloc.h b/contrib/libtcmalloc/src/base/low_level_alloc.h deleted file mode 100644 index 8a20dd8b870..00000000000 --- a/contrib/libtcmalloc/src/base/low_level_alloc.h +++ /dev/null @@ -1,120 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2006, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#if !defined(_BASE_LOW_LEVEL_ALLOC_H_) -#define _BASE_LOW_LEVEL_ALLOC_H_ - -// A simple thread-safe memory allocator that does not depend on -// mutexes or thread-specific data. It is intended to be used -// sparingly, and only when malloc() would introduce an unwanted -// dependency, such as inside the heap-checker. - -#include "../config.h" -#include // for size_t -#include "base/basictypes.h" - -class LowLevelAlloc { - public: - class PagesAllocator { - public: - virtual ~PagesAllocator(); - virtual void *MapPages(int32 flags, size_t size) = 0; - virtual void UnMapPages(int32 flags, void *addr, size_t size) = 0; - }; - - static PagesAllocator *GetDefaultPagesAllocator(void); - - struct Arena; // an arena from which memory may be allocated - - // Returns a pointer to a block of at least "request" bytes - // that have been newly allocated from the specific arena. - // for Alloc() call the DefaultArena() is used. - // Returns 0 if passed request==0. - // Does not return 0 under other circumstances; it crashes if memory - // is not available. - static void *Alloc(size_t request) - ATTRIBUTE_SECTION(malloc_hook); - static void *AllocWithArena(size_t request, Arena *arena) - ATTRIBUTE_SECTION(malloc_hook); - - // Deallocates a region of memory that was previously allocated with - // Alloc(). Does nothing if passed 0. "s" must be either 0, - // or must have been returned from a call to Alloc() and not yet passed to - // Free() since that call to Alloc(). The space is returned to the arena - // from which it was allocated. - static void Free(void *s) ATTRIBUTE_SECTION(malloc_hook); - - // ATTRIBUTE_SECTION(malloc_hook) for Alloc* and Free - // are to put all callers of MallocHook::Invoke* in this module - // into special section, - // so that MallocHook::GetCallerStackTrace can function accurately. - - // Create a new arena. - // The root metadata for the new arena is allocated in the - // meta_data_arena; the DefaultArena() can be passed for meta_data_arena. - // These values may be ored into flags: - enum { - // Report calls to Alloc() and Free() via the MallocHook interface. - // Set in the DefaultArena. - kCallMallocHook = 0x0001, - - // Make calls to Alloc(), Free() be async-signal-safe. Not set in - // DefaultArena(). - kAsyncSignalSafe = 0x0002, - - // When used with DefaultArena(), the NewArena() and DeleteArena() calls - // obey the flags given explicitly in the NewArena() call, even if those - // flags differ from the settings in DefaultArena(). So the call - // NewArena(kAsyncSignalSafe, DefaultArena()) is itself async-signal-safe, - // as well as generatating an arena that provides async-signal-safe - // Alloc/Free. - }; - static Arena *NewArena(int32 flags, Arena *meta_data_arena); - - // note: pages allocator will never be destroyed and allocated pages will never be freed - // When allocator is NULL, it's same as NewArena - static Arena *NewArenaWithCustomAlloc(int32 flags, Arena *meta_data_arena, PagesAllocator *allocator); - - // Destroys an arena allocated by NewArena and returns true, - // provided no allocated blocks remain in the arena. - // If allocated blocks remain in the arena, does nothing and - // returns false. - // It is illegal to attempt to destroy the DefaultArena(). - static bool DeleteArena(Arena *arena); - - // The default arena that always exists. - static Arena *DefaultArena(); - - private: - LowLevelAlloc(); // no instances -}; - -#endif diff --git a/contrib/libtcmalloc/src/base/simple_mutex.h b/contrib/libtcmalloc/src/base/simple_mutex.h deleted file mode 100644 index e57c1079283..00000000000 --- a/contrib/libtcmalloc/src/base/simple_mutex.h +++ /dev/null @@ -1,332 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2007, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// --- -// Author: Craig Silverstein. -// -// A simple mutex wrapper, supporting locks and read-write locks. -// You should assume the locks are *not* re-entrant. -// -// To use: you should define the following macros in your configure.ac: -// ACX_PTHREAD -// AC_RWLOCK -// The latter is defined in ../autoconf. -// -// This class is meant to be internal-only and should be wrapped by an -// internal namespace. Before you use this module, please give the -// name of your internal namespace for this module. Or, if you want -// to expose it, you'll want to move it to the Google namespace. We -// cannot put this class in global namespace because there can be some -// problems when we have multiple versions of Mutex in each shared object. -// -// NOTE: TryLock() is broken for NO_THREADS mode, at least in NDEBUG -// mode. -// -// CYGWIN NOTE: Cygwin support for rwlock seems to be buggy: -// http://www.cygwin.com/ml/cygwin/2008-12/msg00017.html -// Because of that, we might as well use windows locks for -// cygwin. They seem to be more reliable than the cygwin pthreads layer. -// -// TRICKY IMPLEMENTATION NOTE: -// This class is designed to be safe to use during -// dynamic-initialization -- that is, by global constructors that are -// run before main() starts. The issue in this case is that -// dynamic-initialization happens in an unpredictable order, and it -// could be that someone else's dynamic initializer could call a -// function that tries to acquire this mutex -- but that all happens -// before this mutex's constructor has run. (This can happen even if -// the mutex and the function that uses the mutex are in the same .cc -// file.) Basically, because Mutex does non-trivial work in its -// constructor, it's not, in the naive implementation, safe to use -// before dynamic initialization has run on it. -// -// The solution used here is to pair the actual mutex primitive with a -// bool that is set to true when the mutex is dynamically initialized. -// (Before that it's false.) Then we modify all mutex routines to -// look at the bool, and not try to lock/unlock until the bool makes -// it to true (which happens after the Mutex constructor has run.) -// -// This works because before main() starts -- particularly, during -// dynamic initialization -- there are no threads, so a) it's ok that -// the mutex operations are a no-op, since we don't need locking then -// anyway; and b) we can be quite confident our bool won't change -// state between a call to Lock() and a call to Unlock() (that would -// require a global constructor in one translation unit to call Lock() -// and another global constructor in another translation unit to call -// Unlock() later, which is pretty perverse). -// -// That said, it's tricky, and can conceivably fail; it's safest to -// avoid trying to acquire a mutex in a global constructor, if you -// can. One way it can fail is that a really smart compiler might -// initialize the bool to true at static-initialization time (too -// early) rather than at dynamic-initialization time. To discourage -// that, we set is_safe_ to true in code (not the constructor -// colon-initializer) and set it to true via a function that always -// evaluates to true, but that the compiler can't know always -// evaluates to true. This should be good enough. -// -// A related issue is code that could try to access the mutex -// after it's been destroyed in the global destructors (because -// the Mutex global destructor runs before some other global -// destructor, that tries to acquire the mutex). The way we -// deal with this is by taking a constructor arg that global -// mutexes should pass in, that causes the destructor to do no -// work. We still depend on the compiler not doing anything -// weird to a Mutex's memory after it is destroyed, but for a -// static global variable, that's pretty safe. - -#ifndef GOOGLE_MUTEX_H_ -#define GOOGLE_MUTEX_H_ - -#include "../config.h" - -#if defined(NO_THREADS) - typedef int MutexType; // to keep a lock-count -#elif defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) -# ifndef WIN32_LEAN_AND_MEAN -# define WIN32_LEAN_AND_MEAN // We only need minimal includes -# endif - // We need Windows NT or later for TryEnterCriticalSection(). If you - // don't need that functionality, you can remove these _WIN32_WINNT - // lines, and change TryLock() to assert(0) or something. -# ifndef _WIN32_WINNT -# define _WIN32_WINNT 0x0400 -# endif -# include - typedef CRITICAL_SECTION MutexType; -#elif defined(HAVE_PTHREAD) && defined(HAVE_RWLOCK) - // Needed for pthread_rwlock_*. If it causes problems, you could take it - // out, but then you'd have to unset HAVE_RWLOCK (at least on linux -- it - // *does* cause problems for FreeBSD, or MacOSX, but isn't needed - // for locking there.) -# ifdef __linux__ -# define _XOPEN_SOURCE 500 // may be needed to get the rwlock calls -# endif -# include - typedef pthread_rwlock_t MutexType; -#elif defined(HAVE_PTHREAD) -# include - typedef pthread_mutex_t MutexType; -#else -# error Need to implement mutex.h for your architecture, or #define NO_THREADS -#endif - -#include -#include // for abort() - -#define MUTEX_NAMESPACE perftools_mutex_namespace - -namespace MUTEX_NAMESPACE { - -class Mutex { - public: - // This is used for the single-arg constructor - enum LinkerInitialized { LINKER_INITIALIZED }; - - // Create a Mutex that is not held by anybody. This constructor is - // typically used for Mutexes allocated on the heap or the stack. - inline Mutex(); - // This constructor should be used for global, static Mutex objects. - // It inhibits work being done by the destructor, which makes it - // safer for code that tries to acqiure this mutex in their global - // destructor. - inline Mutex(LinkerInitialized); - - // Destructor - inline ~Mutex(); - - inline void Lock(); // Block if needed until free then acquire exclusively - inline void Unlock(); // Release a lock acquired via Lock() - inline bool TryLock(); // If free, Lock() and return true, else return false - // Note that on systems that don't support read-write locks, these may - // be implemented as synonyms to Lock() and Unlock(). So you can use - // these for efficiency, but don't use them anyplace where being able - // to do shared reads is necessary to avoid deadlock. - inline void ReaderLock(); // Block until free or shared then acquire a share - inline void ReaderUnlock(); // Release a read share of this Mutex - inline void WriterLock() { Lock(); } // Acquire an exclusive lock - inline void WriterUnlock() { Unlock(); } // Release a lock from WriterLock() - - private: - MutexType mutex_; - // We want to make sure that the compiler sets is_safe_ to true only - // when we tell it to, and never makes assumptions is_safe_ is - // always true. volatile is the most reliable way to do that. - volatile bool is_safe_; - // This indicates which constructor was called. - bool destroy_; - - inline void SetIsSafe() { is_safe_ = true; } - - // Catch the error of writing Mutex when intending MutexLock. - Mutex(Mutex* /*ignored*/) {} - // Disallow "evil" constructors - Mutex(const Mutex&); - void operator=(const Mutex&); -}; - -// Now the implementation of Mutex for various systems -#if defined(NO_THREADS) - -// When we don't have threads, we can be either reading or writing, -// but not both. We can have lots of readers at once (in no-threads -// mode, that's most likely to happen in recursive function calls), -// but only one writer. We represent this by having mutex_ be -1 when -// writing and a number > 0 when reading (and 0 when no lock is held). -// -// In debug mode, we assert these invariants, while in non-debug mode -// we do nothing, for efficiency. That's why everything is in an -// assert. - -Mutex::Mutex() : mutex_(0) { } -Mutex::Mutex(Mutex::LinkerInitialized) : mutex_(0) { } -Mutex::~Mutex() { assert(mutex_ == 0); } -void Mutex::Lock() { assert(--mutex_ == -1); } -void Mutex::Unlock() { assert(mutex_++ == -1); } -bool Mutex::TryLock() { if (mutex_) return false; Lock(); return true; } -void Mutex::ReaderLock() { assert(++mutex_ > 0); } -void Mutex::ReaderUnlock() { assert(mutex_-- > 0); } - -#elif defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) - -Mutex::Mutex() : destroy_(true) { - InitializeCriticalSection(&mutex_); - SetIsSafe(); -} -Mutex::Mutex(LinkerInitialized) : destroy_(false) { - InitializeCriticalSection(&mutex_); - SetIsSafe(); -} -Mutex::~Mutex() { if (destroy_) DeleteCriticalSection(&mutex_); } -void Mutex::Lock() { if (is_safe_) EnterCriticalSection(&mutex_); } -void Mutex::Unlock() { if (is_safe_) LeaveCriticalSection(&mutex_); } -bool Mutex::TryLock() { return is_safe_ ? - TryEnterCriticalSection(&mutex_) != 0 : true; } -void Mutex::ReaderLock() { Lock(); } // we don't have read-write locks -void Mutex::ReaderUnlock() { Unlock(); } - -#elif defined(HAVE_PTHREAD) && defined(HAVE_RWLOCK) - -#define SAFE_PTHREAD(fncall) do { /* run fncall if is_safe_ is true */ \ - if (is_safe_ && fncall(&mutex_) != 0) abort(); \ -} while (0) - -Mutex::Mutex() : destroy_(true) { - SetIsSafe(); - if (is_safe_ && pthread_rwlock_init(&mutex_, NULL) != 0) abort(); -} -Mutex::Mutex(Mutex::LinkerInitialized) : destroy_(false) { - SetIsSafe(); - if (is_safe_ && pthread_rwlock_init(&mutex_, NULL) != 0) abort(); -} -Mutex::~Mutex() { if (destroy_) SAFE_PTHREAD(pthread_rwlock_destroy); } -void Mutex::Lock() { SAFE_PTHREAD(pthread_rwlock_wrlock); } -void Mutex::Unlock() { SAFE_PTHREAD(pthread_rwlock_unlock); } -bool Mutex::TryLock() { return is_safe_ ? - pthread_rwlock_trywrlock(&mutex_) == 0 : true; } -void Mutex::ReaderLock() { SAFE_PTHREAD(pthread_rwlock_rdlock); } -void Mutex::ReaderUnlock() { SAFE_PTHREAD(pthread_rwlock_unlock); } -#undef SAFE_PTHREAD - -#elif defined(HAVE_PTHREAD) - -#define SAFE_PTHREAD(fncall) do { /* run fncall if is_safe_ is true */ \ - if (is_safe_ && fncall(&mutex_) != 0) abort(); \ -} while (0) - -Mutex::Mutex() : destroy_(true) { - SetIsSafe(); - if (is_safe_ && pthread_mutex_init(&mutex_, NULL) != 0) abort(); -} -Mutex::Mutex(Mutex::LinkerInitialized) : destroy_(false) { - SetIsSafe(); - if (is_safe_ && pthread_mutex_init(&mutex_, NULL) != 0) abort(); -} -Mutex::~Mutex() { if (destroy_) SAFE_PTHREAD(pthread_mutex_destroy); } -void Mutex::Lock() { SAFE_PTHREAD(pthread_mutex_lock); } -void Mutex::Unlock() { SAFE_PTHREAD(pthread_mutex_unlock); } -bool Mutex::TryLock() { return is_safe_ ? - pthread_mutex_trylock(&mutex_) == 0 : true; } -void Mutex::ReaderLock() { Lock(); } -void Mutex::ReaderUnlock() { Unlock(); } -#undef SAFE_PTHREAD - -#endif - -// -------------------------------------------------------------------------- -// Some helper classes - -// MutexLock(mu) acquires mu when constructed and releases it when destroyed. -class MutexLock { - public: - explicit MutexLock(Mutex *mu) : mu_(mu) { mu_->Lock(); } - ~MutexLock() { mu_->Unlock(); } - private: - Mutex * const mu_; - // Disallow "evil" constructors - MutexLock(const MutexLock&); - void operator=(const MutexLock&); -}; - -// ReaderMutexLock and WriterMutexLock do the same, for rwlocks -class ReaderMutexLock { - public: - explicit ReaderMutexLock(Mutex *mu) : mu_(mu) { mu_->ReaderLock(); } - ~ReaderMutexLock() { mu_->ReaderUnlock(); } - private: - Mutex * const mu_; - // Disallow "evil" constructors - ReaderMutexLock(const ReaderMutexLock&); - void operator=(const ReaderMutexLock&); -}; - -class WriterMutexLock { - public: - explicit WriterMutexLock(Mutex *mu) : mu_(mu) { mu_->WriterLock(); } - ~WriterMutexLock() { mu_->WriterUnlock(); } - private: - Mutex * const mu_; - // Disallow "evil" constructors - WriterMutexLock(const WriterMutexLock&); - void operator=(const WriterMutexLock&); -}; - -// Catch bug where variable name is omitted, e.g. MutexLock (&mu); -#define MutexLock(x) COMPILE_ASSERT(0, mutex_lock_decl_missing_var_name) -#define ReaderMutexLock(x) COMPILE_ASSERT(0, rmutex_lock_decl_missing_var_name) -#define WriterMutexLock(x) COMPILE_ASSERT(0, wmutex_lock_decl_missing_var_name) - -} // namespace MUTEX_NAMESPACE - -using namespace MUTEX_NAMESPACE; - -#undef MUTEX_NAMESPACE - -#endif /* #define GOOGLE_SIMPLE_MUTEX_H_ */ diff --git a/contrib/libtcmalloc/src/base/spinlock.cc b/contrib/libtcmalloc/src/base/spinlock.cc deleted file mode 100644 index 48bb163d1de..00000000000 --- a/contrib/libtcmalloc/src/base/spinlock.cc +++ /dev/null @@ -1,129 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2006, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Sanjay Ghemawat - */ - -#include "../config.h" -#include "base/spinlock.h" -#include "base/spinlock_internal.h" -#include "base/sysinfo.h" /* for GetSystemCPUsCount() */ - -// NOTE on the Lock-state values: -// -// kSpinLockFree represents the unlocked state -// kSpinLockHeld represents the locked state with no waiters -// kSpinLockSleeper represents the locked state with waiters - -static int adaptive_spin_count = 0; - -const base::LinkerInitialized SpinLock::LINKER_INITIALIZED = - base::LINKER_INITIALIZED; - -namespace { -struct SpinLock_InitHelper { - SpinLock_InitHelper() { - // On multi-cpu machines, spin for longer before yielding - // the processor or sleeping. Reduces idle time significantly. - if (GetSystemCPUsCount() > 1) { - adaptive_spin_count = 1000; - } - } -}; - -// Hook into global constructor execution: -// We do not do adaptive spinning before that, -// but nothing lock-intensive should be going on at that time. -static SpinLock_InitHelper init_helper; - -inline void SpinlockPause(void) { -#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) - __asm__ __volatile__("rep; nop" : : ); -#endif -} - -} // unnamed namespace - -// Monitor the lock to see if its value changes within some time -// period (adaptive_spin_count loop iterations). The last value read -// from the lock is returned from the method. -Atomic32 SpinLock::SpinLoop() { - int c = adaptive_spin_count; - while (base::subtle::NoBarrier_Load(&lockword_) != kSpinLockFree && --c > 0) { - SpinlockPause(); - } - return base::subtle::Acquire_CompareAndSwap(&lockword_, kSpinLockFree, - kSpinLockSleeper); -} - -void SpinLock::SlowLock() { - Atomic32 lock_value = SpinLoop(); - - int lock_wait_call_count = 0; - while (lock_value != kSpinLockFree) { - // If the lock is currently held, but not marked as having a sleeper, mark - // it as having a sleeper. - if (lock_value == kSpinLockHeld) { - // Here, just "mark" that the thread is going to sleep. Don't store the - // lock wait time in the lock as that will cause the current lock - // owner to think it experienced contention. - lock_value = base::subtle::Acquire_CompareAndSwap(&lockword_, - kSpinLockHeld, - kSpinLockSleeper); - if (lock_value == kSpinLockHeld) { - // Successfully transitioned to kSpinLockSleeper. Pass - // kSpinLockSleeper to the SpinLockDelay routine to properly indicate - // the last lock_value observed. - lock_value = kSpinLockSleeper; - } else if (lock_value == kSpinLockFree) { - // Lock is free again, so try and acquire it before sleeping. The - // new lock state will be the number of cycles this thread waited if - // this thread obtains the lock. - lock_value = base::subtle::Acquire_CompareAndSwap(&lockword_, - kSpinLockFree, - kSpinLockSleeper); - continue; // skip the delay at the end of the loop - } - } - - // Wait for an OS specific delay. - base::internal::SpinLockDelay(&lockword_, lock_value, - ++lock_wait_call_count); - // Spin again after returning from the wait routine to give this thread - // some chance of obtaining the lock. - lock_value = SpinLoop(); - } -} - -void SpinLock::SlowUnlock() { - // wake waiter if necessary - base::internal::SpinLockWake(&lockword_, false); -} diff --git a/contrib/libtcmalloc/src/base/spinlock.h b/contrib/libtcmalloc/src/base/spinlock.h deleted file mode 100644 index 42a4eb906a0..00000000000 --- a/contrib/libtcmalloc/src/base/spinlock.h +++ /dev/null @@ -1,143 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2006, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Sanjay Ghemawat - */ - -// SpinLock is async signal safe. -// If used within a signal handler, all lock holders -// should block the signal even outside the signal handler. - -#ifndef BASE_SPINLOCK_H_ -#define BASE_SPINLOCK_H_ - -#include "../config.h" -#include "base/atomicops.h" -#include "base/basictypes.h" -#include "base/dynamic_annotations.h" -#include "base/thread_annotations.h" - -class LOCKABLE SpinLock { - public: - SpinLock() : lockword_(kSpinLockFree) { } - - // Special constructor for use with static SpinLock objects. E.g., - // - // static SpinLock lock(base::LINKER_INITIALIZED); - // - // When intialized using this constructor, we depend on the fact - // that the linker has already initialized the memory appropriately. - // A SpinLock constructed like this can be freely used from global - // initializers without worrying about the order in which global - // initializers run. - explicit SpinLock(base::LinkerInitialized /*x*/) { - // Does nothing; lockword_ is already initialized - } - - // Acquire this SpinLock. - // TODO(csilvers): uncomment the annotation when we figure out how to - // support this macro with 0 args (see thread_annotations.h) - inline void Lock() /*EXCLUSIVE_LOCK_FUNCTION()*/ { - if (base::subtle::Acquire_CompareAndSwap(&lockword_, kSpinLockFree, - kSpinLockHeld) != kSpinLockFree) { - SlowLock(); - } - ANNOTATE_RWLOCK_ACQUIRED(this, 1); - } - - // Try to acquire this SpinLock without blocking and return true if the - // acquisition was successful. If the lock was not acquired, false is - // returned. If this SpinLock is free at the time of the call, TryLock - // will return true with high probability. - inline bool TryLock() EXCLUSIVE_TRYLOCK_FUNCTION(true) { - bool res = - (base::subtle::Acquire_CompareAndSwap(&lockword_, kSpinLockFree, - kSpinLockHeld) == kSpinLockFree); - if (res) { - ANNOTATE_RWLOCK_ACQUIRED(this, 1); - } - return res; - } - - // Release this SpinLock, which must be held by the calling thread. - // TODO(csilvers): uncomment the annotation when we figure out how to - // support this macro with 0 args (see thread_annotations.h) - inline void Unlock() /*UNLOCK_FUNCTION()*/ { - ANNOTATE_RWLOCK_RELEASED(this, 1); - uint64 prev_value = static_cast( - base::subtle::Release_AtomicExchange(&lockword_, kSpinLockFree)); - if (prev_value != kSpinLockHeld) { - // Speed the wakeup of any waiter. - SlowUnlock(); - } - } - - // Determine if the lock is held. When the lock is held by the invoking - // thread, true will always be returned. Intended to be used as - // CHECK(lock.IsHeld()). - inline bool IsHeld() const { - return base::subtle::NoBarrier_Load(&lockword_) != kSpinLockFree; - } - - static const base::LinkerInitialized LINKER_INITIALIZED; // backwards compat - private: - enum { kSpinLockFree = 0 }; - enum { kSpinLockHeld = 1 }; - enum { kSpinLockSleeper = 2 }; - - volatile Atomic32 lockword_; - - void SlowLock(); - void SlowUnlock(); - Atomic32 SpinLoop(); - - DISALLOW_COPY_AND_ASSIGN(SpinLock); -}; - -// Corresponding locker object that arranges to acquire a spinlock for -// the duration of a C++ scope. -class SCOPED_LOCKABLE SpinLockHolder { - private: - SpinLock* lock_; - public: - inline explicit SpinLockHolder(SpinLock* l) EXCLUSIVE_LOCK_FUNCTION(l) - : lock_(l) { - l->Lock(); - } - // TODO(csilvers): uncomment the annotation when we figure out how to - // support this macro with 0 args (see thread_annotations.h) - inline ~SpinLockHolder() /*UNLOCK_FUNCTION()*/ { lock_->Unlock(); } -}; -// Catch bug where variable name is omitted, e.g. SpinLockHolder (&lock); -#define SpinLockHolder(x) COMPILE_ASSERT(0, spin_lock_decl_missing_var_name) - - -#endif // BASE_SPINLOCK_H_ diff --git a/contrib/libtcmalloc/src/base/spinlock_internal.cc b/contrib/libtcmalloc/src/base/spinlock_internal.cc deleted file mode 100644 index d9629717be1..00000000000 --- a/contrib/libtcmalloc/src/base/spinlock_internal.cc +++ /dev/null @@ -1,102 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2010, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -// The OS-specific header included below must provide two calls: -// base::internal::SpinLockDelay() and base::internal::SpinLockWake(). -// See spinlock_internal.h for the spec of SpinLockWake(). - -// void SpinLockDelay(volatile Atomic32 *w, int32 value, int loop) -// SpinLockDelay() generates an apprproate spin delay on iteration "loop" of a -// spin loop on location *w, whose previously observed value was "value". -// SpinLockDelay() may do nothing, may yield the CPU, may sleep a clock tick, -// or may wait for a delay that can be truncated by a call to SpinlockWake(w). -// In all cases, it must return in bounded time even if SpinlockWake() is not -// called. - -#include "base/spinlock_internal.h" - -// forward declaration for use by spinlock_*-inl.h -namespace base { namespace internal { static int SuggestedDelayNS(int loop); }} - -#if defined(_WIN32) -#include "base/spinlock_win32-inl.h" -#elif defined(__linux__) -#include "base/spinlock_linux-inl.h" -#else -#include "base/spinlock_posix-inl.h" -#endif - -namespace base { -namespace internal { - -// Return a suggested delay in nanoseconds for iteration number "loop" -static int SuggestedDelayNS(int loop) { - // Weak pseudo-random number generator to get some spread between threads - // when many are spinning. -#ifdef BASE_HAS_ATOMIC64 - static base::subtle::Atomic64 rand; - uint64 r = base::subtle::NoBarrier_Load(&rand); - r = 0x5deece66dLL * r + 0xb; // numbers from nrand48() - base::subtle::NoBarrier_Store(&rand, r); - - r <<= 16; // 48-bit random number now in top 48-bits. - if (loop < 0 || loop > 32) { // limit loop to 0..32 - loop = 32; - } - // loop>>3 cannot exceed 4 because loop cannot exceed 32. - // Select top 20..24 bits of lower 48 bits, - // giving approximately 0ms to 16ms. - // Mean is exponential in loop for first 32 iterations, then 8ms. - // The futex path multiplies this by 16, since we expect explicit wakeups - // almost always on that path. - return r >> (44 - (loop >> 3)); -#else - static Atomic32 rand; - uint32 r = base::subtle::NoBarrier_Load(&rand); - r = 0x343fd * r + 0x269ec3; // numbers from MSVC++ - base::subtle::NoBarrier_Store(&rand, r); - - r <<= 1; // 31-bit random number now in top 31-bits. - if (loop < 0 || loop > 32) { // limit loop to 0..32 - loop = 32; - } - // loop>>3 cannot exceed 4 because loop cannot exceed 32. - // Select top 20..24 bits of lower 31 bits, - // giving approximately 0ms to 16ms. - // Mean is exponential in loop for first 32 iterations, then 8ms. - // The futex path multiplies this by 16, since we expect explicit wakeups - // almost always on that path. - return r >> (12 - (loop >> 3)); -#endif -} - -} // namespace internal -} // namespace base diff --git a/contrib/libtcmalloc/src/base/spinlock_internal.h b/contrib/libtcmalloc/src/base/spinlock_internal.h deleted file mode 100644 index 636885cd6e5..00000000000 --- a/contrib/libtcmalloc/src/base/spinlock_internal.h +++ /dev/null @@ -1,51 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2010, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * This file is an internal part spinlock.cc and once.cc - * It may not be used directly by code outside of //base. - */ - -#ifndef BASE_SPINLOCK_INTERNAL_H_ -#define BASE_SPINLOCK_INTERNAL_H_ - -#include "../config.h" -#include "base/basictypes.h" -#include "base/atomicops.h" - -namespace base { -namespace internal { - -void SpinLockWake(volatile Atomic32 *w, bool all); -void SpinLockDelay(volatile Atomic32 *w, int32 value, int loop); - -} // namespace internal -} // namespace base -#endif diff --git a/contrib/libtcmalloc/src/base/spinlock_linux-inl.h b/contrib/libtcmalloc/src/base/spinlock_linux-inl.h deleted file mode 100644 index aadf62a4b67..00000000000 --- a/contrib/libtcmalloc/src/base/spinlock_linux-inl.h +++ /dev/null @@ -1,101 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2009, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * This file is a Linux-specific part of spinlock_internal.cc - */ - -#include -#include -#include -#include -#include "base/linux_syscall_support.h" - -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 -#define FUTEX_PRIVATE_FLAG 128 - -static bool have_futex; -static int futex_private_flag = FUTEX_PRIVATE_FLAG; - -namespace { -static struct InitModule { - InitModule() { - int x = 0; - // futexes are ints, so we can use them only when - // that's the same size as the lockword_ in SpinLock. - have_futex = (sizeof (Atomic32) == sizeof (int) && - sys_futex(&x, FUTEX_WAKE, 1, NULL, NULL, 0) >= 0); - if (have_futex && - sys_futex(&x, FUTEX_WAKE | futex_private_flag, 1, NULL, NULL, 0) < 0) { - futex_private_flag = 0; - } - } -} init_module; - -} // anonymous namespace - - -namespace base { -namespace internal { - -void SpinLockDelay(volatile Atomic32 *w, int32 value, int loop) { - if (loop != 0) { - int save_errno = errno; - struct timespec tm; - tm.tv_sec = 0; - if (have_futex) { - tm.tv_nsec = base::internal::SuggestedDelayNS(loop); - } else { - tm.tv_nsec = 2000001; // above 2ms so linux 2.4 doesn't spin - } - if (have_futex) { - tm.tv_nsec *= 16; // increase the delay; we expect explicit wakeups - sys_futex(reinterpret_cast(const_cast(w)), - FUTEX_WAIT | futex_private_flag, - value, reinterpret_cast(&tm), - NULL, 0); - } else { - nanosleep(&tm, NULL); - } - errno = save_errno; - } -} - -void SpinLockWake(volatile Atomic32 *w, bool all) { - if (have_futex) { - sys_futex(reinterpret_cast(const_cast(w)), - FUTEX_WAKE | futex_private_flag, all? INT_MAX : 1, - NULL, NULL, 0); - } -} - -} // namespace internal -} // namespace base diff --git a/contrib/libtcmalloc/src/base/spinlock_posix-inl.h b/contrib/libtcmalloc/src/base/spinlock_posix-inl.h deleted file mode 100644 index 2695b7b1bb9..00000000000 --- a/contrib/libtcmalloc/src/base/spinlock_posix-inl.h +++ /dev/null @@ -1,63 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2009, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * This file is a Posix-specific part of spinlock_internal.cc - */ - -#include "../config.h" -#include -#ifdef HAVE_SCHED_H -#include /* For sched_yield() */ -#endif -#include /* For nanosleep() */ - -namespace base { -namespace internal { - -void SpinLockDelay(volatile Atomic32 *w, int32 value, int loop) { - int save_errno = errno; - if (loop == 0) { - } else if (loop == 1) { - sched_yield(); - } else { - struct timespec tm; - tm.tv_sec = 0; - tm.tv_nsec = base::internal::SuggestedDelayNS(loop); - nanosleep(&tm, NULL); - } - errno = save_errno; -} - -void SpinLockWake(volatile Atomic32 *w, bool all) { -} - -} // namespace internal -} // namespace base diff --git a/contrib/libtcmalloc/src/base/spinlock_win32-inl.h b/contrib/libtcmalloc/src/base/spinlock_win32-inl.h deleted file mode 100644 index 956b9653e6d..00000000000 --- a/contrib/libtcmalloc/src/base/spinlock_win32-inl.h +++ /dev/null @@ -1,54 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2009, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * This file is a Win32-specific part of spinlock_internal.cc - */ - - -#include - -namespace base { -namespace internal { - -void SpinLockDelay(volatile Atomic32 *w, int32 value, int loop) { - if (loop == 0) { - } else if (loop == 1) { - Sleep(0); - } else { - Sleep(base::internal::SuggestedDelayNS(loop) / 1000000); - } -} - -void SpinLockWake(volatile Atomic32 *w, bool all) { -} - -} // namespace internal -} // namespace base diff --git a/contrib/libtcmalloc/src/base/stl_allocator.h b/contrib/libtcmalloc/src/base/stl_allocator.h deleted file mode 100644 index 4520713622f..00000000000 --- a/contrib/libtcmalloc/src/base/stl_allocator.h +++ /dev/null @@ -1,98 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2006, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Maxim Lifantsev - */ - - -#ifndef BASE_STL_ALLOCATOR_H_ -#define BASE_STL_ALLOCATOR_H_ - -#include "../config.h" - -#include // for ptrdiff_t -#include - -#include "base/logging.h" - -// Generic allocator class for STL objects -// that uses a given type-less allocator Alloc, which must provide: -// static void* Alloc::Allocate(size_t size); -// static void Alloc::Free(void* ptr, size_t size); -// -// STL_Allocator provides the same thread-safety -// guarantees as MyAlloc. -// -// Usage example: -// set, STL_Allocator > my_set; -// CAVEAT: Parts of the code below are probably specific -// to the STL version(s) we are using. -// The code is simply lifted from what std::allocator<> provides. -template -class STL_Allocator { - public: - typedef size_t size_type; - typedef ptrdiff_t difference_type; - typedef T* pointer; - typedef const T* const_pointer; - typedef T& reference; - typedef const T& const_reference; - typedef T value_type; - - template struct rebind { - typedef STL_Allocator other; - }; - - STL_Allocator() { } - STL_Allocator(const STL_Allocator&) { } - template STL_Allocator(const STL_Allocator&) { } - ~STL_Allocator() { } - - pointer address(reference x) const { return &x; } - const_pointer address(const_reference x) const { return &x; } - - pointer allocate(size_type n, const void* = 0) { - RAW_DCHECK((n * sizeof(T)) / sizeof(T) == n, "n is too big to allocate"); - return static_cast(Alloc::Allocate(n * sizeof(T))); - } - void deallocate(pointer p, size_type n) { Alloc::Free(p, n * sizeof(T)); } - - size_type max_size() const { return size_t(-1) / sizeof(T); } - - void construct(pointer p, const T& val) { ::new(p) T(val); } - void construct(pointer p) { ::new(p) T(); } - void destroy(pointer p) { p->~T(); } - - // There's no state, so these allocators are always equal - bool operator==(const STL_Allocator&) const { return true; } -}; - -#endif // BASE_STL_ALLOCATOR_H_ diff --git a/contrib/libtcmalloc/src/base/sysinfo.cc b/contrib/libtcmalloc/src/base/sysinfo.cc deleted file mode 100644 index 75217e6795a..00000000000 --- a/contrib/libtcmalloc/src/base/sysinfo.cc +++ /dev/null @@ -1,860 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2006, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include "../config.h" -#if (defined(_WIN32) || defined(__MINGW32__)) && !defined(__CYGWIN__) && !defined(__CYGWIN32) -# define PLATFORM_WINDOWS 1 -#endif - -#include // for isspace() -#include // for getenv() -#include // for snprintf(), sscanf() -#include // for memmove(), memchr(), etc. -#include // for open() -#include // for errno -#ifdef HAVE_UNISTD_H -#include // for read() -#endif -#if defined __MACH__ // Mac OS X, almost certainly -#include // for iterating over dll's in ProcMapsIter -#include // for iterating over dll's in ProcMapsIter -#include -#include // how we figure out numcpu's on OS X -#elif defined __FreeBSD__ -#include -#elif defined __sun__ // Solaris -#include // for, e.g., prmap_t -#elif defined(PLATFORM_WINDOWS) -#include // for getpid() (actually, _getpid()) -#include // for SHGetValueA() -#include // for Module32First() -#endif -#include "base/sysinfo.h" -#include "base/commandlineflags.h" -#include "base/dynamic_annotations.h" // for RunningOnValgrind -#include "base/logging.h" - -#ifdef PLATFORM_WINDOWS -#ifdef MODULEENTRY32 -// In a change from the usual W-A pattern, there is no A variant of -// MODULEENTRY32. Tlhelp32.h #defines the W variant, but not the A. -// In unicode mode, tlhelp32.h #defines MODULEENTRY32 to be -// MODULEENTRY32W. These #undefs are the only way I see to get back -// access to the original, ascii struct (and related functions). -#undef MODULEENTRY32 -#undef Module32First -#undef Module32Next -#undef PMODULEENTRY32 -#undef LPMODULEENTRY32 -#endif /* MODULEENTRY32 */ -// MinGW doesn't seem to define this, perhaps some windowsen don't either. -#ifndef TH32CS_SNAPMODULE32 -#define TH32CS_SNAPMODULE32 0 -#endif /* TH32CS_SNAPMODULE32 */ -#endif /* PLATFORM_WINDOWS */ - -// Re-run fn until it doesn't cause EINTR. -#define NO_INTR(fn) do {} while ((fn) < 0 && errno == EINTR) - -// open/read/close can set errno, which may be illegal at this -// time, so prefer making the syscalls directly if we can. -#ifdef HAVE_SYS_SYSCALL_H -# include -#endif -#ifdef SYS_open // solaris 11, at least sometimes, only defines SYS_openat -# define safeopen(filename, mode) syscall(SYS_open, filename, mode) -#else -# define safeopen(filename, mode) open(filename, mode) -#endif -#ifdef SYS_read -# define saferead(fd, buffer, size) syscall(SYS_read, fd, buffer, size) -#else -# define saferead(fd, buffer, size) read(fd, buffer, size) -#endif -#ifdef SYS_close -# define safeclose(fd) syscall(SYS_close, fd) -#else -# define safeclose(fd) close(fd) -#endif - -// ---------------------------------------------------------------------- -// GetenvBeforeMain() -// GetUniquePathFromEnv() -// Some non-trivial getenv-related functions. -// ---------------------------------------------------------------------- - -// It's not safe to call getenv() in the malloc hooks, because they -// might be called extremely early, before libc is done setting up -// correctly. In particular, the thread library may not be done -// setting up errno. So instead, we use the built-in __environ array -// if it exists, and otherwise read /proc/self/environ directly, using -// system calls to read the file, and thus avoid setting errno. -// /proc/self/environ has a limit of how much data it exports (around -// 8K), so it's not an ideal solution. -const char* GetenvBeforeMain(const char* name) { -#if defined(HAVE___ENVIRON) // if we have it, it's declared in unistd.h - if (__environ) { // can exist but be NULL, if statically linked - const int namelen = strlen(name); - for (char** p = __environ; *p; p++) { - if (strlen(*p) < namelen) { - continue; - } - if (!memcmp(*p, name, namelen) && (*p)[namelen] == '=') // it's a match - return *p + namelen+1; // point after = - } - return NULL; - } -#endif -#if defined(PLATFORM_WINDOWS) - // TODO(mbelshe) - repeated calls to this function will overwrite the - // contents of the static buffer. - static char envvar_buf[1024]; // enough to hold any envvar we care about - if (!GetEnvironmentVariableA(name, envvar_buf, sizeof(envvar_buf)-1)) - return NULL; - return envvar_buf; -#endif - // static is ok because this function should only be called before - // main(), when we're single-threaded. - static char envbuf[16<<10]; - if (*envbuf == '\0') { // haven't read the environ yet - int fd = safeopen("/proc/self/environ", O_RDONLY); - // The -2 below guarantees the last two bytes of the buffer will be \0\0 - if (fd == -1 || // unable to open the file, fall back onto libc - saferead(fd, envbuf, sizeof(envbuf) - 2) < 0) { // error reading file - RAW_VLOG(1, "Unable to open /proc/self/environ, falling back " - "on getenv(\"%s\"), which may not work", name); - if (fd != -1) safeclose(fd); - return getenv(name); - } - safeclose(fd); - } - const int namelen = strlen(name); - const char* p = envbuf; - while (*p != '\0') { // will happen at the \0\0 that terminates the buffer - // proc file has the format NAME=value\0NAME=value\0NAME=value\0... - const char* endp = (char*)memchr(p, '\0', sizeof(envbuf) - (p - envbuf)); - if (endp == NULL) // this entry isn't NUL terminated - return NULL; - else if (!memcmp(p, name, namelen) && p[namelen] == '=') // it's a match - return p + namelen+1; // point after = - p = endp + 1; - } - return NULL; // env var never found -} - -extern "C" { - const char* TCMallocGetenvSafe(const char* name) { - return GetenvBeforeMain(name); - } -} - -// This takes as an argument an environment-variable name (like -// CPUPROFILE) whose value is supposed to be a file-path, and sets -// path to that path, and returns true. If the env var doesn't exist, -// or is the empty string, leave path unchanged and returns false. -// The reason this is non-trivial is that this function handles munged -// pathnames. Here's why: -// -// If we're a child process of the 'main' process, we can't just use -// getenv("CPUPROFILE") -- the parent process will be using that path. -// Instead we append our pid to the pathname. How do we tell if we're a -// child process? Ideally we'd set an environment variable that all -// our children would inherit. But -- and this is seemingly a bug in -// gcc -- if you do a setenv() in a shared libarary in a global -// constructor, the environment setting is lost by the time main() is -// called. The only safe thing we can do in such a situation is to -// modify the existing envvar. So we do a hack: in the parent, we set -// the high bit of the 1st char of CPUPROFILE. In the child, we -// notice the high bit is set and append the pid(). This works -// assuming cpuprofile filenames don't normally have the high bit set -// in their first character! If that assumption is violated, we'll -// still get a profile, but one with an unexpected name. -// TODO(csilvers): set an envvar instead when we can do it reliably. -bool GetUniquePathFromEnv(const char* env_name, char* path) { - char* envval = getenv(env_name); - if (envval == NULL || *envval == '\0') - return false; - if (envval[0] & 128) { // high bit is set - snprintf(path, PATH_MAX, "%c%s_%u", // add pid and clear high bit - envval[0] & 127, envval+1, (unsigned int)(getpid())); - } else { - snprintf(path, PATH_MAX, "%s", envval); - envval[0] |= 128; // set high bit for kids to see - } - return true; -} - -void SleepForMilliseconds(int milliseconds) { -#ifdef PLATFORM_WINDOWS - _sleep(milliseconds); // Windows's _sleep takes milliseconds argument -#else - // Sleep for a few milliseconds - struct timespec sleep_time; - sleep_time.tv_sec = milliseconds / 1000; - sleep_time.tv_nsec = (milliseconds % 1000) * 1000000; - while (nanosleep(&sleep_time, &sleep_time) != 0 && errno == EINTR) - ; // Ignore signals and wait for the full interval to elapse. -#endif -} - -int GetSystemCPUsCount() -{ -#if defined(PLATFORM_WINDOWS) - // Get the number of processors. - SYSTEM_INFO info; - GetSystemInfo(&info); - return info.dwNumberOfProcessors; -#else - long rv = sysconf(_SC_NPROCESSORS_ONLN); - if (rv < 0) { - return 1; - } - return static_cast(rv); -#endif -} - -// ---------------------------------------------------------------------- - -#if defined __linux__ || defined __FreeBSD__ || defined __sun__ || defined __CYGWIN__ || defined __CYGWIN32__ -static void ConstructFilename(const char* spec, pid_t pid, - char* buf, int buf_size) { - CHECK_LT(snprintf(buf, buf_size, - spec, - static_cast(pid ? pid : getpid())), buf_size); -} -#endif - -// A templatized helper function instantiated for Mach (OS X) only. -// It can handle finding info for both 32 bits and 64 bits. -// Returns true if it successfully handled the hdr, false else. -#ifdef __MACH__ // Mac OS X, almost certainly -template -static bool NextExtMachHelper(const mach_header* hdr, - int current_image, int current_load_cmd, - uint64 *start, uint64 *end, char **flags, - uint64 *offset, int64 *inode, char **filename, - uint64 *file_mapping, uint64 *file_pages, - uint64 *anon_mapping, uint64 *anon_pages, - dev_t *dev) { - static char kDefaultPerms[5] = "r-xp"; - if (hdr->magic != kMagic) - return false; - const char* lc = (const char *)hdr + sizeof(MachHeader); - // TODO(csilvers): make this not-quadradic (increment and hold state) - for (int j = 0; j < current_load_cmd; j++) // advance to *our* load_cmd - lc += ((const load_command *)lc)->cmdsize; - if (((const load_command *)lc)->cmd == kLCSegment) { - const intptr_t dlloff = _dyld_get_image_vmaddr_slide(current_image); - const SegmentCommand* sc = (const SegmentCommand *)lc; - if (start) *start = sc->vmaddr + dlloff; - if (end) *end = sc->vmaddr + sc->vmsize + dlloff; - if (flags) *flags = kDefaultPerms; // can we do better? - if (offset) *offset = sc->fileoff; - if (inode) *inode = 0; - if (filename) - *filename = const_cast(_dyld_get_image_name(current_image)); - if (file_mapping) *file_mapping = 0; - if (file_pages) *file_pages = 0; // could we use sc->filesize? - if (anon_mapping) *anon_mapping = 0; - if (anon_pages) *anon_pages = 0; - if (dev) *dev = 0; - return true; - } - - return false; -} -#endif - -// Finds |c| in |text|, and assign '\0' at the found position. -// The original character at the modified position should be |c|. -// A pointer to the modified position is stored in |endptr|. -// |endptr| should not be NULL. -static bool ExtractUntilChar(char *text, int c, char **endptr) { - CHECK_NE(text, NULL); - CHECK_NE(endptr, NULL); - char *found; - found = strchr(text, c); - if (found == NULL) { - *endptr = NULL; - return false; - } - - *endptr = found; - *found = '\0'; - return true; -} - -// Increments |*text_pointer| while it points a whitespace character. -// It is to follow sscanf's whilespace handling. -static void SkipWhileWhitespace(char **text_pointer, int c) { - if (isspace(c)) { - while (isspace(**text_pointer) && isspace(*((*text_pointer) + 1))) { - ++(*text_pointer); - } - } -} - -template -static T StringToInteger(char *text, char **endptr, int base) { - assert(false); - return T(); -} - -template<> -int StringToInteger(char *text, char **endptr, int base) { - return strtol(text, endptr, base); -} - -template<> -int64 StringToInteger(char *text, char **endptr, int base) { - return strtoll(text, endptr, base); -} - -template<> -uint64 StringToInteger(char *text, char **endptr, int base) { - return strtoull(text, endptr, base); -} - -template -static T StringToIntegerUntilChar( - char *text, int base, int c, char **endptr_result) { - CHECK_NE(endptr_result, NULL); - *endptr_result = NULL; - - char *endptr_extract; - if (!ExtractUntilChar(text, c, &endptr_extract)) - return 0; - - T result; - char *endptr_strto; - result = StringToInteger(text, &endptr_strto, base); - *endptr_extract = c; - - if (endptr_extract != endptr_strto) - return 0; - - *endptr_result = endptr_extract; - SkipWhileWhitespace(endptr_result, c); - - return result; -} - -static char *CopyStringUntilChar( - char *text, unsigned out_len, int c, char *out) { - char *endptr; - if (!ExtractUntilChar(text, c, &endptr)) - return NULL; - - strncpy(out, text, out_len); - out[out_len-1] = '\0'; - *endptr = c; - - SkipWhileWhitespace(&endptr, c); - return endptr; -} - -template -static bool StringToIntegerUntilCharWithCheck( - T *outptr, char *text, int base, int c, char **endptr) { - *outptr = StringToIntegerUntilChar(*endptr, base, c, endptr); - if (*endptr == NULL || **endptr == '\0') return false; - ++(*endptr); - return true; -} - -static bool ParseProcMapsLine(char *text, uint64 *start, uint64 *end, - char *flags, uint64 *offset, - int *major, int *minor, int64 *inode, - unsigned *filename_offset) { -#if defined(__linux__) - /* - * It's similar to: - * sscanf(text, "%"SCNx64"-%"SCNx64" %4s %"SCNx64" %x:%x %"SCNd64" %n", - * start, end, flags, offset, major, minor, inode, filename_offset) - */ - char *endptr = text; - if (endptr == NULL || *endptr == '\0') return false; - - if (!StringToIntegerUntilCharWithCheck(start, endptr, 16, '-', &endptr)) - return false; - - if (!StringToIntegerUntilCharWithCheck(end, endptr, 16, ' ', &endptr)) - return false; - - endptr = CopyStringUntilChar(endptr, 5, ' ', flags); - if (endptr == NULL || *endptr == '\0') return false; - ++endptr; - - if (!StringToIntegerUntilCharWithCheck(offset, endptr, 16, ' ', &endptr)) - return false; - - if (!StringToIntegerUntilCharWithCheck(major, endptr, 16, ':', &endptr)) - return false; - - if (!StringToIntegerUntilCharWithCheck(minor, endptr, 16, ' ', &endptr)) - return false; - - if (!StringToIntegerUntilCharWithCheck(inode, endptr, 10, ' ', &endptr)) - return false; - - *filename_offset = (endptr - text); - return true; -#else - return false; -#endif -} - -ProcMapsIterator::ProcMapsIterator(pid_t pid) { - Init(pid, NULL, false); -} - -ProcMapsIterator::ProcMapsIterator(pid_t pid, Buffer *buffer) { - Init(pid, buffer, false); -} - -ProcMapsIterator::ProcMapsIterator(pid_t pid, Buffer *buffer, - bool use_maps_backing) { - Init(pid, buffer, use_maps_backing); -} - -void ProcMapsIterator::Init(pid_t pid, Buffer *buffer, - bool use_maps_backing) { - pid_ = pid; - using_maps_backing_ = use_maps_backing; - dynamic_buffer_ = NULL; - if (!buffer) { - // If the user didn't pass in any buffer storage, allocate it - // now. This is the normal case; the signal handler passes in a - // static buffer. - buffer = dynamic_buffer_ = new Buffer; - } else { - dynamic_buffer_ = NULL; - } - - ibuf_ = buffer->buf_; - - stext_ = etext_ = nextline_ = ibuf_; - ebuf_ = ibuf_ + Buffer::kBufSize - 1; - nextline_ = ibuf_; - -#if defined(__linux__) || defined(__CYGWIN__) || defined(__CYGWIN32__) - if (use_maps_backing) { // don't bother with clever "self" stuff in this case - ConstructFilename("/proc/%d/maps_backing", pid, ibuf_, Buffer::kBufSize); - } else if (pid == 0) { - // We have to kludge a bit to deal with the args ConstructFilename - // expects. The 1 is never used -- it's only impt. that it's not 0. - ConstructFilename("/proc/self/maps", 1, ibuf_, Buffer::kBufSize); - } else { - ConstructFilename("/proc/%d/maps", pid, ibuf_, Buffer::kBufSize); - } - // No error logging since this can be called from the crash dump - // handler at awkward moments. Users should call Valid() before - // using. - NO_INTR(fd_ = open(ibuf_, O_RDONLY)); -#elif defined(__FreeBSD__) - // We don't support maps_backing on freebsd - if (pid == 0) { - ConstructFilename("/proc/curproc/map", 1, ibuf_, Buffer::kBufSize); - } else { - ConstructFilename("/proc/%d/map", pid, ibuf_, Buffer::kBufSize); - } - NO_INTR(fd_ = open(ibuf_, O_RDONLY)); -#elif defined(__sun__) - if (pid == 0) { - ConstructFilename("/proc/self/map", 1, ibuf_, Buffer::kBufSize); - } else { - ConstructFilename("/proc/%d/map", pid, ibuf_, Buffer::kBufSize); - } - NO_INTR(fd_ = open(ibuf_, O_RDONLY)); -#elif defined(__MACH__) - current_image_ = _dyld_image_count(); // count down from the top - current_load_cmd_ = -1; -#elif defined(PLATFORM_WINDOWS) - snapshot_ = CreateToolhelp32Snapshot(TH32CS_SNAPMODULE | - TH32CS_SNAPMODULE32, - GetCurrentProcessId()); - memset(&module_, 0, sizeof(module_)); -#else - fd_ = -1; // so Valid() is always false -#endif - -} - -ProcMapsIterator::~ProcMapsIterator() { -#if defined(PLATFORM_WINDOWS) - if (snapshot_ != INVALID_HANDLE_VALUE) CloseHandle(snapshot_); -#elif defined(__MACH__) - // no cleanup necessary! -#else - if (fd_ >= 0) NO_INTR(close(fd_)); -#endif - delete dynamic_buffer_; -} - -bool ProcMapsIterator::Valid() const { -#if defined(PLATFORM_WINDOWS) - return snapshot_ != INVALID_HANDLE_VALUE; -#elif defined(__MACH__) - return 1; -#else - return fd_ != -1; -#endif -} - -bool ProcMapsIterator::Next(uint64 *start, uint64 *end, char **flags, - uint64 *offset, int64 *inode, char **filename) { - return NextExt(start, end, flags, offset, inode, filename, NULL, NULL, - NULL, NULL, NULL); -} - -// This has too many arguments. It should really be building -// a map object and returning it. The problem is that this is called -// when the memory allocator state is undefined, hence the arguments. -bool ProcMapsIterator::NextExt(uint64 *start, uint64 *end, char **flags, - uint64 *offset, int64 *inode, char **filename, - uint64 *file_mapping, uint64 *file_pages, - uint64 *anon_mapping, uint64 *anon_pages, - dev_t *dev) { - -#if defined(__linux__) || defined(__FreeBSD__) || defined(__CYGWIN__) || defined(__CYGWIN32__) - do { - // Advance to the start of the next line - stext_ = nextline_; - - // See if we have a complete line in the buffer already - nextline_ = static_cast(memchr (stext_, '\n', etext_ - stext_)); - if (!nextline_) { - // Shift/fill the buffer so we do have a line - int count = etext_ - stext_; - - // Move the current text to the start of the buffer - memmove(ibuf_, stext_, count); - stext_ = ibuf_; - etext_ = ibuf_ + count; - - int nread = 0; // fill up buffer with text - while (etext_ < ebuf_) { - NO_INTR(nread = read(fd_, etext_, ebuf_ - etext_)); - if (nread > 0) - etext_ += nread; - else - break; - } - - // Zero out remaining characters in buffer at EOF to avoid returning - // garbage from subsequent calls. - if (etext_ != ebuf_ && nread == 0) { - memset(etext_, 0, ebuf_ - etext_); - } - *etext_ = '\n'; // sentinel; safe because ibuf extends 1 char beyond ebuf - nextline_ = static_cast(memchr (stext_, '\n', etext_ + 1 - stext_)); - } - *nextline_ = 0; // turn newline into nul - nextline_ += ((nextline_ < etext_)? 1 : 0); // skip nul if not end of text - // stext_ now points at a nul-terminated line - uint64 tmpstart, tmpend, tmpoffset; - int64 tmpinode; - int major, minor; - unsigned filename_offset = 0; -#if defined(__linux__) - // for now, assume all linuxes have the same format - if (!ParseProcMapsLine( - stext_, - start ? start : &tmpstart, - end ? end : &tmpend, - flags_, - offset ? offset : &tmpoffset, - &major, &minor, - inode ? inode : &tmpinode, &filename_offset)) continue; -#elif defined(__CYGWIN__) || defined(__CYGWIN32__) - // cygwin is like linux, except the third field is the "entry point" - // rather than the offset (see format_process_maps at - // http://cygwin.com/cgi-bin/cvsweb.cgi/src/winsup/cygwin/fhandler_process.cc?rev=1.89&content-type=text/x-cvsweb-markup&cvsroot=src - // Offset is always be 0 on cygwin: cygwin implements an mmap - // by loading the whole file and then calling NtMapViewOfSection. - // Cygwin also seems to set its flags kinda randomly; use windows default. - char tmpflags[5]; - if (offset) - *offset = 0; - strcpy(flags_, "r-xp"); - if (sscanf(stext_, "%llx-%llx %4s %llx %x:%x %lld %n", - start ? start : &tmpstart, - end ? end : &tmpend, - tmpflags, - &tmpoffset, - &major, &minor, - inode ? inode : &tmpinode, &filename_offset) != 7) continue; -#elif defined(__FreeBSD__) - // For the format, see http://www.freebsd.org/cgi/cvsweb.cgi/src/sys/fs/procfs/procfs_map.c?rev=1.31&content-type=text/x-cvsweb-markup - tmpstart = tmpend = tmpoffset = 0; - tmpinode = 0; - major = minor = 0; // can't get this info in freebsd - if (inode) - *inode = 0; // nor this - if (offset) - *offset = 0; // seems like this should be in there, but maybe not - // start end resident privateresident obj(?) prot refcnt shadowcnt - // flags copy_on_write needs_copy type filename: - // 0x8048000 0x804a000 2 0 0xc104ce70 r-x 1 0 0x0 COW NC vnode /bin/cat - if (sscanf(stext_, "0x%" SCNx64 " 0x%" SCNx64 " %*d %*d %*p %3s %*d %*d 0x%*x %*s %*s %*s %n", - start ? start : &tmpstart, - end ? end : &tmpend, - flags_, - &filename_offset) != 3) continue; -#endif - - // Depending on the Linux kernel being used, there may or may not be a space - // after the inode if there is no filename. sscanf will in such situations - // nondeterministically either fill in filename_offset or not (the results - // differ on multiple calls in the same run even with identical arguments). - // We don't want to wander off somewhere beyond the end of the string. - size_t stext_length = strlen(stext_); - if (filename_offset == 0 || filename_offset > stext_length) - filename_offset = stext_length; - - // We found an entry - if (flags) *flags = flags_; - if (filename) *filename = stext_ + filename_offset; - if (dev) *dev = minor | (major << 8); - - if (using_maps_backing_) { - // Extract and parse physical page backing info. - char *backing_ptr = stext_ + filename_offset + - strlen(stext_+filename_offset); - - // find the second '(' - int paren_count = 0; - while (--backing_ptr > stext_) { - if (*backing_ptr == '(') { - ++paren_count; - if (paren_count >= 2) { - uint64 tmp_file_mapping; - uint64 tmp_file_pages; - uint64 tmp_anon_mapping; - uint64 tmp_anon_pages; - - sscanf(backing_ptr+1, "F %" SCNx64 " %" SCNd64 ") (A %" SCNx64 " %" SCNd64 ")", - file_mapping ? file_mapping : &tmp_file_mapping, - file_pages ? file_pages : &tmp_file_pages, - anon_mapping ? anon_mapping : &tmp_anon_mapping, - anon_pages ? anon_pages : &tmp_anon_pages); - // null terminate the file name (there is a space - // before the first (. - backing_ptr[-1] = 0; - break; - } - } - } - } - - return true; - } while (etext_ > ibuf_); -#elif defined(__sun__) - // This is based on MA_READ == 4, MA_WRITE == 2, MA_EXEC == 1 - static char kPerms[8][4] = { "---", "--x", "-w-", "-wx", - "r--", "r-x", "rw-", "rwx" }; - COMPILE_ASSERT(MA_READ == 4, solaris_ma_read_must_equal_4); - COMPILE_ASSERT(MA_WRITE == 2, solaris_ma_write_must_equal_2); - COMPILE_ASSERT(MA_EXEC == 1, solaris_ma_exec_must_equal_1); - Buffer object_path; - int nread = 0; // fill up buffer with text - NO_INTR(nread = read(fd_, ibuf_, sizeof(prmap_t))); - if (nread == sizeof(prmap_t)) { - long inode_from_mapname = 0; - prmap_t* mapinfo = reinterpret_cast(ibuf_); - // Best-effort attempt to get the inode from the filename. I think the - // two middle ints are major and minor device numbers, but I'm not sure. - sscanf(mapinfo->pr_mapname, "ufs.%*d.%*d.%ld", &inode_from_mapname); - - if (pid_ == 0) { - CHECK_LT(snprintf(object_path.buf_, Buffer::kBufSize, - "/proc/self/path/%s", mapinfo->pr_mapname), - Buffer::kBufSize); - } else { - CHECK_LT(snprintf(object_path.buf_, Buffer::kBufSize, - "/proc/%d/path/%s", - static_cast(pid_), mapinfo->pr_mapname), - Buffer::kBufSize); - } - ssize_t len = readlink(object_path.buf_, current_filename_, PATH_MAX); - CHECK_LT(len, PATH_MAX); - if (len < 0) - len = 0; - current_filename_[len] = '\0'; - - if (start) *start = mapinfo->pr_vaddr; - if (end) *end = mapinfo->pr_vaddr + mapinfo->pr_size; - if (flags) *flags = kPerms[mapinfo->pr_mflags & 7]; - if (offset) *offset = mapinfo->pr_offset; - if (inode) *inode = inode_from_mapname; - if (filename) *filename = current_filename_; - if (file_mapping) *file_mapping = 0; - if (file_pages) *file_pages = 0; - if (anon_mapping) *anon_mapping = 0; - if (anon_pages) *anon_pages = 0; - if (dev) *dev = 0; - return true; - } -#elif defined(__MACH__) - // We return a separate entry for each segment in the DLL. (TODO(csilvers): - // can we do better?) A DLL ("image") has load-commands, some of which - // talk about segment boundaries. - // cf image_for_address from http://svn.digium.com/view/asterisk/team/oej/minivoicemail/dlfcn.c?revision=53912 - for (; current_image_ >= 0; current_image_--) { - const mach_header* hdr = _dyld_get_image_header(current_image_); - if (!hdr) continue; - if (current_load_cmd_ < 0) // set up for this image - current_load_cmd_ = hdr->ncmds; // again, go from the top down - - // We start with the next load command (we've already looked at this one). - for (current_load_cmd_--; current_load_cmd_ >= 0; current_load_cmd_--) { -#ifdef MH_MAGIC_64 - if (NextExtMachHelper( - hdr, current_image_, current_load_cmd_, - start, end, flags, offset, inode, filename, - file_mapping, file_pages, anon_mapping, - anon_pages, dev)) { - return true; - } -#endif - if (NextExtMachHelper( - hdr, current_image_, current_load_cmd_, - start, end, flags, offset, inode, filename, - file_mapping, file_pages, anon_mapping, - anon_pages, dev)) { - return true; - } - } - // If we get here, no more load_cmd's in this image talk about - // segments. Go on to the next image. - } -#elif defined(PLATFORM_WINDOWS) - static char kDefaultPerms[5] = "r-xp"; - BOOL ok; - if (module_.dwSize == 0) { // only possible before first call - module_.dwSize = sizeof(module_); - ok = Module32First(snapshot_, &module_); - } else { - ok = Module32Next(snapshot_, &module_); - } - if (ok) { - uint64 base_addr = reinterpret_cast(module_.modBaseAddr); - if (start) *start = base_addr; - if (end) *end = base_addr + module_.modBaseSize; - if (flags) *flags = kDefaultPerms; - if (offset) *offset = 0; - if (inode) *inode = 0; - if (filename) *filename = module_.szExePath; - if (file_mapping) *file_mapping = 0; - if (file_pages) *file_pages = 0; - if (anon_mapping) *anon_mapping = 0; - if (anon_pages) *anon_pages = 0; - if (dev) *dev = 0; - return true; - } -#endif - - // We didn't find anything - return false; -} - -int ProcMapsIterator::FormatLine(char* buffer, int bufsize, - uint64 start, uint64 end, const char *flags, - uint64 offset, int64 inode, - const char *filename, dev_t dev) { - // We assume 'flags' looks like 'rwxp' or 'rwx'. - char r = (flags && flags[0] == 'r') ? 'r' : '-'; - char w = (flags && flags[0] && flags[1] == 'w') ? 'w' : '-'; - char x = (flags && flags[0] && flags[1] && flags[2] == 'x') ? 'x' : '-'; - // p always seems set on linux, so we set the default to 'p', not '-' - char p = (flags && flags[0] && flags[1] && flags[2] && flags[3] != 'p') - ? '-' : 'p'; - - const int rc = snprintf(buffer, bufsize, - "%08" PRIx64 "-%08" PRIx64 " %c%c%c%c %08" PRIx64 " %02x:%02x %-11" PRId64 " %s\n", - start, end, r,w,x,p, offset, - static_cast(dev/256), static_cast(dev%256), - inode, filename); - return (rc < 0 || rc >= bufsize) ? 0 : rc; -} - -namespace tcmalloc { - -// Helper to add the list of mapped shared libraries to a profile. -// Fill formatted "/proc/self/maps" contents into buffer 'buf' of size 'size' -// and return the actual size occupied in 'buf'. We fill wrote_all to true -// if we successfully wrote all proc lines to buf, false else. -// We do not provision for 0-terminating 'buf'. -int FillProcSelfMaps(char buf[], int size, bool* wrote_all) { - ProcMapsIterator::Buffer iterbuf; - ProcMapsIterator it(0, &iterbuf); // 0 means "current pid" - - uint64 start, end, offset; - int64 inode; - char *flags, *filename; - int bytes_written = 0; - *wrote_all = true; - while (it.Next(&start, &end, &flags, &offset, &inode, &filename)) { - const int line_length = it.FormatLine(buf + bytes_written, - size - bytes_written, - start, end, flags, offset, - inode, filename, 0); - if (line_length == 0) - *wrote_all = false; // failed to write this line out - else - bytes_written += line_length; - - } - return bytes_written; -} - -// Dump the same data as FillProcSelfMaps reads to fd. -// It seems easier to repeat parts of FillProcSelfMaps here than to -// reuse it via a call. -void DumpProcSelfMaps(RawFD fd) { - ProcMapsIterator::Buffer iterbuf; - ProcMapsIterator it(0, &iterbuf); // 0 means "current pid" - - uint64 start, end, offset; - int64 inode; - char *flags, *filename; - ProcMapsIterator::Buffer linebuf; - while (it.Next(&start, &end, &flags, &offset, &inode, &filename)) { - int written = it.FormatLine(linebuf.buf_, sizeof(linebuf.buf_), - start, end, flags, offset, inode, filename, - 0); - RawWrite(fd, linebuf.buf_, written); - } -} - -} // namespace tcmalloc diff --git a/contrib/libtcmalloc/src/base/sysinfo.h b/contrib/libtcmalloc/src/base/sysinfo.h deleted file mode 100644 index 75b101376c5..00000000000 --- a/contrib/libtcmalloc/src/base/sysinfo.h +++ /dev/null @@ -1,232 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2006, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// All functions here are thread-hostile due to file caching unless -// commented otherwise. - -#ifndef _SYSINFO_H_ -#define _SYSINFO_H_ - -#include "../config.h" - -#include -#if (defined(_WIN32) || defined(__MINGW32__)) && (!defined(__CYGWIN__) && !defined(__CYGWIN32__)) -#include // for DWORD -#include // for CreateToolhelp32Snapshot -#endif -#ifdef HAVE_UNISTD_H -#include // for pid_t -#endif -#include // for size_t -#include // for PATH_MAX -#include "base/basictypes.h" -#include "base/logging.h" // for RawFD - -// This getenv function is safe to call before the C runtime is initialized. -// On Windows, it utilizes GetEnvironmentVariable() and on unix it uses -// /proc/self/environ instead calling getenv(). It's intended to be used in -// routines that run before main(), when the state required for getenv() may -// not be set up yet. In particular, errno isn't set up until relatively late -// (after the pthreads library has a chance to make it threadsafe), and -// getenv() doesn't work until then. -// On some platforms, this call will utilize the same, static buffer for -// repeated GetenvBeforeMain() calls. Callers should not expect pointers from -// this routine to be long lived. -// Note that on unix, /proc only has the environment at the time the -// application was started, so this routine ignores setenv() calls/etc. Also -// note it only reads the first 16K of the environment. -extern const char* GetenvBeforeMain(const char* name); - -// This takes as an argument an environment-variable name (like -// CPUPROFILE) whose value is supposed to be a file-path, and sets -// path to that path, and returns true. Non-trivial for surprising -// reasons, as documented in sysinfo.cc. path must have space PATH_MAX. -extern bool GetUniquePathFromEnv(const char* env_name, char* path); - -extern int GetSystemCPUsCount(); - -void SleepForMilliseconds(int milliseconds); - -// Return true if we're running POSIX (e.g., NPTL on Linux) threads, -// as opposed to a non-POSIX thread library. The thing that we care -// about is whether a thread's pid is the same as the thread that -// spawned it. If so, this function returns true. -// Thread-safe. -// Note: We consider false negatives to be OK. -bool HasPosixThreads(); - -#ifndef SWIG // SWIG doesn't like struct Buffer and variable arguments. - -// A ProcMapsIterator abstracts access to /proc/maps for a given -// process. Needs to be stack-allocatable and avoid using stdio/malloc -// so it can be used in the google stack dumper, heap-profiler, etc. -// -// On Windows and Mac OS X, this iterator iterates *only* over DLLs -// mapped into this process space. For Linux, FreeBSD, and Solaris, -// it iterates over *all* mapped memory regions, including anonymous -// mmaps. For other O/Ss, it is unlikely to work at all, and Valid() -// will always return false. Also note: this routine only works on -// FreeBSD if procfs is mounted: make sure this is in your /etc/fstab: -// proc /proc procfs rw 0 0 -class ProcMapsIterator { - public: - struct Buffer { -#ifdef __FreeBSD__ - // FreeBSD requires us to read all of the maps file at once, so - // we have to make a buffer that's "always" big enough - static const size_t kBufSize = 102400; -#else // a one-line buffer is good enough - static const size_t kBufSize = PATH_MAX + 1024; -#endif - char buf_[kBufSize]; - }; - - - // Create a new iterator for the specified pid. pid can be 0 for "self". - explicit ProcMapsIterator(pid_t pid); - - // Create an iterator with specified storage (for use in signal - // handler). "buffer" should point to a ProcMapsIterator::Buffer - // buffer can be NULL in which case a bufer will be allocated. - ProcMapsIterator(pid_t pid, Buffer *buffer); - - // Iterate through maps_backing instead of maps if use_maps_backing - // is true. Otherwise the same as above. buffer can be NULL and - // it will allocate a buffer itself. - ProcMapsIterator(pid_t pid, Buffer *buffer, - bool use_maps_backing); - - // Returns true if the iterator successfully initialized; - bool Valid() const; - - // Returns a pointer to the most recently parsed line. Only valid - // after Next() returns true, and until the iterator is destroyed or - // Next() is called again. This may give strange results on non-Linux - // systems. Prefer FormatLine() if that may be a concern. - const char *CurrentLine() const { return stext_; } - - // Writes the "canonical" form of the /proc/xxx/maps info for a single - // line to the passed-in buffer. Returns the number of bytes written, - // or 0 if it was not able to write the complete line. (To guarantee - // success, buffer should have size at least Buffer::kBufSize.) - // Takes as arguments values set via a call to Next(). The - // "canonical" form of the line (taken from linux's /proc/xxx/maps): - // - + - // : Note: the - // eg - // 08048000-0804c000 r-xp 00000000 03:01 3793678 /bin/cat - // If you don't have the dev_t (dev), feel free to pass in 0. - // (Next() doesn't return a dev_t, though NextExt does.) - // - // Note: if filename and flags were obtained via a call to Next(), - // then the output of this function is only valid if Next() returned - // true, and only until the iterator is destroyed or Next() is - // called again. (Since filename, at least, points into CurrentLine.) - static int FormatLine(char* buffer, int bufsize, - uint64 start, uint64 end, const char *flags, - uint64 offset, int64 inode, const char *filename, - dev_t dev); - - // Find the next entry in /proc/maps; return true if found or false - // if at the end of the file. - // - // Any of the result pointers can be NULL if you're not interested - // in those values. - // - // If "flags" and "filename" are passed, they end up pointing to - // storage within the ProcMapsIterator that is valid only until the - // iterator is destroyed or Next() is called again. The caller may - // modify the contents of these strings (up as far as the first NUL, - // and only until the subsequent call to Next()) if desired. - - // The offsets are all uint64 in order to handle the case of a - // 32-bit process running on a 64-bit kernel - // - // IMPORTANT NOTE: see top-of-class notes for details about what - // mapped regions Next() iterates over, depending on O/S. - // TODO(csilvers): make flags and filename const. - bool Next(uint64 *start, uint64 *end, char **flags, - uint64 *offset, int64 *inode, char **filename); - - bool NextExt(uint64 *start, uint64 *end, char **flags, - uint64 *offset, int64 *inode, char **filename, - uint64 *file_mapping, uint64 *file_pages, - uint64 *anon_mapping, uint64 *anon_pages, - dev_t *dev); - - ~ProcMapsIterator(); - - private: - void Init(pid_t pid, Buffer *buffer, bool use_maps_backing); - - char *ibuf_; // input buffer - char *stext_; // start of text - char *etext_; // end of text - char *nextline_; // start of next line - char *ebuf_; // end of buffer (1 char for a nul) -#if (defined(_WIN32) || defined(__MINGW32__)) && (!defined(__CYGWIN__) && !defined(__CYGWIN32__)) - HANDLE snapshot_; // filehandle on dll info - // In a change from the usual W-A pattern, there is no A variant of - // MODULEENTRY32. Tlhelp32.h #defines the W variant, but not the A. - // We want the original A variants, and this #undef is the only - // way I see to get them. Redefining it when we're done prevents us - // from affecting other .cc files. -# ifdef MODULEENTRY32 // Alias of W -# undef MODULEENTRY32 - MODULEENTRY32 module_; // info about current dll (and dll iterator) -# define MODULEENTRY32 MODULEENTRY32W -# else // It's the ascii, the one we want. - MODULEENTRY32 module_; // info about current dll (and dll iterator) -# endif -#elif defined(__MACH__) - int current_image_; // dll's are called "images" in macos parlance - int current_load_cmd_; // the segment of this dll we're examining -#elif defined(__sun__) // Solaris - int fd_; - char current_filename_[PATH_MAX]; -#else - int fd_; // filehandle on /proc/*/maps -#endif - pid_t pid_; - char flags_[10]; - Buffer* dynamic_buffer_; // dynamically-allocated Buffer - bool using_maps_backing_; // true if we are looking at maps_backing instead of maps. -}; - -#endif /* #ifndef SWIG */ - -// Helper routines - -namespace tcmalloc { -int FillProcSelfMaps(char buf[], int size, bool* wrote_all); -void DumpProcSelfMaps(RawFD fd); -} - -#endif /* #ifndef _SYSINFO_H_ */ diff --git a/contrib/libtcmalloc/src/base/thread_annotations.h b/contrib/libtcmalloc/src/base/thread_annotations.h deleted file mode 100644 index f57b2999ee7..00000000000 --- a/contrib/libtcmalloc/src/base/thread_annotations.h +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Le-Chun Wu -// -// This header file contains the macro definitions for thread safety -// annotations that allow the developers to document the locking policies -// of their multi-threaded code. The annotations can also help program -// analysis tools to identify potential thread safety issues. -// -// The annotations are implemented using GCC's "attributes" extension. -// Using the macros defined here instead of the raw GCC attributes allows -// for portability and future compatibility. -// -// This functionality is not yet fully implemented in perftools, -// but may be one day. - -#ifndef BASE_THREAD_ANNOTATIONS_H_ -#define BASE_THREAD_ANNOTATIONS_H_ - - -#if defined(__GNUC__) \ - && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4)) \ - && defined(__SUPPORT_TS_ANNOTATION__) && (!defined(SWIG)) -#define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) -#else -#define THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op -#endif - - -// Document if a shared variable/field needs to be protected by a lock. -// GUARDED_BY allows the user to specify a particular lock that should be -// held when accessing the annotated variable, while GUARDED_VAR only -// indicates a shared variable should be guarded (by any lock). GUARDED_VAR -// is primarily used when the client cannot express the name of the lock. -#define GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x)) -#define GUARDED_VAR THREAD_ANNOTATION_ATTRIBUTE__(guarded) - -// Document if the memory location pointed to by a pointer should be guarded -// by a lock when dereferencing the pointer. Similar to GUARDED_VAR, -// PT_GUARDED_VAR is primarily used when the client cannot express the name -// of the lock. Note that a pointer variable to a shared memory location -// could itself be a shared variable. For example, if a shared global pointer -// q, which is guarded by mu1, points to a shared memory location that is -// guarded by mu2, q should be annotated as follows: -// int *q GUARDED_BY(mu1) PT_GUARDED_BY(mu2); -#define PT_GUARDED_BY(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(point_to_guarded_by(x)) -#define PT_GUARDED_VAR \ - THREAD_ANNOTATION_ATTRIBUTE__(point_to_guarded) - -// Document the acquisition order between locks that can be held -// simultaneously by a thread. For any two locks that need to be annotated -// to establish an acquisition order, only one of them needs the annotation. -// (i.e. You don't have to annotate both locks with both ACQUIRED_AFTER -// and ACQUIRED_BEFORE.) -#define ACQUIRED_AFTER(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(x)) -#define ACQUIRED_BEFORE(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(x)) - -// The following three annotations document the lock requirements for -// functions/methods. - -// Document if a function expects certain locks to be held before it is called -#define EXCLUSIVE_LOCKS_REQUIRED(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(exclusive_locks_required(x)) - -#define SHARED_LOCKS_REQUIRED(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(shared_locks_required(x)) - -// Document the locks acquired in the body of the function. These locks -// cannot be held when calling this function (as google3's Mutex locks are -// non-reentrant). -#define LOCKS_EXCLUDED(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(x)) - -// Document the lock the annotated function returns without acquiring it. -#define LOCK_RETURNED(x) THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x)) - -// Document if a class/type is a lockable type (such as the Mutex class). -#define LOCKABLE THREAD_ANNOTATION_ATTRIBUTE__(lockable) - -// Document if a class is a scoped lockable type (such as the MutexLock class). -#define SCOPED_LOCKABLE THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable) - -// The following annotations specify lock and unlock primitives. -#define EXCLUSIVE_LOCK_FUNCTION(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(exclusive_lock(x)) - -#define SHARED_LOCK_FUNCTION(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(shared_lock(x)) - -#define EXCLUSIVE_TRYLOCK_FUNCTION(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(exclusive_trylock(x)) - -#define SHARED_TRYLOCK_FUNCTION(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(shared_trylock(x)) - -#define UNLOCK_FUNCTION(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(unlock(x)) - -// An escape hatch for thread safety analysis to ignore the annotated function. -#define NO_THREAD_SAFETY_ANALYSIS \ - THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis) - -#endif // BASE_THREAD_ANNOTATIONS_H_ diff --git a/contrib/libtcmalloc/src/base/thread_lister.c b/contrib/libtcmalloc/src/base/thread_lister.c deleted file mode 100644 index 9dc8d721892..00000000000 --- a/contrib/libtcmalloc/src/base/thread_lister.c +++ /dev/null @@ -1,83 +0,0 @@ -/* Copyright (c) 2005-2007, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Markus Gutschke - */ - -#include "config.h" - -#include "base/thread_lister.h" - -#include /* needed for NULL on some powerpc platforms (?!) */ -#include -#include /* for getpid */ - -#ifdef HAVE_SYS_PRCTL -# include -#endif - -#include "base/linuxthreads.h" -/* Include other thread listers here that define THREADS macro - * only when they can provide a good implementation. - */ - -#ifndef THREADS - -/* Default trivial thread lister for single-threaded applications, - * or if the multi-threading code has not been ported, yet. - */ - -int TCMalloc_ListAllProcessThreads(void *parameter, - ListAllProcessThreadsCallBack callback, ...) { - int rc; - va_list ap; - pid_t pid; - -#ifdef HAVE_SYS_PRCTL - int dumpable = prctl(PR_GET_DUMPABLE, 0); - if (!dumpable) - prctl(PR_SET_DUMPABLE, 1); -#endif - va_start(ap, callback); - pid = getpid(); - rc = callback(parameter, 1, &pid, ap); - va_end(ap); -#ifdef HAVE_SYS_PRCTL - if (!dumpable) - prctl(PR_SET_DUMPABLE, 0); -#endif - return rc; -} - -int TCMalloc_ResumeAllProcessThreads(int num_threads, pid_t *thread_pids) { - return 1; -} - -#endif /* ifndef THREADS */ diff --git a/contrib/libtcmalloc/src/base/thread_lister.h b/contrib/libtcmalloc/src/base/thread_lister.h deleted file mode 100644 index 6e70b89fef5..00000000000 --- a/contrib/libtcmalloc/src/base/thread_lister.h +++ /dev/null @@ -1,83 +0,0 @@ -/* -*- Mode: c; c-basic-offset: 2; indent-tabs-mode: nil -*- */ -/* Copyright (c) 2005-2007, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Markus Gutschke - */ - -#ifndef _THREAD_LISTER_H -#define _THREAD_LISTER_H - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef int (*ListAllProcessThreadsCallBack)(void *parameter, - int num_threads, - pid_t *thread_pids, - va_list ap); - -/* This function gets the list of all linux threads of the current process - * passes them to the 'callback' along with the 'parameter' pointer; at the - * call back call time all the threads are paused via - * PTRACE_ATTACH. - * The callback is executed from a separate thread which shares only the - * address space, the filesystem, and the filehandles with the caller. Most - * notably, it does not share the same pid and ppid; and if it terminates, - * the rest of the application is still there. 'callback' is supposed to do - * or arrange for TCMalloc_ResumeAllProcessThreads. This happens automatically, if - * the thread raises a synchronous signal (e.g. SIGSEGV); asynchronous - * signals are blocked. If the 'callback' decides to unblock them, it must - * ensure that they cannot terminate the application, or that - * TCMalloc_ResumeAllProcessThreads will get called. - * It is an error for the 'callback' to make any library calls that could - * acquire locks. Most notably, this means that most system calls have to - * avoid going through libc. Also, this means that it is not legal to call - * exit() or abort(). - * We return -1 on error and the return value of 'callback' on success. - */ -int TCMalloc_ListAllProcessThreads(void *parameter, - ListAllProcessThreadsCallBack callback, ...); - -/* This function resumes the list of all linux threads that - * TCMalloc_ListAllProcessThreads pauses before giving to its - * callback. The function returns non-zero if at least one thread was - * suspended and has now been resumed. - */ -int TCMalloc_ResumeAllProcessThreads(int num_threads, pid_t *thread_pids); - -#ifdef __cplusplus -} -#endif - -#endif /* _THREAD_LISTER_H */ diff --git a/contrib/libtcmalloc/src/base/vdso_support.cc b/contrib/libtcmalloc/src/base/vdso_support.cc deleted file mode 100644 index 730df3011d6..00000000000 --- a/contrib/libtcmalloc/src/base/vdso_support.cc +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Paul Pluzhnikov -// -// Allow dynamic symbol lookup in the kernel VDSO page. -// -// VDSOSupport -- a class representing kernel VDSO (if present). -// - -#include "base/vdso_support.h" - -#ifdef HAVE_VDSO_SUPPORT // defined in vdso_support.h - -#include -#include // for ptrdiff_t - -#include "base/atomicops.h" // for MemoryBarrier -#include "base/linux_syscall_support.h" -#include "base/logging.h" -#include "base/dynamic_annotations.h" -#include "base/basictypes.h" // for COMPILE_ASSERT - -using base::subtle::MemoryBarrier; - -#ifndef AT_SYSINFO_EHDR -#define AT_SYSINFO_EHDR 33 -#endif - -namespace base { - -const void *VDSOSupport::vdso_base_ = ElfMemImage::kInvalidBase; -VDSOSupport::VDSOSupport() - // If vdso_base_ is still set to kInvalidBase, we got here - // before VDSOSupport::Init has been called. Call it now. - : image_(vdso_base_ == ElfMemImage::kInvalidBase ? Init() : vdso_base_) { -} - -// NOTE: we can't use GoogleOnceInit() below, because we can be -// called by tcmalloc, and none of the *once* stuff may be functional yet. -// -// In addition, we hope that the VDSOSupportHelper constructor -// causes this code to run before there are any threads, and before -// InitGoogle() has executed any chroot or setuid calls. -// -// Finally, even if there is a race here, it is harmless, because -// the operation should be idempotent. -const void *VDSOSupport::Init() { - if (vdso_base_ == ElfMemImage::kInvalidBase) { - // Valgrind zaps AT_SYSINFO_EHDR and friends from the auxv[] - // on stack, and so glibc works as if VDSO was not present. - // But going directly to kernel via /proc/self/auxv below bypasses - // Valgrind zapping. So we check for Valgrind separately. - if (RunningOnValgrind()) { - vdso_base_ = NULL; - return NULL; - } - int fd = open("/proc/self/auxv", O_RDONLY); - if (fd == -1) { - // Kernel too old to have a VDSO. - vdso_base_ = NULL; - return NULL; - } - ElfW(auxv_t) aux; - while (read(fd, &aux, sizeof(aux)) == sizeof(aux)) { - if (aux.a_type == AT_SYSINFO_EHDR) { - COMPILE_ASSERT(sizeof(vdso_base_) == sizeof(aux.a_un.a_val), - unexpected_sizeof_pointer_NE_sizeof_a_val); - vdso_base_ = reinterpret_cast(aux.a_un.a_val); - break; - } - } - close(fd); - if (vdso_base_ == ElfMemImage::kInvalidBase) { - // Didn't find AT_SYSINFO_EHDR in auxv[]. - vdso_base_ = NULL; - } - } - return vdso_base_; -} - -const void *VDSOSupport::SetBase(const void *base) { - CHECK(base != ElfMemImage::kInvalidBase); - const void *old_base = vdso_base_; - vdso_base_ = base; - image_.Init(base); - return old_base; -} - -bool VDSOSupport::LookupSymbol(const char *name, - const char *version, - int type, - SymbolInfo *info) const { - return image_.LookupSymbol(name, version, type, info); -} - -bool VDSOSupport::LookupSymbolByAddress(const void *address, - SymbolInfo *info_out) const { - return image_.LookupSymbolByAddress(address, info_out); -} - -// We need to make sure VDSOSupport::Init() is called before -// the main() runs, since it might do something like setuid or -// chroot. If VDSOSupport -// is used in any global constructor, this will happen, since -// VDSOSupport's constructor calls Init. But if not, we need to -// ensure it here, with a global constructor of our own. This -// is an allowed exception to the normal rule against non-trivial -// global constructors. -static class VDSOInitHelper { - public: - VDSOInitHelper() { VDSOSupport::Init(); } -} vdso_init_helper; -} - -#endif // HAVE_VDSO_SUPPORT diff --git a/contrib/libtcmalloc/src/base/vdso_support.h b/contrib/libtcmalloc/src/base/vdso_support.h deleted file mode 100644 index 0c2213c7c06..00000000000 --- a/contrib/libtcmalloc/src/base/vdso_support.h +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Paul Pluzhnikov -// -// Allow dynamic symbol lookup in the kernel VDSO page. -// -// VDSO stands for "Virtual Dynamic Shared Object" -- a page of -// executable code, which looks like a shared library, but doesn't -// necessarily exist anywhere on disk, and which gets mmap()ed into -// every process by kernels which support VDSO, such as 2.6.x for 32-bit -// executables, and 2.6.24 and above for 64-bit executables. -// -// More details could be found here: -// http://www.trilithium.com/johan/2005/08/linux-gate/ -// -// VDSOSupport -- a class representing kernel VDSO (if present). -// -// Example usage: -// VDSOSupport vdso; -// VDSOSupport::SymbolInfo info; -// typedef (*FN)(unsigned *, void *, void *); -// FN fn = NULL; -// if (vdso.LookupSymbol("__vdso_getcpu", "LINUX_2.6", STT_FUNC, &info)) { -// fn = reinterpret_cast(info.address); -// } - -#ifndef BASE_VDSO_SUPPORT_H_ -#define BASE_VDSO_SUPPORT_H_ - -#include "../config.h" -#include "base/basictypes.h" -#include "base/elf_mem_image.h" - -#ifdef HAVE_ELF_MEM_IMAGE - -#define HAVE_VDSO_SUPPORT 1 - -#include // for NULL - -namespace base { - -// NOTE: this class may be used from within tcmalloc, and can not -// use any memory allocation routines. -class VDSOSupport { - public: - VDSOSupport(); - - typedef ElfMemImage::SymbolInfo SymbolInfo; - typedef ElfMemImage::SymbolIterator SymbolIterator; - - // Answers whether we have a vdso at all. - bool IsPresent() const { return image_.IsPresent(); } - - // Allow to iterate over all VDSO symbols. - SymbolIterator begin() const { return image_.begin(); } - SymbolIterator end() const { return image_.end(); } - - // Look up versioned dynamic symbol in the kernel VDSO. - // Returns false if VDSO is not present, or doesn't contain given - // symbol/version/type combination. - // If info_out != NULL, additional details are filled in. - bool LookupSymbol(const char *name, const char *version, - int symbol_type, SymbolInfo *info_out) const; - - // Find info about symbol (if any) which overlaps given address. - // Returns true if symbol was found; false if VDSO isn't present - // or doesn't have a symbol overlapping given address. - // If info_out != NULL, additional details are filled in. - bool LookupSymbolByAddress(const void *address, SymbolInfo *info_out) const; - - // Used only for testing. Replace real VDSO base with a mock. - // Returns previous value of vdso_base_. After you are done testing, - // you are expected to call SetBase() with previous value, in order to - // reset state to the way it was. - const void *SetBase(const void *s); - - // Computes vdso_base_ and returns it. Should be called as early as - // possible; before any thread creation, chroot or setuid. - static const void *Init(); - - private: - // image_ represents VDSO ELF image in memory. - // image_.ehdr_ == NULL implies there is no VDSO. - ElfMemImage image_; - - // Cached value of auxv AT_SYSINFO_EHDR, computed once. - // This is a tri-state: - // kInvalidBase => value hasn't been determined yet. - // 0 => there is no VDSO. - // else => vma of VDSO Elf{32,64}_Ehdr. - // - // When testing with mock VDSO, low bit is set. - // The low bit is always available because vdso_base_ is - // page-aligned. - static const void *vdso_base_; - - DISALLOW_COPY_AND_ASSIGN(VDSOSupport); -}; - -} // namespace base - -#endif // HAVE_ELF_MEM_IMAGE - -#endif // BASE_VDSO_SUPPORT_H_ diff --git a/contrib/libtcmalloc/src/central_freelist.cc b/contrib/libtcmalloc/src/central_freelist.cc deleted file mode 100644 index 11b190dcfee..00000000000 --- a/contrib/libtcmalloc/src/central_freelist.cc +++ /dev/null @@ -1,387 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat - -#include "config.h" -#include -#include "central_freelist.h" -#include "internal_logging.h" // for ASSERT, MESSAGE -#include "linked_list.h" // for SLL_Next, SLL_Push, etc -#include "page_heap.h" // for PageHeap -#include "static_vars.h" // for Static - -using std::min; -using std::max; - -namespace tcmalloc { - -void CentralFreeList::Init(size_t cl) { - size_class_ = cl; - tcmalloc::DLL_Init(&empty_); - tcmalloc::DLL_Init(&nonempty_); - num_spans_ = 0; - counter_ = 0; - - max_cache_size_ = kMaxNumTransferEntries; -#ifdef TCMALLOC_SMALL_BUT_SLOW - // Disable the transfer cache for the small footprint case. - cache_size_ = 0; -#else - cache_size_ = 16; -#endif - if (cl > 0) { - // Limit the maximum size of the cache based on the size class. If this - // is not done, large size class objects will consume a lot of memory if - // they just sit in the transfer cache. - int32_t bytes = Static::sizemap()->ByteSizeForClass(cl); - int32_t objs_to_move = Static::sizemap()->num_objects_to_move(cl); - - ASSERT(objs_to_move > 0 && bytes > 0); - // Limit each size class cache to at most 1MB of objects or one entry, - // whichever is greater. Total transfer cache memory used across all - // size classes then can't be greater than approximately - // 1MB * kMaxNumTransferEntries. - // min and max are in parens to avoid macro-expansion on windows. - max_cache_size_ = (min)(max_cache_size_, - (max)(1, (1024 * 1024) / (bytes * objs_to_move))); - cache_size_ = (min)(cache_size_, max_cache_size_); - } - used_slots_ = 0; - ASSERT(cache_size_ <= max_cache_size_); -} - -void CentralFreeList::ReleaseListToSpans(void* start) { - while (start) { - void *next = SLL_Next(start); - ReleaseToSpans(start); - start = next; - } -} - -// MapObjectToSpan should logically be part of ReleaseToSpans. But -// this triggers an optimization bug in gcc 4.5.0. Moving to a -// separate function, and making sure that function isn't inlined, -// seems to fix the problem. It also should be fixed for gcc 4.5.1. -static -#if __GNUC__ == 4 && __GNUC_MINOR__ == 5 && __GNUC_PATCHLEVEL__ == 0 -__attribute__ ((noinline)) -#endif -Span* MapObjectToSpan(void* object) { - const PageID p = reinterpret_cast(object) >> kPageShift; - Span* span = Static::pageheap()->GetDescriptor(p); - return span; -} - -void CentralFreeList::ReleaseToSpans(void* object) { - Span* span = MapObjectToSpan(object); - ASSERT(span != NULL); - ASSERT(span->refcount > 0); - - // If span is empty, move it to non-empty list - if (span->objects == NULL) { - tcmalloc::DLL_Remove(span); - tcmalloc::DLL_Prepend(&nonempty_, span); - Event(span, 'N', 0); - } - - // The following check is expensive, so it is disabled by default - if (false) { - // Check that object does not occur in list - int got = 0; - for (void* p = span->objects; p != NULL; p = *((void**) p)) { - ASSERT(p != object); - got++; - } - ASSERT(got + span->refcount == - (span->length<ByteSizeForClass(span->sizeclass)); - } - - counter_++; - span->refcount--; - if (span->refcount == 0) { - Event(span, '#', 0); - counter_ -= ((span->length<ByteSizeForClass(span->sizeclass)); - tcmalloc::DLL_Remove(span); - --num_spans_; - - // Release central list lock while operating on pageheap - lock_.Unlock(); - { - SpinLockHolder h(Static::pageheap_lock()); - Static::pageheap()->Delete(span); - } - lock_.Lock(); - } else { - *(reinterpret_cast(object)) = span->objects; - span->objects = object; - } -} - -bool CentralFreeList::EvictRandomSizeClass( - int locked_size_class, bool force) { - static int race_counter = 0; - int t = race_counter++; // Updated without a lock, but who cares. - if (t >= kNumClasses) { - while (t >= kNumClasses) { - t -= kNumClasses; - } - race_counter = t; - } - ASSERT(t >= 0); - ASSERT(t < kNumClasses); - if (t == locked_size_class) return false; - return Static::central_cache()[t].ShrinkCache(locked_size_class, force); -} - -bool CentralFreeList::MakeCacheSpace() { - // Is there room in the cache? - if (used_slots_ < cache_size_) return true; - // Check if we can expand this cache? - if (cache_size_ == max_cache_size_) return false; - // Ok, we'll try to grab an entry from some other size class. - if (EvictRandomSizeClass(size_class_, false) || - EvictRandomSizeClass(size_class_, true)) { - // Succeeded in evicting, we're going to make our cache larger. - // However, we may have dropped and re-acquired the lock in - // EvictRandomSizeClass (via ShrinkCache and the LockInverter), so the - // cache_size may have changed. Therefore, check and verify that it is - // still OK to increase the cache_size. - if (cache_size_ < max_cache_size_) { - cache_size_++; - return true; - } - } - return false; -} - - -namespace { -class LockInverter { - private: - SpinLock *held_, *temp_; - public: - inline explicit LockInverter(SpinLock* held, SpinLock *temp) - : held_(held), temp_(temp) { held_->Unlock(); temp_->Lock(); } - inline ~LockInverter() { temp_->Unlock(); held_->Lock(); } -}; -} - -// This function is marked as NO_THREAD_SAFETY_ANALYSIS because it uses -// LockInverter to release one lock and acquire another in scoped-lock -// style, which our current annotation/analysis does not support. -bool CentralFreeList::ShrinkCache(int locked_size_class, bool force) - NO_THREAD_SAFETY_ANALYSIS { - // Start with a quick check without taking a lock. - if (cache_size_ == 0) return false; - // We don't evict from a full cache unless we are 'forcing'. - if (force == false && used_slots_ == cache_size_) return false; - - // Grab lock, but first release the other lock held by this thread. We use - // the lock inverter to ensure that we never hold two size class locks - // concurrently. That can create a deadlock because there is no well - // defined nesting order. - LockInverter li(&Static::central_cache()[locked_size_class].lock_, &lock_); - ASSERT(used_slots_ <= cache_size_); - ASSERT(0 <= cache_size_); - if (cache_size_ == 0) return false; - if (used_slots_ == cache_size_) { - if (force == false) return false; - // ReleaseListToSpans releases the lock, so we have to make all the - // updates to the central list before calling it. - cache_size_--; - used_slots_--; - ReleaseListToSpans(tc_slots_[used_slots_].head); - return true; - } - cache_size_--; - return true; -} - -void CentralFreeList::InsertRange(void *start, void *end, int N) { - SpinLockHolder h(&lock_); - if (N == Static::sizemap()->num_objects_to_move(size_class_) && - MakeCacheSpace()) { - int slot = used_slots_++; - ASSERT(slot >=0); - ASSERT(slot < max_cache_size_); - TCEntry *entry = &tc_slots_[slot]; - entry->head = start; - entry->tail = end; - return; - } - ReleaseListToSpans(start); -} - -int CentralFreeList::RemoveRange(void **start, void **end, int N) { - ASSERT(N > 0); - lock_.Lock(); - if (N == Static::sizemap()->num_objects_to_move(size_class_) && - used_slots_ > 0) { - int slot = --used_slots_; - ASSERT(slot >= 0); - TCEntry *entry = &tc_slots_[slot]; - *start = entry->head; - *end = entry->tail; - lock_.Unlock(); - return N; - } - - int result = 0; - *start = NULL; - *end = NULL; - // TODO: Prefetch multiple TCEntries? - result = FetchFromOneSpansSafe(N, start, end); - if (result != 0) { - while (result < N) { - int n; - void* head = NULL; - void* tail = NULL; - n = FetchFromOneSpans(N - result, &head, &tail); - if (!n) break; - result += n; - SLL_PushRange(start, head, tail); - } - } - lock_.Unlock(); - return result; -} - - -int CentralFreeList::FetchFromOneSpansSafe(int N, void **start, void **end) { - int result = FetchFromOneSpans(N, start, end); - if (!result) { - Populate(); - result = FetchFromOneSpans(N, start, end); - } - return result; -} - -int CentralFreeList::FetchFromOneSpans(int N, void **start, void **end) { - if (tcmalloc::DLL_IsEmpty(&nonempty_)) return 0; - Span* span = nonempty_.next; - - ASSERT(span->objects != NULL); - - int result = 0; - void *prev, *curr; - curr = span->objects; - do { - prev = curr; - curr = *(reinterpret_cast(curr)); - } while (++result < N && curr != NULL); - - if (curr == NULL) { - // Move to empty list - tcmalloc::DLL_Remove(span); - tcmalloc::DLL_Prepend(&empty_, span); - Event(span, 'E', 0); - } - - *start = span->objects; - *end = prev; - span->objects = curr; - SLL_SetNext(*end, NULL); - span->refcount += result; - counter_ -= result; - return result; -} - -// Fetch memory from the system and add to the central cache freelist. -void CentralFreeList::Populate() { - // Release central list lock while operating on pageheap - lock_.Unlock(); - const size_t npages = Static::sizemap()->class_to_pages(size_class_); - - Span* span; - { - SpinLockHolder h(Static::pageheap_lock()); - span = Static::pageheap()->New(npages); - if (span) Static::pageheap()->RegisterSizeClass(span, size_class_); - } - if (span == NULL) { - Log(kLog, __FILE__, __LINE__, - "tcmalloc: allocation failed", npages << kPageShift); - lock_.Lock(); - return; - } - ASSERT(span->length == npages); - // Cache sizeclass info eagerly. Locking is not necessary. - // (Instead of being eager, we could just replace any stale info - // about this span, but that seems to be no better in practice.) - for (int i = 0; i < npages; i++) { - Static::pageheap()->CacheSizeClass(span->start + i, size_class_); - } - - // Split the block into pieces and add to the free-list - // TODO: coloring of objects to avoid cache conflicts? - void** tail = &span->objects; - char* ptr = reinterpret_cast(span->start << kPageShift); - char* limit = ptr + (npages << kPageShift); - const size_t size = Static::sizemap()->ByteSizeForClass(size_class_); - int num = 0; - while (ptr + size <= limit) { - *tail = ptr; - tail = reinterpret_cast(ptr); - ptr += size; - num++; - } - ASSERT(ptr <= limit); - *tail = NULL; - span->refcount = 0; // No sub-object in use yet - - // Add span to list of non-empty spans - lock_.Lock(); - tcmalloc::DLL_Prepend(&nonempty_, span); - ++num_spans_; - counter_ += num; -} - -int CentralFreeList::tc_length() { - SpinLockHolder h(&lock_); - return used_slots_ * Static::sizemap()->num_objects_to_move(size_class_); -} - -size_t CentralFreeList::OverheadBytes() { - SpinLockHolder h(&lock_); - if (size_class_ == 0) { // 0 holds the 0-sized allocations - return 0; - } - const size_t pages_per_span = Static::sizemap()->class_to_pages(size_class_); - const size_t object_size = Static::sizemap()->class_to_size(size_class_); - ASSERT(object_size > 0); - const size_t overhead_per_span = (pages_per_span * kPageSize) % object_size; - return num_spans_ * overhead_per_span; -} - -} // namespace tcmalloc diff --git a/contrib/libtcmalloc/src/central_freelist.h b/contrib/libtcmalloc/src/central_freelist.h deleted file mode 100644 index 4148680d20a..00000000000 --- a/contrib/libtcmalloc/src/central_freelist.h +++ /dev/null @@ -1,211 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat - -#ifndef TCMALLOC_CENTRAL_FREELIST_H_ -#define TCMALLOC_CENTRAL_FREELIST_H_ - -#include "config.h" -#include // for size_t -#ifdef HAVE_STDINT_H -#include // for int32_t -#endif -#include "base/spinlock.h" -#include "base/thread_annotations.h" -#include "common.h" -#include "span.h" - -namespace tcmalloc { - -// Data kept per size-class in central cache. -class CentralFreeList { - public: - // A CentralFreeList may be used before its constructor runs. - // So we prevent lock_'s constructor from doing anything to the - // lock_ state. - CentralFreeList() : lock_(base::LINKER_INITIALIZED) { } - - void Init(size_t cl); - - // These methods all do internal locking. - - // Insert the specified range into the central freelist. N is the number of - // elements in the range. RemoveRange() is the opposite operation. - void InsertRange(void *start, void *end, int N); - - // Returns the actual number of fetched elements and sets *start and *end. - int RemoveRange(void **start, void **end, int N); - - // Returns the number of free objects in cache. - int length() { - SpinLockHolder h(&lock_); - return counter_; - } - - // Returns the number of free objects in the transfer cache. - int tc_length(); - - // Returns the memory overhead (internal fragmentation) attributable - // to the freelist. This is memory lost when the size of elements - // in a freelist doesn't exactly divide the page-size (an 8192-byte - // page full of 5-byte objects would have 2 bytes memory overhead). - size_t OverheadBytes(); - - // Lock/Unlock the internal SpinLock. Used on the pthread_atfork call - // to set the lock in a consistent state before the fork. - void Lock() { - lock_.Lock(); - } - - void Unlock() { - lock_.Unlock(); - } - - private: - // TransferCache is used to cache transfers of - // sizemap.num_objects_to_move(size_class) back and forth between - // thread caches and the central cache for a given size class. - struct TCEntry { - void *head; // Head of chain of objects. - void *tail; // Tail of chain of objects. - }; - - // A central cache freelist can have anywhere from 0 to kMaxNumTransferEntries - // slots to put link list chains into. -#ifdef TCMALLOC_SMALL_BUT_SLOW - // For the small memory model, the transfer cache is not used. - static const int kMaxNumTransferEntries = 0; -#else - // Starting point for the the maximum number of entries in the transfer cache. - // This actual maximum for a given size class may be lower than this - // maximum value. - static const int kMaxNumTransferEntries = 64; -#endif - - // REQUIRES: lock_ is held - // Remove object from cache and return. - // Return NULL if no free entries in cache. - int FetchFromOneSpans(int N, void **start, void **end) EXCLUSIVE_LOCKS_REQUIRED(lock_); - - // REQUIRES: lock_ is held - // Remove object from cache and return. Fetches - // from pageheap if cache is empty. Only returns - // NULL on allocation failure. - int FetchFromOneSpansSafe(int N, void **start, void **end) EXCLUSIVE_LOCKS_REQUIRED(lock_); - - // REQUIRES: lock_ is held - // Release a linked list of objects to spans. - // May temporarily release lock_. - void ReleaseListToSpans(void *start) EXCLUSIVE_LOCKS_REQUIRED(lock_); - - // REQUIRES: lock_ is held - // Release an object to spans. - // May temporarily release lock_. - void ReleaseToSpans(void* object) EXCLUSIVE_LOCKS_REQUIRED(lock_); - - // REQUIRES: lock_ is held - // Populate cache by fetching from the page heap. - // May temporarily release lock_. - void Populate() EXCLUSIVE_LOCKS_REQUIRED(lock_); - - // REQUIRES: lock is held. - // Tries to make room for a TCEntry. If the cache is full it will try to - // expand it at the cost of some other cache size. Return false if there is - // no space. - bool MakeCacheSpace() EXCLUSIVE_LOCKS_REQUIRED(lock_); - - // REQUIRES: lock_ for locked_size_class is held. - // Picks a "random" size class to steal TCEntry slot from. In reality it - // just iterates over the sizeclasses but does so without taking a lock. - // Returns true on success. - // May temporarily lock a "random" size class. - static bool EvictRandomSizeClass(int locked_size_class, bool force); - - // REQUIRES: lock_ is *not* held. - // Tries to shrink the Cache. If force is true it will relase objects to - // spans if it allows it to shrink the cache. Return false if it failed to - // shrink the cache. Decrements cache_size_ on succeess. - // May temporarily take lock_. If it takes lock_, the locked_size_class - // lock is released to keep the thread from holding two size class locks - // concurrently which could lead to a deadlock. - bool ShrinkCache(int locked_size_class, bool force) LOCKS_EXCLUDED(lock_); - - // This lock protects all the data members. cached_entries and cache_size_ - // may be looked at without holding the lock. - SpinLock lock_; - - // We keep linked lists of empty and non-empty spans. - size_t size_class_; // My size class - Span empty_; // Dummy header for list of empty spans - Span nonempty_; // Dummy header for list of non-empty spans - size_t num_spans_; // Number of spans in empty_ plus nonempty_ - size_t counter_; // Number of free objects in cache entry - - // Here we reserve space for TCEntry cache slots. Space is preallocated - // for the largest possible number of entries than any one size class may - // accumulate. Not all size classes are allowed to accumulate - // kMaxNumTransferEntries, so there is some wasted space for those size - // classes. - TCEntry tc_slots_[kMaxNumTransferEntries]; - - // Number of currently used cached entries in tc_slots_. This variable is - // updated under a lock but can be read without one. - int32_t used_slots_; - // The current number of slots for this size class. This is an - // adaptive value that is increased if there is lots of traffic - // on a given size class. - int32_t cache_size_; - // Maximum size of the cache for a given size class. - int32_t max_cache_size_; -}; - -// Pads each CentralCache object to multiple of 64 bytes. Since some -// compilers (such as MSVC) don't like it when the padding is 0, I use -// template specialization to remove the padding entirely when -// sizeof(CentralFreeList) is a multiple of 64. -template -class CentralFreeListPaddedTo : public CentralFreeList { - private: - char pad_[64 - kFreeListSizeMod64]; -}; - -template<> -class CentralFreeListPaddedTo<0> : public CentralFreeList { -}; - -class CentralFreeListPadded : public CentralFreeListPaddedTo< - sizeof(CentralFreeList) % 64> { -}; - -} // namespace tcmalloc - -#endif // TCMALLOC_CENTRAL_FREELIST_H_ diff --git a/contrib/libtcmalloc/src/common.cc b/contrib/libtcmalloc/src/common.cc deleted file mode 100644 index 313848c37b6..00000000000 --- a/contrib/libtcmalloc/src/common.cc +++ /dev/null @@ -1,275 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat - -#include // for getenv and strtol -#include "config.h" -#include "common.h" -#include "system-alloc.h" -#include "base/spinlock.h" -#include "getenv_safe.h" // TCMallocGetenvSafe - -namespace tcmalloc { - -// Define the maximum number of object per classe type to transfer between -// thread and central caches. -static int32 FLAGS_tcmalloc_transfer_num_objects; - -static const int32 kDefaultTransferNumObjecs = 512; - -// The init function is provided to explicit initialize the variable value -// from the env. var to avoid C++ global construction that might defer its -// initialization after a malloc/new call. -static inline void InitTCMallocTransferNumObjects() -{ - if (UNLIKELY(FLAGS_tcmalloc_transfer_num_objects == 0)) { - const char *envval = TCMallocGetenvSafe("TCMALLOC_TRANSFER_NUM_OBJ"); - FLAGS_tcmalloc_transfer_num_objects = !envval ? kDefaultTransferNumObjecs : - strtol(envval, NULL, 10); - } -} - -// Note: the following only works for "n"s that fit in 32-bits, but -// that is fine since we only use it for small sizes. -static inline int LgFloor(size_t n) { - int log = 0; - for (int i = 4; i >= 0; --i) { - int shift = (1 << i); - size_t x = n >> shift; - if (x != 0) { - n = x; - log += shift; - } - } - ASSERT(n == 1); - return log; -} - -int AlignmentForSize(size_t size) { - int alignment = kAlignment; - if (size > kMaxSize) { - // Cap alignment at kPageSize for large sizes. - alignment = kPageSize; - } else if (size >= 128) { - // Space wasted due to alignment is at most 1/8, i.e., 12.5%. - alignment = (1 << LgFloor(size)) / 8; - } else if (size >= kMinAlign) { - // We need an alignment of at least 16 bytes to satisfy - // requirements for some SSE types. - alignment = kMinAlign; - } - // Maximum alignment allowed is page size alignment. - if (alignment > kPageSize) { - alignment = kPageSize; - } - CHECK_CONDITION(size < kMinAlign || alignment >= kMinAlign); - CHECK_CONDITION((alignment & (alignment - 1)) == 0); - return alignment; -} - -int SizeMap::NumMoveSize(size_t size) { - if (size == 0) return 0; - // Use approx 64k transfers between thread and central caches. - int num = static_cast(64.0 * 1024.0 / size); - if (num < 2) num = 2; - - // Avoid bringing too many objects into small object free lists. - // If this value is too large: - // - We waste memory with extra objects sitting in the thread caches. - // - The central freelist holds its lock for too long while - // building a linked list of objects, slowing down the allocations - // of other threads. - // If this value is too small: - // - We go to the central freelist too often and we have to acquire - // its lock each time. - // This value strikes a balance between the constraints above. - if (num > FLAGS_tcmalloc_transfer_num_objects) - num = FLAGS_tcmalloc_transfer_num_objects; - - return num; -} - -// Initialize the mapping arrays -void SizeMap::Init() { - InitTCMallocTransferNumObjects(); - - // Do some sanity checking on add_amount[]/shift_amount[]/class_array[] - if (ClassIndex(0) != 0) { - Log(kCrash, __FILE__, __LINE__, - "Invalid class index for size 0", ClassIndex(0)); - } - if (ClassIndex(kMaxSize) >= sizeof(class_array_)) { - Log(kCrash, __FILE__, __LINE__, - "Invalid class index for kMaxSize", ClassIndex(kMaxSize)); - } - - // Compute the size classes we want to use - int sc = 1; // Next size class to assign - int alignment = kAlignment; - CHECK_CONDITION(kAlignment <= kMinAlign); - for (size_t size = kAlignment; size <= kMaxSize; size += alignment) { - alignment = AlignmentForSize(size); - CHECK_CONDITION((size % alignment) == 0); - - int blocks_to_move = NumMoveSize(size) / 4; - size_t psize = 0; - do { - psize += kPageSize; - // Allocate enough pages so leftover is less than 1/8 of total. - // This bounds wasted space to at most 12.5%. - while ((psize % size) > (psize >> 3)) { - psize += kPageSize; - } - // Continue to add pages until there are at least as many objects in - // the span as are needed when moving objects from the central - // freelists and spans to the thread caches. - } while ((psize / size) < (blocks_to_move)); - const size_t my_pages = psize >> kPageShift; - - if (sc > 1 && my_pages == class_to_pages_[sc-1]) { - // See if we can merge this into the previous class without - // increasing the fragmentation of the previous class. - const size_t my_objects = (my_pages << kPageShift) / size; - const size_t prev_objects = (class_to_pages_[sc-1] << kPageShift) - / class_to_size_[sc-1]; - if (my_objects == prev_objects) { - // Adjust last class to include this size - class_to_size_[sc-1] = size; - continue; - } - } - - // Add new class - class_to_pages_[sc] = my_pages; - class_to_size_[sc] = size; - sc++; - } - if (sc != kNumClasses) { - Log(kCrash, __FILE__, __LINE__, - "wrong number of size classes: (found vs. expected )", sc, kNumClasses); - } - - // Initialize the mapping arrays - int next_size = 0; - for (int c = 1; c < kNumClasses; c++) { - const int max_size_in_class = class_to_size_[c]; - for (int s = next_size; s <= max_size_in_class; s += kAlignment) { - class_array_[ClassIndex(s)] = c; - } - next_size = max_size_in_class + kAlignment; - } - - // Double-check sizes just to be safe - for (size_t size = 0; size <= kMaxSize;) { - const int sc = SizeClass(size); - if (sc <= 0 || sc >= kNumClasses) { - Log(kCrash, __FILE__, __LINE__, - "Bad size class (class, size)", sc, size); - } - if (sc > 1 && size <= class_to_size_[sc-1]) { - Log(kCrash, __FILE__, __LINE__, - "Allocating unnecessarily large class (class, size)", sc, size); - } - const size_t s = class_to_size_[sc]; - if (size > s || s == 0) { - Log(kCrash, __FILE__, __LINE__, - "Bad (class, size, requested)", sc, s, size); - } - if (size <= kMaxSmallSize) { - size += 8; - } else { - size += 128; - } - } - - // Initialize the num_objects_to_move array. - for (size_t cl = 1; cl < kNumClasses; ++cl) { - num_objects_to_move_[cl] = NumMoveSize(ByteSizeForClass(cl)); - } -} - -// Metadata allocator -- keeps stats about how many bytes allocated. -static uint64_t metadata_system_bytes_ = 0; -static const size_t kMetadataAllocChunkSize = 8*1024*1024; -// As ThreadCache objects are allocated with MetaDataAlloc, and also -// CACHELINE_ALIGNED, we must use the same alignment as TCMalloc_SystemAlloc. -static const size_t kMetadataAllignment = sizeof(MemoryAligner); - -static char *metadata_chunk_alloc_; -static size_t metadata_chunk_avail_; - -static SpinLock metadata_alloc_lock(SpinLock::LINKER_INITIALIZED); - -void* MetaDataAlloc(size_t bytes) { - if (bytes >= kMetadataAllocChunkSize) { - void *rv = TCMalloc_SystemAlloc(bytes, - NULL, kMetadataAllignment); - if (rv != NULL) { - metadata_system_bytes_ += bytes; - } - return rv; - } - - SpinLockHolder h(&metadata_alloc_lock); - - // the following works by essentially turning address to integer of - // log_2 kMetadataAllignment size and negating it. I.e. negated - // value + original value gets 0 and that's what we want modulo - // kMetadataAllignment. Note, we negate before masking higher bits - // off, otherwise we'd have to mask them off after negation anyways. - intptr_t alignment = -reinterpret_cast(metadata_chunk_alloc_) & (kMetadataAllignment-1); - - if (metadata_chunk_avail_ < bytes + alignment) { - size_t real_size; - void *ptr = TCMalloc_SystemAlloc(kMetadataAllocChunkSize, - &real_size, kMetadataAllignment); - if (ptr == NULL) { - return NULL; - } - - metadata_chunk_alloc_ = static_cast(ptr); - metadata_chunk_avail_ = real_size; - - alignment = 0; - } - - void *rv = static_cast(metadata_chunk_alloc_ + alignment); - bytes += alignment; - metadata_chunk_alloc_ += bytes; - metadata_chunk_avail_ -= bytes; - metadata_system_bytes_ += bytes; - return rv; -} - -uint64_t metadata_system_bytes() { return metadata_system_bytes_; } - -} // namespace tcmalloc diff --git a/contrib/libtcmalloc/src/common.h b/contrib/libtcmalloc/src/common.h deleted file mode 100644 index e8a1ba6972c..00000000000 --- a/contrib/libtcmalloc/src/common.h +++ /dev/null @@ -1,295 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat -// -// Common definitions for tcmalloc code. - -#ifndef TCMALLOC_COMMON_H_ -#define TCMALLOC_COMMON_H_ - -#include "config.h" -#include // for size_t -#ifdef HAVE_STDINT_H -#include // for uintptr_t, uint64_t -#endif -#include "internal_logging.h" // for ASSERT, etc -#include "base/basictypes.h" // for LIKELY, etc - -#ifdef HAVE_BUILTIN_EXPECT -#define LIKELY(x) __builtin_expect(!!(x), 1) -#define UNLIKELY(x) __builtin_expect(!!(x), 0) -#else -#define LIKELY(x) (x) -#define UNLIKELY(x) (x) -#endif - -// Type that can hold a page number -typedef uintptr_t PageID; - -// Type that can hold the length of a run of pages -typedef uintptr_t Length; - -//------------------------------------------------------------------- -// Configuration -//------------------------------------------------------------------- - -#if defined(TCMALLOC_ALIGN_8BYTES) -// Unless we force to use 8 bytes alignment we use an alignment of -// at least 16 bytes to statisfy requirements for some SSE types. -// Keep in mind when using the 16 bytes alignment you can have a space -// waste due alignment of 25%. (eg malloc of 24 bytes will get 32 bytes) -static const size_t kMinAlign = 8; -// Number of classes created until reach page size 128. -static const size_t kBaseClasses = 16; -#else -static const size_t kMinAlign = 16; -static const size_t kBaseClasses = 9; -#endif - -// Using large pages speeds up the execution at a cost of larger memory use. -// Deallocation may speed up by a factor as the page map gets 8x smaller, so -// lookups in the page map result in fewer L2 cache misses, which translates to -// speedup for application/platform combinations with high L2 cache pressure. -// As the number of size classes increases with large pages, we increase -// the thread cache allowance to avoid passing more free ranges to and from -// central lists. Also, larger pages are less likely to get freed. -// These two factors cause a bounded increase in memory use. -#if defined(TCMALLOC_32K_PAGES) -static const size_t kPageShift = 15; -static const size_t kNumClasses = kBaseClasses + 69; -#elif defined(TCMALLOC_64K_PAGES) -static const size_t kPageShift = 16; -static const size_t kNumClasses = kBaseClasses + 73; -#else -static const size_t kPageShift = 13; -static const size_t kNumClasses = kBaseClasses + 79; -#endif - -static const size_t kMaxThreadCacheSize = 4 << 20; - -static const size_t kPageSize = 1 << kPageShift; -static const size_t kMaxSize = 256 * 1024; -static const size_t kAlignment = 8; -static const size_t kLargeSizeClass = 0; -// For all span-lengths < kMaxPages we keep an exact-size list. -static const size_t kMaxPages = 1 << (20 - kPageShift); - -// Default bound on the total amount of thread caches. -#ifdef TCMALLOC_SMALL_BUT_SLOW -// Make the overall thread cache no bigger than that of a single thread -// for the small memory footprint case. -static const size_t kDefaultOverallThreadCacheSize = kMaxThreadCacheSize; -#else -static const size_t kDefaultOverallThreadCacheSize = 8u * kMaxThreadCacheSize; -#endif - -// Lower bound on the per-thread cache sizes -static const size_t kMinThreadCacheSize = kMaxSize * 2; - -// The number of bytes one ThreadCache will steal from another when -// the first ThreadCache is forced to Scavenge(), delaying the -// next call to Scavenge for this thread. -static const size_t kStealAmount = 1 << 16; - -// The number of times that a deallocation can cause a freelist to -// go over its max_length() before shrinking max_length(). -static const int kMaxOverages = 3; - -// Maximum length we allow a per-thread free-list to have before we -// move objects from it into the corresponding central free-list. We -// want this big to avoid locking the central free-list too often. It -// should not hurt to make this list somewhat big because the -// scavenging code will shrink it down when its contents are not in use. -static const int kMaxDynamicFreeListLength = 8192; - -static const Length kMaxValidPages = (~static_cast(0)) >> kPageShift; - -#if defined __x86_64__ -// All current and planned x86_64 processors only look at the lower 48 bits -// in virtual to physical address translation. The top 16 are thus unused. -// TODO(rus): Under what operating systems can we increase it safely to 17? -// This lets us use smaller page maps. On first allocation, a 36-bit page map -// uses only 96 KB instead of the 4.5 MB used by a 52-bit page map. -static const int kAddressBits = (sizeof(void*) < 8 ? (8 * sizeof(void*)) : 48); -#else -static const int kAddressBits = 8 * sizeof(void*); -#endif - -namespace tcmalloc { - -// Convert byte size into pages. This won't overflow, but may return -// an unreasonably large value if bytes is huge enough. -inline Length pages(size_t bytes) { - return (bytes >> kPageShift) + - ((bytes & (kPageSize - 1)) > 0 ? 1 : 0); -} - -// For larger allocation sizes, we use larger memory alignments to -// reduce the number of size classes. -int AlignmentForSize(size_t size); - -// Size-class information + mapping -class SizeMap { - private: - // Number of objects to move between a per-thread list and a central - // list in one shot. We want this to be not too small so we can - // amortize the lock overhead for accessing the central list. Making - // it too big may temporarily cause unnecessary memory wastage in the - // per-thread free list until the scavenger cleans up the list. - int num_objects_to_move_[kNumClasses]; - - //------------------------------------------------------------------- - // Mapping from size to size_class and vice versa - //------------------------------------------------------------------- - - // Sizes <= 1024 have an alignment >= 8. So for such sizes we have an - // array indexed by ceil(size/8). Sizes > 1024 have an alignment >= 128. - // So for these larger sizes we have an array indexed by ceil(size/128). - // - // We flatten both logical arrays into one physical array and use - // arithmetic to compute an appropriate index. The constants used by - // ClassIndex() were selected to make the flattening work. - // - // Examples: - // Size Expression Index - // ------------------------------------------------------- - // 0 (0 + 7) / 8 0 - // 1 (1 + 7) / 8 1 - // ... - // 1024 (1024 + 7) / 8 128 - // 1025 (1025 + 127 + (120<<7)) / 128 129 - // ... - // 32768 (32768 + 127 + (120<<7)) / 128 376 - static const int kMaxSmallSize = 1024; - static const size_t kClassArraySize = - ((kMaxSize + 127 + (120 << 7)) >> 7) + 1; - unsigned char class_array_[kClassArraySize]; - - static inline size_t SmallSizeClass(size_t s) { - return (static_cast(s) + 7) >> 3; - } - - static inline size_t LargeSizeClass(size_t s) { - return (static_cast(s) + 127 + (120 << 7)) >> 7; - } - - // Compute index of the class_array[] entry for a given size - static inline size_t ClassIndex(size_t s) { - // Use unsigned arithmetic to avoid unnecessary sign extensions. - ASSERT(0 <= s); - ASSERT(s <= kMaxSize); - if (LIKELY(s <= kMaxSmallSize)) { - return SmallSizeClass(s); - } else { - return LargeSizeClass(s); - } - } - - int NumMoveSize(size_t size); - - // Mapping from size class to max size storable in that class - size_t class_to_size_[kNumClasses]; - - // Mapping from size class to number of pages to allocate at a time - size_t class_to_pages_[kNumClasses]; - - public: - // Constructor should do nothing since we rely on explicit Init() - // call, which may or may not be called before the constructor runs. - SizeMap() { } - - // Initialize the mapping arrays - void Init(); - - inline int SizeClass(size_t size) { - return class_array_[ClassIndex(size)]; - } - - inline bool MaybeSizeClass(size_t size, size_t *size_class) { - size_t class_idx; - if (LIKELY(size <= kMaxSmallSize)) { - class_idx = SmallSizeClass(size); - } else if (size <= kMaxSize) { - class_idx = LargeSizeClass(size); - } else { - return false; - } - *size_class = class_array_[class_idx]; - return true; - } - - // Get the byte-size for a specified class - inline size_t ByteSizeForClass(size_t cl) { - return class_to_size_[cl]; - } - - // Mapping from size class to max size storable in that class - inline size_t class_to_size(size_t cl) { - return class_to_size_[cl]; - } - - // Mapping from size class to number of pages to allocate at a time - inline size_t class_to_pages(size_t cl) { - return class_to_pages_[cl]; - } - - // Number of objects to move between a per-thread list and a central - // list in one shot. We want this to be not too small so we can - // amortize the lock overhead for accessing the central list. Making - // it too big may temporarily cause unnecessary memory wastage in the - // per-thread free list until the scavenger cleans up the list. - inline int num_objects_to_move(size_t cl) { - return num_objects_to_move_[cl]; - } -}; - -// Allocates "bytes" worth of memory and returns it. Increments -// metadata_system_bytes appropriately. May return NULL if allocation -// fails. Requires pageheap_lock is held. -void* MetaDataAlloc(size_t bytes); - -// Returns the total number of bytes allocated from the system. -// Requires pageheap_lock is held. -uint64_t metadata_system_bytes(); - -// size/depth are made the same size as a pointer so that some generic -// code below can conveniently cast them back and forth to void*. -static const int kMaxStackDepth = 31; -struct StackTrace { - uintptr_t size; // Size of object - uintptr_t depth; // Number of PC values stored in array below - void* stack[kMaxStackDepth]; -}; - -} // namespace tcmalloc - -#endif // TCMALLOC_COMMON_H_ diff --git a/contrib/libtcmalloc/src/config.h b/contrib/libtcmalloc/src/config.h deleted file mode 100644 index 9f9a7a259e5..00000000000 --- a/contrib/libtcmalloc/src/config.h +++ /dev/null @@ -1,323 +0,0 @@ -/* src/config.h. Generated from config.h.in by configure. */ -/* src/config.h.in. Generated from configure.ac by autoheader. */ - - -#ifndef GPERFTOOLS_CONFIG_H_ -#define GPERFTOOLS_CONFIG_H_ - - -/* Build runtime detection for sized delete */ -/* #undef ENABLE_DYNAMIC_SIZED_DELETE */ - -/* Build sized deletion operators */ -/* #undef ENABLE_SIZED_DELETE */ - -/* Define to 1 if compiler supports __builtin_expect */ -#if _MSC_VER -#define HAVE_BUILTIN_EXPECT 0 -#else -#define HAVE_BUILTIN_EXPECT 1 -#endif - -/* Define to 1 if compiler supports __builtin_stack_pointer */ -/* #undef HAVE_BUILTIN_STACK_POINTER */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_CONFLICT_SIGNAL_H */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_CYGWIN_SIGNAL_H */ - -/* Define to 1 if you have the declaration of `backtrace', and to 0 if you - don't. */ -/* #undef HAVE_DECL_BACKTRACE */ - -/* Define to 1 if you have the declaration of `cfree', and to 0 if you don't. - */ -#define HAVE_DECL_CFREE 1 - -/* Define to 1 if you have the declaration of `memalign', and to 0 if you - don't. */ -#define HAVE_DECL_MEMALIGN 1 - -/* Define to 1 if you have the declaration of `nanosleep', and to 0 if you - don't. */ -/* #undef HAVE_DECL_NANOSLEEP */ - -/* Define to 1 if you have the declaration of `posix_memalign', and to 0 if - you don't. */ -#define HAVE_DECL_POSIX_MEMALIGN 1 - -/* Define to 1 if you have the declaration of `pvalloc', and to 0 if you - don't. */ -#define HAVE_DECL_PVALLOC 1 - -/* Define to 1 if you have the declaration of `sleep', and to 0 if you don't. - */ -/* #undef HAVE_DECL_SLEEP */ - -/* Define to 1 if you have the declaration of `uname', and to 0 if you don't. - */ -#define HAVE_DECL_UNAME 1 - -/* Define to 1 if you have the declaration of `valloc', and to 0 if you don't. - */ -#define HAVE_DECL_VALLOC 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_DLFCN_H 1 - -/* Define to 1 if the system has the type `Elf32_Versym'. */ -#define HAVE_ELF32_VERSYM 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_EXECINFO_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_FCNTL_H 1 - -/* Define to 1 if you have the header file. */ -#if !defined(__APPLE__) && !defined(__FreeBSD__) -#define HAVE_FEATURES_H 1 -#endif - -/* Define to 1 if you have the `fork' function. */ -#define HAVE_FORK 1 - -/* Define to 1 if you have the `geteuid' function. */ -#define HAVE_GETEUID 1 - -/* Define to 1 if you have the `getpagesize' function. */ -#define HAVE_GETPAGESIZE 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_GLOB_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_GRP_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_INTTYPES_H 1 - -/* Define to 1 if you have the header file. */ -//#define HAVE_LIBUNWIND_H 1 - -/* Define to 1 if you have the header file. */ -#if !defined(__APPLE__) && !defined(__FreeBSD__) -#define HAVE_LINUX_PTRACE_H 1 -#endif - -/* Define if this is Linux that has SIGEV_THREAD_ID */ -#define HAVE_LINUX_SIGEV_THREAD_ID 1 - -/* Define to 1 if you have the header file. */ -#if !defined(__FreeBSD__) -#define HAVE_MALLOC_H 1 -#endif - -/* Define to 1 if you have the header file. */ -#define HAVE_MEMORY_H 1 - -/* Define to 1 if you have a working `mmap' system call. */ -#define HAVE_MMAP 1 - -/* define if the compiler implements namespaces */ -#define HAVE_NAMESPACES 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_POLL_H 1 - -/* define if libc has program_invocation_name */ -#if !defined(__APPLE__) && !defined(__FreeBSD__) -#define HAVE_PROGRAM_INVOCATION_NAME 1 -#endif - -/* Define if you have POSIX threads libraries and header files. */ -#define HAVE_PTHREAD 1 - -/* defined to 1 if pthread symbols are exposed even without include pthread.h - */ -/* #undef HAVE_PTHREAD_DESPITE_ASKING_FOR */ - -/* Define to 1 if you have the header file. */ -#define HAVE_PWD_H 1 - -/* Define to 1 if you have the `sbrk' function. */ -#define HAVE_SBRK 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SCHED_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDINT_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDLIB_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STRINGS_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STRING_H 1 - -/* Define to 1 if the system has the type `struct mallinfo'. */ -//#if !defined(__APPLE__) && !defined(__FreeBSD__) -#if !defined(__APPLE__) -#define HAVE_STRUCT_MALLINFO 1 -#endif - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_CDEFS_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_PARAM_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_PRCTL_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_RESOURCE_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_SOCKET_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_STAT_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_SYSCALL_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_TYPES_H 1 - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_SYS_UCONTEXT_H */ - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_WAIT_H 1 - -/* Define to 1 if compiler supports __thread */ -#define HAVE_TLS 1 - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_UCONTEXT_H */ - -/* Define to 1 if you have the header file. */ -#define HAVE_UNISTD_H 1 - -/* Whether contains _Unwind_Backtrace */ -#define HAVE_UNWIND_BACKTRACE 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_UNWIND_H 1 - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_VALGRIND_H */ - -/* define if your compiler has __attribute__ */ -#define HAVE___ATTRIBUTE__ 1 - -/* Define to 1 if compiler supports __environ */ -#if !defined(__APPLE__) && !defined(__FreeBSD__) -#define HAVE___ENVIRON 1 -#endif - -/* Define to 1 if the system has the type `__int64'. */ -/* #undef HAVE___INT64 */ - -/* prefix where we look for installed files */ -#define INSTALL_PREFIX "/usr/local" - -/* Define to 1 if int32_t is equivalent to intptr_t */ -/* #undef INT32_EQUALS_INTPTR */ - -/* Define to the sub-directory in which libtool stores uninstalled libraries. - */ -#define LT_OBJDIR ".libs/" - -/* Name of package */ -#define PACKAGE "gperftools" - -/* Define to the address where bug reports for this package should be sent. */ -#define PACKAGE_BUGREPORT "gperftools@googlegroups.com" - -/* Define to the full name of this package. */ -#define PACKAGE_NAME "gperftools" - -/* Define to the full name and version of this package. */ -#define PACKAGE_STRING "gperftools 2.5" - -/* Define to the one symbol short name of this package. */ -#define PACKAGE_TARNAME "gperftools" - -/* Define to the home page for this package. */ -#define PACKAGE_URL "" - -/* Define to the version of this package. */ -#define PACKAGE_VERSION "2.5" - -/* How to access the PC from a struct ucontext */ -/* #undef PC_FROM_UCONTEXT */ - -/* Always the empty-string on non-windows systems. On windows, should be - "__declspec(dllexport)". This way, when we compile the dll, we export our - functions/classes. It's safe to define this here because config.h is only - used internally, to compile the DLL, and every DLL source file #includes - "config.h" before anything else. */ -#define PERFTOOLS_DLL_DECL /**/ - -/* printf format code for printing a size_t and ssize_t */ -#define PRIdS "ld" - -/* printf format code for printing a size_t and ssize_t */ -#define PRIuS "lu" - -/* printf format code for printing a size_t and ssize_t */ -#define PRIxS "lx" - -/* Mark the systems where we know it's bad if pthreads runs too - early before main (before threads are initialized, presumably). */ -#ifdef __FreeBSD__ -#define PTHREADS_CRASHES_IF_RUN_TOO_EARLY 1 -#endif - -/* Define to necessary symbol if this constant uses a non-standard name on - your system. */ -/* #undef PTHREAD_CREATE_JOINABLE */ - -/* Define to 1 if you have the ANSI C header files. */ -#define STDC_HEADERS 1 - -/* the namespace where STL code like vector<> is defined */ -#define STL_NAMESPACE std - -/* Define 32K of internal pages size for tcmalloc */ -/* #undef TCMALLOC_32K_PAGES */ - -/* Define 64K of internal pages size for tcmalloc */ -/* #undef TCMALLOC_64K_PAGES */ - -/* Define 8 bytes of allocation alignment for tcmalloc */ -/* #undef TCMALLOC_ALIGN_8BYTES */ - -/* Version number of package */ -#define VERSION "2.5" - -/* C99 says: define this to get the PRI... macros from stdint.h */ -#ifndef __STDC_FORMAT_MACROS -# define __STDC_FORMAT_MACROS 1 -#endif - -/* Define to `__inline__' or `__inline' if that's what the C compiler - calls it, or to nothing if 'inline' is not supported under any name. */ -#ifndef __cplusplus -/* #undef inline */ -#endif - - -#ifdef __MINGW32__ -#include "windows/mingw.h" -#endif - -#endif /* #ifndef GPERFTOOLS_CONFIG_H_ */ - diff --git a/contrib/libtcmalloc/src/debugallocation.cc b/contrib/libtcmalloc/src/debugallocation.cc deleted file mode 100644 index 178809bc8a3..00000000000 --- a/contrib/libtcmalloc/src/debugallocation.cc +++ /dev/null @@ -1,1500 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2000, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Urs Holzle - -#include "config.h" -#include -#ifdef HAVE_FCNTL_H -#include -#endif -#ifdef HAVE_INTTYPES_H -#include -#endif -// We only need malloc.h for struct mallinfo. -#ifdef HAVE_STRUCT_MALLINFO -// Malloc can be in several places on older versions of OS X. -# if defined(HAVE_MALLOC_H) -# include -# elif defined(HAVE_MALLOC_MALLOC_H) -# include -# elif defined(HAVE_SYS_MALLOC_H) -# include -# endif -#endif -#ifdef HAVE_PTHREAD -#include -#endif -#include -#include -#include -#ifdef HAVE_MMAP -#include -#endif -#include -#include -#ifdef HAVE_UNISTD_H -#include -#endif - -#include -#include -#include -#include "addressmap-inl.h" -#include "base/commandlineflags.h" -#include "base/googleinit.h" -#include "base/logging.h" -#include "base/spinlock.h" -#include "malloc_hook-inl.h" -#include "symbolize.h" - -// NOTE: due to #define below, tcmalloc.cc will omit tc_XXX -// definitions. So that debug implementations can be defined -// instead. We're going to use do_malloc, do_free and other do_XXX -// functions that are defined in tcmalloc.cc for actual memory -// management -#define TCMALLOC_USING_DEBUGALLOCATION -#include "tcmalloc.cc" - -// __THROW is defined in glibc systems. It means, counter-intuitively, -// "This function will never throw an exception." It's an optional -// optimization tool, but we may need to use it to match glibc prototypes. -#ifndef __THROW // I guess we're not on a glibc system -# define __THROW // __THROW is just an optimization, so ok to make it "" -#endif - -// On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old -// form of the name instead. -#ifndef MAP_ANONYMOUS -# define MAP_ANONYMOUS MAP_ANON -#endif - -#pragma GCC diagnostic push -#ifdef __clang__ -#pragma GCC diagnostic ignored "-Wunused-private-field" -#pragma GCC diagnostic ignored "-Wgnu-alignof-expression" -#endif - -// ========================================================================= // - -DEFINE_bool(malloctrace, - EnvToBool("TCMALLOC_TRACE", false), - "Enables memory (de)allocation tracing to /tmp/google.alloc."); -#ifdef HAVE_MMAP -DEFINE_bool(malloc_page_fence, - EnvToBool("TCMALLOC_PAGE_FENCE", false), - "Enables putting of memory allocations at page boundaries " - "with a guard page following the allocation (to catch buffer " - "overruns right when they happen)."); -DEFINE_bool(malloc_page_fence_never_reclaim, - EnvToBool("TCMALLOC_PAGE_FRANCE_NEVER_RECLAIM", false), - "Enables making the virtual address space inaccessible " - "upon a deallocation instead of returning it and reusing later."); -#else -DEFINE_bool(malloc_page_fence, false, "Not usable (requires mmap)"); -DEFINE_bool(malloc_page_fence_never_reclaim, false, "Not usable (required mmap)"); -#endif -DEFINE_bool(malloc_reclaim_memory, - EnvToBool("TCMALLOC_RECLAIM_MEMORY", true), - "If set to false, we never return memory to malloc " - "when an object is deallocated. This ensures that all " - "heap object addresses are unique."); -DEFINE_int32(max_free_queue_size, - EnvToInt("TCMALLOC_MAX_FREE_QUEUE_SIZE", 10*1024*1024), - "If greater than 0, keep freed blocks in a queue instead of " - "releasing them to the allocator immediately. Release them when " - "the total size of all blocks in the queue would otherwise exceed " - "this limit."); - -DEFINE_bool(symbolize_stacktrace, - EnvToBool("TCMALLOC_SYMBOLIZE_STACKTRACE", true), - "Symbolize the stack trace when provided (on some error exits)"); - -// If we are LD_PRELOAD-ed against a non-pthreads app, then -// pthread_once won't be defined. We declare it here, for that -// case (with weak linkage) which will cause the non-definition to -// resolve to NULL. We can then check for NULL or not in Instance. -extern "C" int pthread_once(pthread_once_t *, void (*)(void)) - ATTRIBUTE_WEAK; - -// ========================================================================= // - -// A safe version of printf() that does not do any allocation and -// uses very little stack space. -static void TracePrintf(int fd, const char *fmt, ...) - __attribute__ ((__format__ (__printf__, 2, 3))); - -// Round "value" up to next "alignment" boundary. -// Requires that "alignment" be a power of two. -static intptr_t RoundUp(intptr_t value, intptr_t alignment) { - return (value + alignment - 1) & ~(alignment - 1); -} - -// ========================================================================= // - -class MallocBlock; - -// A circular buffer to hold freed blocks of memory. MallocBlock::Deallocate -// (below) pushes blocks into this queue instead of returning them to the -// underlying allocator immediately. See MallocBlock::Deallocate for more -// information. -// -// We can't use an STL class for this because we need to be careful not to -// perform any heap de-allocations in any of the code in this class, since the -// code in MallocBlock::Deallocate is not re-entrant. -template -class FreeQueue { - public: - FreeQueue() : q_front_(0), q_back_(0) {} - - bool Full() { - return (q_front_ + 1) % kFreeQueueSize == q_back_; - } - - void Push(const QueueEntry& block) { - q_[q_front_] = block; - q_front_ = (q_front_ + 1) % kFreeQueueSize; - } - - QueueEntry Pop() { - RAW_CHECK(q_back_ != q_front_, "Queue is empty"); - const QueueEntry& ret = q_[q_back_]; - q_back_ = (q_back_ + 1) % kFreeQueueSize; - return ret; - } - - size_t size() const { - return (q_front_ - q_back_ + kFreeQueueSize) % kFreeQueueSize; - } - - private: - // Maximum number of blocks kept in the free queue before being freed. - static const int kFreeQueueSize = 1024; - - QueueEntry q_[kFreeQueueSize]; - int q_front_; - int q_back_; -}; - -struct MallocBlockQueueEntry { - MallocBlockQueueEntry() : block(NULL), size(0), - num_deleter_pcs(0), deleter_threadid(0) {} - MallocBlockQueueEntry(MallocBlock* b, size_t s) : block(b), size(s) { - if (FLAGS_max_free_queue_size != 0 && b != NULL) { - // Adjust the number of frames to skip (4) if you change the - // location of this call. - num_deleter_pcs = - GetStackTrace(deleter_pcs, - sizeof(deleter_pcs) / sizeof(deleter_pcs[0]), - 4); - deleter_threadid = pthread_self(); - } else { - num_deleter_pcs = 0; - // Zero is an illegal pthread id by my reading of the pthread - // implementation: - deleter_threadid = 0; - } - } - - MallocBlock* block; - size_t size; - - // When deleted and put in the free queue, we (flag-controlled) - // record the stack so that if corruption is later found, we can - // print the deleter's stack. (These three vars add 144 bytes of - // overhead under the LP64 data model.) - void* deleter_pcs[16]; - int num_deleter_pcs; - pthread_t deleter_threadid; -}; - -class MallocBlock { - public: // allocation type constants - - // Different allocation types we distinguish. - // Note: The lower 4 bits are not random: we index kAllocName array - // by these values masked with kAllocTypeMask; - // the rest are "random" magic bits to help catch memory corruption. - static const int kMallocType = 0xEFCDAB90; - static const int kNewType = 0xFEBADC81; - static const int kArrayNewType = 0xBCEADF72; - - private: // constants - - // A mask used on alloc types above to get to 0, 1, 2 - static const int kAllocTypeMask = 0x3; - // An additional bit to set in AllocType constants - // to mark now deallocated regions. - static const int kDeallocatedTypeBit = 0x4; - - // For better memory debugging, we initialize all storage to known - // values, and overwrite the storage when it's deallocated: - // Byte that fills uninitialized storage. - static const int kMagicUninitializedByte = 0xAB; - // Byte that fills deallocated storage. - // NOTE: tcmalloc.cc depends on the value of kMagicDeletedByte - // to work around a bug in the pthread library. - static const int kMagicDeletedByte = 0xCD; - // A size_t (type of alloc_type_ below) in a deallocated storage - // filled with kMagicDeletedByte. - static const size_t kMagicDeletedSizeT = - 0xCDCDCDCD | (((size_t)0xCDCDCDCD << 16) << 16); - // Initializer works for 32 and 64 bit size_ts; - // "<< 16 << 16" is to fool gcc from issuing a warning - // when size_ts are 32 bits. - - // NOTE: on Linux, you can enable malloc debugging support in libc by - // setting the environment variable MALLOC_CHECK_ to 1 before you - // start the program (see man malloc). - - // We use either do_malloc or mmap to make the actual allocation. In - // order to remember which one of the two was used for any block, we store an - // appropriate magic word next to the block. - static const size_t kMagicMalloc = 0xDEADBEEF; - static const size_t kMagicMMap = 0xABCDEFAB; - - // This array will be filled with 0xCD, for use with memcmp. - static unsigned char kMagicDeletedBuffer[1024]; - static pthread_once_t deleted_buffer_initialized_; - static bool deleted_buffer_initialized_no_pthreads_; - - private: // data layout - - // The four fields size1_,offset_,magic1_,alloc_type_ - // should together occupy a multiple of 16 bytes. (At the - // moment, sizeof(size_t) == 4 or 8 depending on piii vs - // k8, and 4 of those sum to 16 or 32 bytes). - // This, combined with do_malloc's alignment guarantees, - // ensures that SSE types can be stored into the returned - // block, at &size2_. - size_t size1_; - size_t offset_; // normally 0 unless memaligned memory - // see comments in memalign() and FromRawPointer(). - size_t magic1_; - size_t alloc_type_; - // here comes the actual data (variable length) - // ... - // then come the size2_ and magic2_, or a full page of mprotect-ed memory - // if the malloc_page_fence feature is enabled. - size_t size2_; - size_t magic2_; - - private: // static data and helpers - - // Allocation map: stores the allocation type for each allocated object, - // or the type or'ed with kDeallocatedTypeBit - // for each formerly allocated object. - typedef AddressMap AllocMap; - static AllocMap* alloc_map_; - // This protects alloc_map_ and consistent state of metadata - // for each still-allocated object in it. - // We use spin locks instead of pthread_mutex_t locks - // to prevent crashes via calls to pthread_mutex_(un)lock - // for the (de)allocations coming from pthreads initialization itself. - static SpinLock alloc_map_lock_; - - // A queue of freed blocks. Instead of releasing blocks to the allocator - // immediately, we put them in a queue, freeing them only when necessary - // to keep the total size of all the freed blocks below the limit set by - // FLAGS_max_free_queue_size. - static FreeQueue* free_queue_; - - static size_t free_queue_size_; // total size of blocks in free_queue_ - // protects free_queue_ and free_queue_size_ - static SpinLock free_queue_lock_; - - // Names of allocation types (kMallocType, kNewType, kArrayNewType) - static const char* const kAllocName[]; - // Names of corresponding deallocation types - static const char* const kDeallocName[]; - - static const char* AllocName(int type) { - return kAllocName[type & kAllocTypeMask]; - } - - static const char* DeallocName(int type) { - return kDeallocName[type & kAllocTypeMask]; - } - - private: // helper accessors - - bool IsMMapped() const { return kMagicMMap == magic1_; } - - bool IsValidMagicValue(size_t value) const { - return kMagicMMap == value || kMagicMalloc == value; - } - - static size_t real_malloced_size(size_t size) { - return size + sizeof(MallocBlock); - } - - /* - * Here we assume size of page is kMinAlign aligned, - * so if size is MALLOC_ALIGNMENT aligned too, then we could - * guarantee return address is also kMinAlign aligned, because - * mmap return address at nearby page boundary on Linux. - */ - static size_t real_mmapped_size(size_t size) { - size_t tmp = size + MallocBlock::data_offset(); - tmp = RoundUp(tmp, kMinAlign); - return tmp; - } - - size_t real_size() { - return IsMMapped() ? real_mmapped_size(size1_) : real_malloced_size(size1_); - } - - // NOTE: if the block is mmapped (that is, we're using the - // malloc_page_fence option) then there's no size2 or magic2 - // (instead, the guard page begins where size2 would be). - - size_t* size2_addr() { return (size_t*)((char*)&size2_ + size1_); } - const size_t* size2_addr() const { - return (const size_t*)((char*)&size2_ + size1_); - } - - size_t* magic2_addr() { return (size_t*)(size2_addr() + 1); } - const size_t* magic2_addr() const { return (const size_t*)(size2_addr() + 1); } - - private: // other helpers - - void Initialize(size_t size, int type) { - RAW_CHECK(IsValidMagicValue(magic1_), ""); - // record us as allocated in the map - alloc_map_lock_.Lock(); - if (!alloc_map_) { - void* p = do_malloc(sizeof(AllocMap)); - alloc_map_ = new(p) AllocMap(do_malloc, do_free); - } - alloc_map_->Insert(data_addr(), type); - // initialize us - size1_ = size; - offset_ = 0; - alloc_type_ = type; - if (!IsMMapped()) { - bit_store(magic2_addr(), &magic1_); - bit_store(size2_addr(), &size); - } - alloc_map_lock_.Unlock(); - memset(data_addr(), kMagicUninitializedByte, size); - if (!IsMMapped()) { - RAW_CHECK(memcmp(&size1_, size2_addr(), sizeof(size1_)) == 0, "should hold"); - RAW_CHECK(memcmp(&magic1_, magic2_addr(), sizeof(magic1_)) == 0, "should hold"); - } - } - - size_t CheckAndClear(int type, size_t given_size) { - alloc_map_lock_.Lock(); - CheckLocked(type); - if (!IsMMapped()) { - RAW_CHECK(memcmp(&size1_, size2_addr(), sizeof(size1_)) == 0, "should hold"); - } - // record us as deallocated in the map - alloc_map_->Insert(data_addr(), type | kDeallocatedTypeBit); - alloc_map_lock_.Unlock(); - // clear us - const size_t size = real_size(); - RAW_CHECK(!given_size || given_size == size1_, - "right size must be passed to sized delete"); - memset(this, kMagicDeletedByte, size); - return size; - } - - void CheckLocked(int type) const { - int map_type = 0; - const int* found_type = - alloc_map_ != NULL ? alloc_map_->Find(data_addr()) : NULL; - if (found_type == NULL) { - RAW_LOG(FATAL, "memory allocation bug: object at %p " - "has never been allocated", data_addr()); - } else { - map_type = *found_type; - } - if ((map_type & kDeallocatedTypeBit) != 0) { - RAW_LOG(FATAL, "memory allocation bug: object at %p " - "has been already deallocated (it was allocated with %s)", - data_addr(), AllocName(map_type & ~kDeallocatedTypeBit)); - } - if (alloc_type_ == kMagicDeletedSizeT) { - RAW_LOG(FATAL, "memory stomping bug: a word before object at %p " - "has been corrupted; or else the object has been already " - "deallocated and our memory map has been corrupted", - data_addr()); - } - if (!IsValidMagicValue(magic1_)) { - RAW_LOG(FATAL, "memory stomping bug: a word before object at %p " - "has been corrupted; " - "or else our memory map has been corrupted and this is a " - "deallocation for not (currently) heap-allocated object", - data_addr()); - } - if (!IsMMapped()) { - if (memcmp(&size1_, size2_addr(), sizeof(size1_))) { - RAW_LOG(FATAL, "memory stomping bug: a word after object at %p " - "has been corrupted", data_addr()); - } - size_t addr; - bit_store(&addr, magic2_addr()); - if (!IsValidMagicValue(addr)) { - RAW_LOG(FATAL, "memory stomping bug: a word after object at %p " - "has been corrupted", data_addr()); - } - } - if (alloc_type_ != type) { - if ((alloc_type_ != MallocBlock::kMallocType) && - (alloc_type_ != MallocBlock::kNewType) && - (alloc_type_ != MallocBlock::kArrayNewType)) { - RAW_LOG(FATAL, "memory stomping bug: a word before object at %p " - "has been corrupted", data_addr()); - } - RAW_LOG(FATAL, "memory allocation/deallocation mismatch at %p: " - "allocated with %s being deallocated with %s", - data_addr(), AllocName(alloc_type_), DeallocName(type)); - } - if (alloc_type_ != map_type) { - RAW_LOG(FATAL, "memory stomping bug: our memory map has been corrupted : " - "allocation at %p made with %s " - "is recorded in the map to be made with %s", - data_addr(), AllocName(alloc_type_), AllocName(map_type)); - } - } - - public: // public accessors - - void* data_addr() { return (void*)&size2_; } - const void* data_addr() const { return (const void*)&size2_; } - - static size_t data_offset() { return OFFSETOF_MEMBER(MallocBlock, size2_); } - - size_t data_size() const { return size1_; } - - void set_offset(int offset) { this->offset_ = offset; } - - public: // our main interface - - static MallocBlock* Allocate(size_t size, int type) { - // Prevent an integer overflow / crash with large allocation sizes. - // TODO - Note that for a e.g. 64-bit size_t, max_size_t may not actually - // be the maximum value, depending on how the compiler treats ~0. The worst - // practical effect is that allocations are limited to 4Gb or so, even if - // the address space could take more. - static size_t max_size_t = ~0; - if (size > max_size_t - sizeof(MallocBlock)) { - RAW_LOG(ERROR, "Massive size passed to malloc: %" PRIuS "", size); - return NULL; - } - MallocBlock* b = NULL; - const bool use_malloc_page_fence = FLAGS_malloc_page_fence; -#ifdef HAVE_MMAP - if (use_malloc_page_fence) { - // Put the block towards the end of the page and make the next page - // inaccessible. This will catch buffer overrun right when it happens. - size_t sz = real_mmapped_size(size); - int pagesize = getpagesize(); - int num_pages = (sz + pagesize - 1) / pagesize + 1; - char* p = (char*) mmap(NULL, num_pages * pagesize, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); - if (p == MAP_FAILED) { - // If the allocation fails, abort rather than returning NULL to - // malloc. This is because in most cases, the program will run out - // of memory in this mode due to tremendous amount of wastage. There - // is no point in propagating the error elsewhere. - RAW_LOG(FATAL, "Out of memory: possibly due to page fence overhead: %s", - strerror(errno)); - } - // Mark the page after the block inaccessible - if (mprotect(p + (num_pages - 1) * pagesize, pagesize, PROT_NONE)) { - RAW_LOG(FATAL, "Guard page setup failed: %s", strerror(errno)); - } - b = (MallocBlock*) (p + (num_pages - 1) * pagesize - sz); - } else { - b = (MallocBlock*) do_malloc(real_malloced_size(size)); - } -#else - b = (MallocBlock*) do_malloc(real_malloced_size(size)); -#endif - - // It would be nice to output a diagnostic on allocation failure - // here, but logging (other than FATAL) requires allocating - // memory, which could trigger a nasty recursion. Instead, preserve - // malloc semantics and return NULL on failure. - if (b != NULL) { - b->magic1_ = use_malloc_page_fence ? kMagicMMap : kMagicMalloc; - b->Initialize(size, type); - } - return b; - } - - void Deallocate(int type, size_t given_size) { - if (IsMMapped()) { // have to do this before CheckAndClear -#ifdef HAVE_MMAP - int size = CheckAndClear(type, given_size); - int pagesize = getpagesize(); - int num_pages = (size + pagesize - 1) / pagesize + 1; - char* p = (char*) this; - if (FLAGS_malloc_page_fence_never_reclaim || - !FLAGS_malloc_reclaim_memory) { - mprotect(p - (num_pages - 1) * pagesize + size, - num_pages * pagesize, PROT_NONE); - } else { - munmap(p - (num_pages - 1) * pagesize + size, num_pages * pagesize); - } -#endif - } else { - const size_t size = CheckAndClear(type, given_size); - if (FLAGS_malloc_reclaim_memory) { - // Instead of freeing the block immediately, push it onto a queue of - // recently freed blocks. Free only enough blocks to keep from - // exceeding the capacity of the queue or causing the total amount of - // un-released memory in the queue from exceeding - // FLAGS_max_free_queue_size. - ProcessFreeQueue(this, size, FLAGS_max_free_queue_size); - } - } - } - - static size_t FreeQueueSize() { - SpinLockHolder l(&free_queue_lock_); - return free_queue_size_; - } - - static void ProcessFreeQueue(MallocBlock* b, size_t size, - int max_free_queue_size) { - // MallocBlockQueueEntry are about 144 in size, so we can only - // use a small array of them on the stack. - MallocBlockQueueEntry entries[4]; - int num_entries = 0; - MallocBlockQueueEntry new_entry(b, size); - free_queue_lock_.Lock(); - if (free_queue_ == NULL) - free_queue_ = new FreeQueue; - RAW_CHECK(!free_queue_->Full(), "Free queue mustn't be full!"); - - if (b != NULL) { - free_queue_size_ += size + sizeof(MallocBlockQueueEntry); - free_queue_->Push(new_entry); - } - - // Free blocks until the total size of unfreed blocks no longer exceeds - // max_free_queue_size, and the free queue has at least one free - // space in it. - while (free_queue_size_ > max_free_queue_size || free_queue_->Full()) { - RAW_CHECK(num_entries < arraysize(entries), "entries array overflow"); - entries[num_entries] = free_queue_->Pop(); - free_queue_size_ -= - entries[num_entries].size + sizeof(MallocBlockQueueEntry); - num_entries++; - if (num_entries == arraysize(entries)) { - // The queue will not be full at this point, so it is ok to - // release the lock. The queue may still contain more than - // max_free_queue_size, but this is not a strict invariant. - free_queue_lock_.Unlock(); - for (int i = 0; i < num_entries; i++) { - CheckForDanglingWrites(entries[i]); - do_free(entries[i].block); - } - num_entries = 0; - free_queue_lock_.Lock(); - } - } - RAW_CHECK(free_queue_size_ >= 0, "Free queue size went negative!"); - free_queue_lock_.Unlock(); - for (int i = 0; i < num_entries; i++) { - CheckForDanglingWrites(entries[i]); - do_free(entries[i].block); - } - } - - static void InitDeletedBuffer() { - memset(kMagicDeletedBuffer, kMagicDeletedByte, sizeof(kMagicDeletedBuffer)); - deleted_buffer_initialized_no_pthreads_ = true; - } - - static void CheckForDanglingWrites(const MallocBlockQueueEntry& queue_entry) { - // Initialize the buffer if necessary. - if (pthread_once) - pthread_once(&deleted_buffer_initialized_, &InitDeletedBuffer); - if (!deleted_buffer_initialized_no_pthreads_) { - // This will be the case on systems that don't link in pthreads, - // including on FreeBSD where pthread_once has a non-zero address - // (but doesn't do anything) even when pthreads isn't linked in. - InitDeletedBuffer(); - } - - const unsigned char* p = - reinterpret_cast(queue_entry.block); - - static const size_t size_of_buffer = sizeof(kMagicDeletedBuffer); - const size_t size = queue_entry.size; - const size_t buffers = size / size_of_buffer; - const size_t remainder = size % size_of_buffer; - size_t buffer_idx; - for (buffer_idx = 0; buffer_idx < buffers; ++buffer_idx) { - CheckForCorruptedBuffer(queue_entry, buffer_idx, p, size_of_buffer); - p += size_of_buffer; - } - CheckForCorruptedBuffer(queue_entry, buffer_idx, p, remainder); - } - - static void CheckForCorruptedBuffer(const MallocBlockQueueEntry& queue_entry, - size_t buffer_idx, - const unsigned char* buffer, - size_t size_of_buffer) { - if (memcmp(buffer, kMagicDeletedBuffer, size_of_buffer) == 0) { - return; - } - - RAW_LOG(ERROR, - "Found a corrupted memory buffer in MallocBlock (may be offset " - "from user ptr): buffer index: %zd, buffer ptr: %p, size of " - "buffer: %zd", buffer_idx, buffer, size_of_buffer); - - // The magic deleted buffer should only be 1024 bytes, but in case - // this changes, let's put an upper limit on the number of debug - // lines we'll output: - if (size_of_buffer <= 1024) { - for (int i = 0; i < size_of_buffer; ++i) { - if (buffer[i] != kMagicDeletedByte) { - RAW_LOG(ERROR, "Buffer byte %d is 0x%02x (should be 0x%02x).", - i, buffer[i], kMagicDeletedByte); - } - } - } else { - RAW_LOG(ERROR, "Buffer too large to print corruption."); - } - - const MallocBlock* b = queue_entry.block; - const size_t size = queue_entry.size; - if (queue_entry.num_deleter_pcs > 0) { - TracePrintf(STDERR_FILENO, "Deleted by thread %p\n", - reinterpret_cast( - PRINTABLE_PTHREAD(queue_entry.deleter_threadid))); - - // We don't want to allocate or deallocate memory here, so we use - // placement-new. It's ok that we don't destroy this, since we're - // just going to error-exit below anyway. Union is for alignment. - union { void* alignment; char buf[sizeof(SymbolTable)]; } tablebuf; - SymbolTable* symbolization_table = new (tablebuf.buf) SymbolTable; - for (int i = 0; i < queue_entry.num_deleter_pcs; i++) { - // Symbolizes the previous address of pc because pc may be in the - // next function. This may happen when the function ends with - // a call to a function annotated noreturn (e.g. CHECK). - char *pc = reinterpret_cast(queue_entry.deleter_pcs[i]); - symbolization_table->Add(pc - 1); - } - if (FLAGS_symbolize_stacktrace) - symbolization_table->Symbolize(); - for (int i = 0; i < queue_entry.num_deleter_pcs; i++) { - char *pc = reinterpret_cast(queue_entry.deleter_pcs[i]); - TracePrintf(STDERR_FILENO, " @ %p %s\n", - pc, symbolization_table->GetSymbol(pc - 1)); - } - } else { - RAW_LOG(ERROR, - "Skipping the printing of the deleter's stack! Its stack was " - "not found; either the corruption occurred too early in " - "execution to obtain a stack trace or --max_free_queue_size was " - "set to 0."); - } - - RAW_LOG(FATAL, - "Memory was written to after being freed. MallocBlock: %p, user " - "ptr: %p, size: %zd. If you can't find the source of the error, " - "try using ASan (http://code.google.com/p/address-sanitizer/), " - "Valgrind, or Purify, or study the " - "output of the deleter's stack printed above.", - b, b->data_addr(), size); - } - - static MallocBlock* FromRawPointer(void* p) { - const size_t data_offset = MallocBlock::data_offset(); - // Find the header just before client's memory. - MallocBlock *mb = reinterpret_cast( - reinterpret_cast(p) - data_offset); - // If mb->alloc_type_ is kMagicDeletedSizeT, we're not an ok pointer. - if (mb->alloc_type_ == kMagicDeletedSizeT) { - RAW_LOG(FATAL, "memory allocation bug: object at %p has been already" - " deallocated; or else a word before the object has been" - " corrupted (memory stomping bug)", p); - } - // If mb->offset_ is zero (common case), mb is the real header. - // If mb->offset_ is non-zero, this block was allocated by debug - // memallign implementation, and mb->offset_ is the distance - // backwards to the real header from mb, which is a fake header. - if (mb->offset_ == 0) { - return mb; - } - - MallocBlock *main_block = reinterpret_cast( - reinterpret_cast(mb) - mb->offset_); - - if (main_block->offset_ != 0) { - RAW_LOG(FATAL, "memory corruption bug: offset_ field is corrupted." - " Need 0 but got %x", - (unsigned)(main_block->offset_)); - } - if (main_block >= p) { - RAW_LOG(FATAL, "memory corruption bug: offset_ field is corrupted." - " Detected main_block address overflow: %x", - (unsigned)(mb->offset_)); - } - if (main_block->size2_addr() < p) { - RAW_LOG(FATAL, "memory corruption bug: offset_ field is corrupted." - " It points below it's own main_block: %x", - (unsigned)(mb->offset_)); - } - - return main_block; - } - - static const MallocBlock* FromRawPointer(const void* p) { - // const-safe version: we just cast about - return FromRawPointer(const_cast(p)); - } - - void Check(int type) const { - alloc_map_lock_.Lock(); - CheckLocked(type); - alloc_map_lock_.Unlock(); - } - - static bool CheckEverything() { - alloc_map_lock_.Lock(); - if (alloc_map_ != NULL) alloc_map_->Iterate(CheckCallback, 0); - alloc_map_lock_.Unlock(); - return true; // if we get here, we're okay - } - - static bool MemoryStats(int* blocks, size_t* total, - int histogram[kMallocHistogramSize]) { - memset(histogram, 0, kMallocHistogramSize * sizeof(int)); - alloc_map_lock_.Lock(); - stats_blocks_ = 0; - stats_total_ = 0; - stats_histogram_ = histogram; - if (alloc_map_ != NULL) alloc_map_->Iterate(StatsCallback, 0); - *blocks = stats_blocks_; - *total = stats_total_; - alloc_map_lock_.Unlock(); - return true; - } - - private: // helpers for CheckEverything and MemoryStats - - static void CheckCallback(const void* ptr, int* type, int dummy) { - if ((*type & kDeallocatedTypeBit) == 0) { - FromRawPointer(ptr)->CheckLocked(*type); - } - } - - // Accumulation variables for StatsCallback protected by alloc_map_lock_ - static int stats_blocks_; - static size_t stats_total_; - static int* stats_histogram_; - - static void StatsCallback(const void* ptr, int* type, int dummy) { - if ((*type & kDeallocatedTypeBit) == 0) { - const MallocBlock* b = FromRawPointer(ptr); - b->CheckLocked(*type); - ++stats_blocks_; - size_t mysize = b->size1_; - int entry = 0; - stats_total_ += mysize; - while (mysize) { - ++entry; - mysize >>= 1; - } - RAW_CHECK(entry < kMallocHistogramSize, - "kMallocHistogramSize should be at least as large as log2 " - "of the maximum process memory size"); - stats_histogram_[entry] += 1; - } - } -}; - -void DanglingWriteChecker() { - // Clear out the remaining free queue to check for dangling writes. - MallocBlock::ProcessFreeQueue(NULL, 0, 0); -} - -// ========================================================================= // - -const size_t MallocBlock::kMagicMalloc; -const size_t MallocBlock::kMagicMMap; - -MallocBlock::AllocMap* MallocBlock::alloc_map_ = NULL; -SpinLock MallocBlock::alloc_map_lock_(SpinLock::LINKER_INITIALIZED); - -FreeQueue* MallocBlock::free_queue_ = NULL; -size_t MallocBlock::free_queue_size_ = 0; -SpinLock MallocBlock::free_queue_lock_(SpinLock::LINKER_INITIALIZED); - -unsigned char MallocBlock::kMagicDeletedBuffer[1024]; -pthread_once_t MallocBlock::deleted_buffer_initialized_ = PTHREAD_ONCE_INIT; -bool MallocBlock::deleted_buffer_initialized_no_pthreads_ = false; - -const char* const MallocBlock::kAllocName[] = { - "malloc", - "new", - "new []", - NULL, -}; - -const char* const MallocBlock::kDeallocName[] = { - "free", - "delete", - "delete []", - NULL, -}; - -int MallocBlock::stats_blocks_; -size_t MallocBlock::stats_total_; -int* MallocBlock::stats_histogram_; - -// ========================================================================= // - -// The following cut-down version of printf() avoids -// using stdio or ostreams. -// This is to guarantee no recursive calls into -// the allocator and to bound the stack space consumed. (The pthread -// manager thread in linuxthreads has a very small stack, -// so fprintf can't be called.) -static void TracePrintf(int fd, const char *fmt, ...) { - char buf[64]; - int i = 0; - va_list ap; - va_start(ap, fmt); - const char *p = fmt; - char numbuf[25]; - if (fd < 0) { - return; - } - numbuf[sizeof(numbuf)-1] = 0; - while (*p != '\0') { // until end of format string - char *s = &numbuf[sizeof(numbuf)-1]; - if (p[0] == '%' && p[1] != 0) { // handle % formats - int64 l = 0; - unsigned long base = 0; - if (*++p == 's') { // %s - s = va_arg(ap, char *); - } else if (*p == 'l' && p[1] == 'd') { // %ld - l = va_arg(ap, long); - base = 10; - p++; - } else if (*p == 'l' && p[1] == 'u') { // %lu - l = va_arg(ap, unsigned long); - base = 10; - p++; - } else if (*p == 'z' && p[1] == 'u') { // %zu - l = va_arg(ap, size_t); - base = 10; - p++; - } else if (*p == 'u') { // %u - l = va_arg(ap, unsigned int); - base = 10; - } else if (*p == 'd') { // %d - l = va_arg(ap, int); - base = 10; - } else if (*p == 'p') { // %p - l = va_arg(ap, intptr_t); - base = 16; - } else { - write(STDERR_FILENO, "Unimplemented TracePrintf format\n", 33); - write(STDERR_FILENO, p, 2); - write(STDERR_FILENO, "\n", 1); - abort(); - } - p++; - if (base != 0) { - bool minus = (l < 0 && base == 10); - uint64 ul = minus? -l : l; - do { - *--s = "0123456789abcdef"[ul % base]; - ul /= base; - } while (ul != 0); - if (base == 16) { - *--s = 'x'; - *--s = '0'; - } else if (minus) { - *--s = '-'; - } - } - } else { // handle normal characters - *--s = *p++; - } - while (*s != 0) { - if (i == sizeof(buf)) { - write(fd, buf, i); - i = 0; - } - buf[i++] = *s++; - } - } - if (i != 0) { - write(fd, buf, i); - } - va_end(ap); -} - -// Return the file descriptor we're writing a log to -static int TraceFd() { - static int trace_fd = -1; - if (trace_fd == -1) { // Open the trace file on the first call - const char *val = getenv("TCMALLOC_TRACE_FILE"); - bool fallback_to_stderr = false; - if (!val) { - val = "/tmp/google.alloc"; - fallback_to_stderr = true; - } - trace_fd = open(val, O_CREAT|O_TRUNC|O_WRONLY, 0666); - if (trace_fd == -1) { - if (fallback_to_stderr) { - trace_fd = 2; - TracePrintf(trace_fd, "Can't open %s. Logging to stderr.\n", val); - } else { - TracePrintf(2, "Can't open %s. Logging disabled.\n", val); - } - } - // Add a header to the log. - TracePrintf(trace_fd, "Trace started: %lu\n", - static_cast(time(NULL))); - TracePrintf(trace_fd, - "func\tsize\tptr\tthread_id\tstack pcs for tools/symbolize\n"); - } - return trace_fd; -} - -// Print the hex stack dump on a single line. PCs are separated by tabs. -static void TraceStack(void) { - void *pcs[16]; - int n = GetStackTrace(pcs, sizeof(pcs)/sizeof(pcs[0]), 0); - for (int i = 0; i != n; i++) { - TracePrintf(TraceFd(), "\t%p", pcs[i]); - } -} - -// This protects MALLOC_TRACE, to make sure its info is atomically written. -static SpinLock malloc_trace_lock(SpinLock::LINKER_INITIALIZED); - -#define MALLOC_TRACE(name, size, addr) \ - do { \ - if (FLAGS_malloctrace) { \ - SpinLockHolder l(&malloc_trace_lock); \ - TracePrintf(TraceFd(), "%s\t%" PRIuS "\t%p\t%" GPRIuPTHREAD, \ - name, size, addr, PRINTABLE_PTHREAD(pthread_self())); \ - TraceStack(); \ - TracePrintf(TraceFd(), "\n"); \ - } \ - } while (0) - -// ========================================================================= // - -// Write the characters buf[0, ..., size-1] to -// the malloc trace buffer. -// This function is intended for debugging, -// and is not declared in any header file. -// You must insert a declaration of it by hand when you need -// to use it. -void __malloctrace_write(const char *buf, size_t size) { - if (FLAGS_malloctrace) { - write(TraceFd(), buf, size); - } -} - -// ========================================================================= // - -// General debug allocation/deallocation - -static inline void* DebugAllocate(size_t size, int type) { - MallocBlock* ptr = MallocBlock::Allocate(size, type); - if (ptr == NULL) return NULL; - MALLOC_TRACE("malloc", size, ptr->data_addr()); - return ptr->data_addr(); -} - -static inline void DebugDeallocate(void* ptr, int type, size_t given_size) { - MALLOC_TRACE("free", - (ptr != 0 ? MallocBlock::FromRawPointer(ptr)->data_size() : 0), - ptr); - if (ptr) MallocBlock::FromRawPointer(ptr)->Deallocate(type, given_size); -} - -// ========================================================================= // - -// The following functions may be called via MallocExtension::instance() -// for memory verification and statistics. -class DebugMallocImplementation : public TCMallocImplementation { - public: - virtual bool GetNumericProperty(const char* name, size_t* value) { - bool result = TCMallocImplementation::GetNumericProperty(name, value); - if (result && (strcmp(name, "generic.current_allocated_bytes") == 0)) { - // Subtract bytes kept in the free queue - size_t qsize = MallocBlock::FreeQueueSize(); - if (*value >= qsize) { - *value -= qsize; - } - } - return result; - } - - virtual bool VerifyNewMemory(const void* p) { - if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kNewType); - return true; - } - - virtual bool VerifyArrayNewMemory(const void* p) { - if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kArrayNewType); - return true; - } - - virtual bool VerifyMallocMemory(const void* p) { - if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kMallocType); - return true; - } - - virtual bool VerifyAllMemory() { - return MallocBlock::CheckEverything(); - } - - virtual bool MallocMemoryStats(int* blocks, size_t* total, - int histogram[kMallocHistogramSize]) { - return MallocBlock::MemoryStats(blocks, total, histogram); - } - - virtual size_t GetEstimatedAllocatedSize(size_t size) { - return size; - } - - virtual size_t GetAllocatedSize(const void* p) { - if (p) { - RAW_CHECK(GetOwnership(p) != MallocExtension::kNotOwned, - "ptr not allocated by tcmalloc"); - return MallocBlock::FromRawPointer(p)->data_size(); - } - return 0; - } - - virtual MallocExtension::Ownership GetOwnership(const void* p) { - if (!p) { - // nobody owns NULL - return MallocExtension::kNotOwned; - } - - // FIXME: note that correct GetOwnership should not touch memory - // that is not owned by tcmalloc. Main implementation is using - // pagemap to discover if page in question is owned by us or - // not. But pagemap only has marks for first and last page of - // spans. Note that if p was returned out of our memalign with - // big alignment, then it will point outside of marked pages. Also - // note that FromRawPointer call below requires touching memory - // before pointer in order to handle memalign-ed chunks - // (offset_). This leaves us with two options: - // - // * do FromRawPointer first and have possibility of crashing if - // we're given not owned pointer - // - // * return incorrect ownership for those large memalign chunks - // - // I've decided to choose later, which appears to happen rarer and - // therefore is arguably a lesser evil - - MallocExtension::Ownership rv = TCMallocImplementation::GetOwnership(p); - if (rv != MallocExtension::kOwned) { - return rv; - } - - const MallocBlock* mb = MallocBlock::FromRawPointer(p); - return TCMallocImplementation::GetOwnership(mb); - } - - virtual void GetFreeListSizes(vector* v) { - static const char* kDebugFreeQueue = "debug.free_queue"; - - TCMallocImplementation::GetFreeListSizes(v); - - MallocExtension::FreeListInfo i; - i.type = kDebugFreeQueue; - i.min_object_size = 0; - i.max_object_size = numeric_limits::max(); - i.total_bytes_free = MallocBlock::FreeQueueSize(); - v->push_back(i); - } - - }; - -static union { - char chars[sizeof(DebugMallocImplementation)]; - void *ptr; -} debug_malloc_implementation_space; - -REGISTER_MODULE_INITIALIZER(debugallocation, { -#if (__cplusplus >= 201103L) - COMPILE_ASSERT(alignof(debug_malloc_implementation_space) >= alignof(DebugMallocImplementation), - debug_malloc_implementation_space_is_not_properly_aligned); -#endif - // Either we or valgrind will control memory management. We - // register our extension if we're the winner. Otherwise let - // Valgrind use its own malloc (so don't register our extension). - if (!RunningOnValgrind()) { - DebugMallocImplementation *impl = new (debug_malloc_implementation_space.chars) DebugMallocImplementation(); - MallocExtension::Register(impl); - } -}); - -REGISTER_MODULE_DESTRUCTOR(debugallocation, { - if (!RunningOnValgrind()) { - // When the program exits, check all blocks still in the free - // queue for corruption. - DanglingWriteChecker(); - } -}); - -// ========================================================================= // - -struct debug_alloc_retry_data { - size_t size; - int new_type; -}; - -static void *retry_debug_allocate(void *arg) { - debug_alloc_retry_data *data = static_cast(arg); - return DebugAllocate(data->size, data->new_type); -} - -// This is mostly the same a cpp_alloc in tcmalloc.cc. -// TODO(csilvers): change Allocate() above to call cpp_alloc, so we -// don't have to reproduce the logic here. To make tc_new_mode work -// properly, I think we'll need to separate out the logic of throwing -// from the logic of calling the new-handler. -inline void* debug_cpp_alloc(size_t size, int new_type, bool nothrow) { - void* p = DebugAllocate(size, new_type); - if (p != NULL) { - return p; - } - struct debug_alloc_retry_data data; - data.size = size; - data.new_type = new_type; - return handle_oom(retry_debug_allocate, &data, - true, nothrow); -} - -inline void* do_debug_malloc_or_debug_cpp_alloc(size_t size) { - void* p = DebugAllocate(size, MallocBlock::kMallocType); - if (p != NULL) { - return p; - } - struct debug_alloc_retry_data data; - data.size = size; - data.new_type = MallocBlock::kMallocType; - return handle_oom(retry_debug_allocate, &data, - false, true); -} - -// Exported routines - -extern "C" PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) PERFTOOLS_THROW { - if (ThreadCache::IsUseEmergencyMalloc()) { - return tcmalloc::EmergencyMalloc(size); - } - void* ptr = do_debug_malloc_or_debug_cpp_alloc(size); - MallocHook::InvokeNewHook(ptr, size); - return ptr; -} - -extern "C" PERFTOOLS_DLL_DECL void tc_free(void* ptr) PERFTOOLS_THROW { - if (tcmalloc::IsEmergencyPtr(ptr)) { - return tcmalloc::EmergencyFree(ptr); - } - MallocHook::InvokeDeleteHook(ptr); - DebugDeallocate(ptr, MallocBlock::kMallocType, 0); -} - -extern "C" PERFTOOLS_DLL_DECL void tc_free_sized(void *ptr, size_t size) PERFTOOLS_THROW { - MallocHook::InvokeDeleteHook(ptr); - DebugDeallocate(ptr, MallocBlock::kMallocType, size); -} - -extern "C" PERFTOOLS_DLL_DECL void* tc_calloc(size_t count, size_t size) PERFTOOLS_THROW { - if (ThreadCache::IsUseEmergencyMalloc()) { - return tcmalloc::EmergencyCalloc(count, size); - } - // Overflow check - const size_t total_size = count * size; - if (size != 0 && total_size / size != count) return NULL; - - void* block = do_debug_malloc_or_debug_cpp_alloc(total_size); - MallocHook::InvokeNewHook(block, total_size); - if (block) memset(block, 0, total_size); - return block; -} - -extern "C" PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) PERFTOOLS_THROW { - if (tcmalloc::IsEmergencyPtr(ptr)) { - return tcmalloc::EmergencyFree(ptr); - } - MallocHook::InvokeDeleteHook(ptr); - DebugDeallocate(ptr, MallocBlock::kMallocType, 0); -} - -extern "C" PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) PERFTOOLS_THROW { - if (tcmalloc::IsEmergencyPtr(ptr)) { - return tcmalloc::EmergencyRealloc(ptr, size); - } - if (ptr == NULL) { - ptr = do_debug_malloc_or_debug_cpp_alloc(size); - MallocHook::InvokeNewHook(ptr, size); - return ptr; - } - if (size == 0) { - MallocHook::InvokeDeleteHook(ptr); - DebugDeallocate(ptr, MallocBlock::kMallocType, 0); - return NULL; - } - MallocBlock* old = MallocBlock::FromRawPointer(ptr); - old->Check(MallocBlock::kMallocType); - MallocBlock* p = MallocBlock::Allocate(size, MallocBlock::kMallocType); - - // If realloc fails we are to leave the old block untouched and - // return null - if (p == NULL) return NULL; - - // if ptr was allocated via memalign, then old->data_size() is not - // start of user data. So we must be careful to copy only user-data - char *old_begin = (char *)old->data_addr(); - char *old_end = old_begin + old->data_size(); - - ssize_t old_ssize = old_end - (char *)ptr; - CHECK_CONDITION(old_ssize >= 0); - - size_t old_size = (size_t)old_ssize; - CHECK_CONDITION(old_size <= old->data_size()); - - memcpy(p->data_addr(), ptr, (old_size < size) ? old_size : size); - MallocHook::InvokeDeleteHook(ptr); - MallocHook::InvokeNewHook(p->data_addr(), size); - DebugDeallocate(ptr, MallocBlock::kMallocType, 0); - MALLOC_TRACE("realloc", p->data_size(), p->data_addr()); - return p->data_addr(); -} - -extern "C" PERFTOOLS_DLL_DECL void* tc_new(size_t size) { - void* ptr = debug_cpp_alloc(size, MallocBlock::kNewType, false); - MallocHook::InvokeNewHook(ptr, size); - if (ptr == NULL) { - RAW_LOG(FATAL, "Unable to allocate %" PRIuS " bytes: new failed.", size); - } - return ptr; -} - -extern "C" PERFTOOLS_DLL_DECL void* tc_new_nothrow(size_t size, const std::nothrow_t&) PERFTOOLS_THROW { - void* ptr = debug_cpp_alloc(size, MallocBlock::kNewType, true); - MallocHook::InvokeNewHook(ptr, size); - return ptr; -} - -extern "C" PERFTOOLS_DLL_DECL void tc_delete(void* p) PERFTOOLS_THROW { - MallocHook::InvokeDeleteHook(p); - DebugDeallocate(p, MallocBlock::kNewType, 0); -} - -extern "C" PERFTOOLS_DLL_DECL void tc_delete_sized(void* p, size_t size) throw() { - MallocHook::InvokeDeleteHook(p); - DebugDeallocate(p, MallocBlock::kNewType, size); -} - -// Some STL implementations explicitly invoke this. -// It is completely equivalent to a normal delete (delete never throws). -extern "C" PERFTOOLS_DLL_DECL void tc_delete_nothrow(void* p, const std::nothrow_t&) PERFTOOLS_THROW { - MallocHook::InvokeDeleteHook(p); - DebugDeallocate(p, MallocBlock::kNewType, 0); -} - -extern "C" PERFTOOLS_DLL_DECL void* tc_newarray(size_t size) { - void* ptr = debug_cpp_alloc(size, MallocBlock::kArrayNewType, false); - MallocHook::InvokeNewHook(ptr, size); - if (ptr == NULL) { - RAW_LOG(FATAL, "Unable to allocate %" PRIuS " bytes: new[] failed.", size); - } - return ptr; -} - -extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_nothrow(size_t size, const std::nothrow_t&) - PERFTOOLS_THROW { - void* ptr = debug_cpp_alloc(size, MallocBlock::kArrayNewType, true); - MallocHook::InvokeNewHook(ptr, size); - return ptr; -} - -extern "C" PERFTOOLS_DLL_DECL void tc_deletearray(void* p) PERFTOOLS_THROW { - MallocHook::InvokeDeleteHook(p); - DebugDeallocate(p, MallocBlock::kArrayNewType, 0); -} - -extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_sized(void* p, size_t size) throw() { - MallocHook::InvokeDeleteHook(p); - DebugDeallocate(p, MallocBlock::kArrayNewType, size); -} - -// Some STL implementations explicitly invoke this. -// It is completely equivalent to a normal delete (delete never throws). -extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_nothrow(void* p, const std::nothrow_t&) PERFTOOLS_THROW { - MallocHook::InvokeDeleteHook(p); - DebugDeallocate(p, MallocBlock::kArrayNewType, 0); -} - -// This is mostly the same as do_memalign in tcmalloc.cc. -static void *do_debug_memalign(size_t alignment, size_t size) { - // Allocate >= size bytes aligned on "alignment" boundary - // "alignment" is a power of two. - void *p = 0; - RAW_CHECK((alignment & (alignment-1)) == 0, "must be power of two"); - const size_t data_offset = MallocBlock::data_offset(); - // Allocate "alignment-1" extra bytes to ensure alignment is possible, and - // a further data_offset bytes for an additional fake header. - size_t extra_bytes = data_offset + alignment - 1; - if (size + extra_bytes < size) return NULL; // Overflow - p = DebugAllocate(size + extra_bytes, MallocBlock::kMallocType); - if (p != 0) { - intptr_t orig_p = reinterpret_cast(p); - // Leave data_offset bytes for fake header, and round up to meet - // alignment. - p = reinterpret_cast(RoundUp(orig_p + data_offset, alignment)); - // Create a fake header block with an offset_ that points back to the - // real header. FromRawPointer uses this value. - MallocBlock *fake_hdr = reinterpret_cast( - reinterpret_cast(p) - data_offset); - // offset_ is distance between real and fake headers. - // p is now end of fake header (beginning of client area), - // and orig_p is the end of the real header, so offset_ - // is their difference. - // - // Note that other fields of fake_hdr are initialized with - // kMagicUninitializedByte - fake_hdr->set_offset(reinterpret_cast(p) - orig_p); - } - return p; -} - -struct memalign_retry_data { - size_t align; - size_t size; -}; - -static void *retry_debug_memalign(void *arg) { - memalign_retry_data *data = static_cast(arg); - return do_debug_memalign(data->align, data->size); -} - -inline void* do_debug_memalign_or_debug_cpp_memalign(size_t align, - size_t size) { - void* p = do_debug_memalign(align, size); - if (p != NULL) { - return p; - } - - struct memalign_retry_data data; - data.align = align; - data.size = size; - return handle_oom(retry_debug_memalign, &data, - false, true); -} - -extern "C" PERFTOOLS_DLL_DECL void* tc_memalign(size_t align, size_t size) PERFTOOLS_THROW { - void *p = do_debug_memalign_or_debug_cpp_memalign(align, size); - MallocHook::InvokeNewHook(p, size); - return p; -} - -// Implementation taken from tcmalloc/tcmalloc.cc -extern "C" PERFTOOLS_DLL_DECL int tc_posix_memalign(void** result_ptr, size_t align, size_t size) - PERFTOOLS_THROW { - if (((align % sizeof(void*)) != 0) || - ((align & (align - 1)) != 0) || - (align == 0)) { - return EINVAL; - } - - void* result = do_debug_memalign_or_debug_cpp_memalign(align, size); - MallocHook::InvokeNewHook(result, size); - if (result == NULL) { - return ENOMEM; - } else { - *result_ptr = result; - return 0; - } -} - -extern "C" PERFTOOLS_DLL_DECL void* tc_valloc(size_t size) PERFTOOLS_THROW { - // Allocate >= size bytes starting on a page boundary - void *p = do_debug_memalign_or_debug_cpp_memalign(getpagesize(), size); - MallocHook::InvokeNewHook(p, size); - return p; -} - -extern "C" PERFTOOLS_DLL_DECL void* tc_pvalloc(size_t size) PERFTOOLS_THROW { - // Round size up to a multiple of pages - // then allocate memory on a page boundary - int pagesize = getpagesize(); - size = RoundUp(size, pagesize); - if (size == 0) { // pvalloc(0) should allocate one page, according to - size = pagesize; // http://man.free4web.biz/man3/libmpatrol.3.html - } - void *p = do_debug_memalign_or_debug_cpp_memalign(pagesize, size); - MallocHook::InvokeNewHook(p, size); - return p; -} - -// malloc_stats just falls through to the base implementation. -extern "C" PERFTOOLS_DLL_DECL void tc_malloc_stats(void) PERFTOOLS_THROW { - do_malloc_stats(); -} - -extern "C" PERFTOOLS_DLL_DECL int tc_mallopt(int cmd, int value) PERFTOOLS_THROW { - return do_mallopt(cmd, value); -} - -#ifdef HAVE_STRUCT_MALLINFO -extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) PERFTOOLS_THROW { - return do_mallinfo(); -} -#endif - -extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) PERFTOOLS_THROW { - return MallocExtension::instance()->GetAllocatedSize(ptr); -} - -extern "C" PERFTOOLS_DLL_DECL void* tc_malloc_skip_new_handler(size_t size) PERFTOOLS_THROW { - void* result = DebugAllocate(size, MallocBlock::kMallocType); - MallocHook::InvokeNewHook(result, size); - return result; -} - -#pragma GCC diagnostic pop diff --git a/contrib/libtcmalloc/src/getenv_safe.h b/contrib/libtcmalloc/src/getenv_safe.h deleted file mode 100644 index 3b9f4dbbcb2..00000000000 --- a/contrib/libtcmalloc/src/getenv_safe.h +++ /dev/null @@ -1,63 +0,0 @@ -/* -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- - * Copyright (c) 2014, gperftools Contributors - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef GETENV_SAFE_H -#define GETENV_SAFE_H - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * This getenv function is safe to call before the C runtime is initialized. - * On Windows, it utilizes GetEnvironmentVariable() and on unix it uses - * /proc/self/environ instead calling getenv(). It's intended to be used in - * routines that run before main(), when the state required for getenv() may - * not be set up yet. In particular, errno isn't set up until relatively late - * (after the pthreads library has a chance to make it threadsafe), and - * getenv() doesn't work until then. - * On some platforms, this call will utilize the same, static buffer for - * repeated GetenvBeforeMain() calls. Callers should not expect pointers from - * this routine to be long lived. - * Note that on unix, /proc only has the environment at the time the - * application was started, so this routine ignores setenv() calls/etc. Also - * note it only reads the first 16K of the environment. - * - * NOTE: this is version of GetenvBeforeMain that's usable from - * C. Implementation is in sysinfo.cc - */ -const char* TCMallocGetenvSafe(const char* name); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/contrib/libtcmalloc/src/getpc.h b/contrib/libtcmalloc/src/getpc.h deleted file mode 100644 index 163873eabc6..00000000000 --- a/contrib/libtcmalloc/src/getpc.h +++ /dev/null @@ -1,192 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Craig Silverstein -// -// This is an internal header file used by profiler.cc. It defines -// the single (inline) function GetPC. GetPC is used in a signal -// handler to figure out the instruction that was being executed when -// the signal-handler was triggered. -// -// To get this, we use the ucontext_t argument to the signal-handler -// callback, which holds the full context of what was going on when -// the signal triggered. How to get from a ucontext_t to a Program -// Counter is OS-dependent. - -#ifndef BASE_GETPC_H_ -#define BASE_GETPC_H_ - -#include "config.h" - -// On many linux systems, we may need _GNU_SOURCE to get access to -// the defined constants that define the register we want to see (eg -// REG_EIP). Note this #define must come first! -#define _GNU_SOURCE 1 -// If #define _GNU_SOURCE causes problems, this might work instead. -// It will cause problems for FreeBSD though!, because it turns off -// the needed __BSD_VISIBLE. -//#define _XOPEN_SOURCE 500 - -#include // for memcmp -#if defined(HAVE_SYS_UCONTEXT_H) -#include -#elif defined(HAVE_UCONTEXT_H) -#include // for ucontext_t (and also mcontext_t) -#elif defined(HAVE_CYGWIN_SIGNAL_H) -#include -typedef ucontext ucontext_t; -#endif - - -// Take the example where function Foo() calls function Bar(). For -// many architectures, Bar() is responsible for setting up and tearing -// down its own stack frame. In that case, it's possible for the -// interrupt to happen when execution is in Bar(), but the stack frame -// is not properly set up (either before it's done being set up, or -// after it's been torn down but before Bar() returns). In those -// cases, the stack trace cannot see the caller function anymore. -// -// GetPC can try to identify this situation, on architectures where it -// might occur, and unwind the current function call in that case to -// avoid false edges in the profile graph (that is, edges that appear -// to show a call skipping over a function). To do this, we hard-code -// in the asm instructions we might see when setting up or tearing -// down a stack frame. -// -// This is difficult to get right: the instructions depend on the -// processor, the compiler ABI, and even the optimization level. This -// is a best effort patch -- if we fail to detect such a situation, or -// mess up the PC, nothing happens; the returned PC is not used for -// any further processing. -struct CallUnrollInfo { - // Offset from (e)ip register where this instruction sequence - // should be matched. Interpreted as bytes. Offset 0 is the next - // instruction to execute. Be extra careful with negative offsets in - // architectures of variable instruction length (like x86) - it is - // not that easy as taking an offset to step one instruction back! - int pc_offset; - // The actual instruction bytes. Feel free to make it larger if you - // need a longer sequence. - unsigned char ins[16]; - // How many bytes to match from ins array? - int ins_size; - // The offset from the stack pointer (e)sp where to look for the - // call return address. Interpreted as bytes. - int return_sp_offset; -}; - - -// The dereferences needed to get the PC from a struct ucontext were -// determined at configure time, and stored in the macro -// PC_FROM_UCONTEXT in config.h. The only thing we need to do here, -// then, is to do the magic call-unrolling for systems that support it. - -// -- Special case 1: linux x86, for which we have CallUnrollInfo -#if defined(__linux) && defined(__i386) && defined(__GNUC__) -static const CallUnrollInfo callunrollinfo[] = { - // Entry to a function: push %ebp; mov %esp,%ebp - // Top-of-stack contains the caller IP. - { 0, - {0x55, 0x89, 0xe5}, 3, - 0 - }, - // Entry to a function, second instruction: push %ebp; mov %esp,%ebp - // Top-of-stack contains the old frame, caller IP is +4. - { -1, - {0x55, 0x89, 0xe5}, 3, - 4 - }, - // Return from a function: RET. - // Top-of-stack contains the caller IP. - { 0, - {0xc3}, 1, - 0 - } -}; - -inline void* GetPC(const ucontext_t& signal_ucontext) { - // See comment above struct CallUnrollInfo. Only try instruction - // flow matching if both eip and esp looks reasonable. - const int eip = signal_ucontext.uc_mcontext.gregs[REG_EIP]; - const int esp = signal_ucontext.uc_mcontext.gregs[REG_ESP]; - if ((eip & 0xffff0000) != 0 && (~eip & 0xffff0000) != 0 && - (esp & 0xffff0000) != 0) { - char* eip_char = reinterpret_cast(eip); - for (int i = 0; i < sizeof(callunrollinfo)/sizeof(*callunrollinfo); ++i) { - if (!memcmp(eip_char + callunrollinfo[i].pc_offset, - callunrollinfo[i].ins, callunrollinfo[i].ins_size)) { - // We have a match. - void **retaddr = (void**)(esp + callunrollinfo[i].return_sp_offset); - return *retaddr; - } - } - } - return (void*)eip; -} - -// Special case #2: Windows, which has to do something totally different. -#elif defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__MINGW32__) -// If this is ever implemented, probably the way to do it is to have -// profiler.cc use a high-precision timer via timeSetEvent: -// http://msdn2.microsoft.com/en-us/library/ms712713.aspx -// We'd use it in mode TIME_CALLBACK_FUNCTION/TIME_PERIODIC. -// The callback function would be something like prof_handler, but -// alas the arguments are different: no ucontext_t! I don't know -// how we'd get the PC (using StackWalk64?) -// http://msdn2.microsoft.com/en-us/library/ms680650.aspx - -#include "base/logging.h" // for RAW_LOG -#ifndef HAVE_CYGWIN_SIGNAL_H -typedef int ucontext_t; -#endif - -inline void* GetPC(const struct ucontext_t& signal_ucontext) { - RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n"); - return NULL; -} - -// Normal cases. If this doesn't compile, it's probably because -// PC_FROM_UCONTEXT is the empty string. You need to figure out -// the right value for your system, and add it to the list in -// configure.ac (or set it manually in your config.h). -#else -inline void* GetPC(const ucontext_t& signal_ucontext) { -#if defined(__s390__) && !defined(__s390x__) - // Mask out the AMODE31 bit from the PC recorded in the context. - return (void*)((unsigned long)signal_ucontext.PC_FROM_UCONTEXT & 0x7fffffffUL); -#else - return (void*)signal_ucontext.PC_FROM_UCONTEXT; // defined in config.h -#endif -} - -#endif - -#endif // BASE_GETPC_H_ diff --git a/contrib/libtcmalloc/src/heap-checker-bcad.cc b/contrib/libtcmalloc/src/heap-checker-bcad.cc deleted file mode 100644 index 00efdb7cfd4..00000000000 --- a/contrib/libtcmalloc/src/heap-checker-bcad.cc +++ /dev/null @@ -1,93 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// All Rights Reserved. -// -// Author: Maxim Lifantsev -// -// A file to ensure that components of heap leak checker run before -// all global object constructors and after all global object -// destructors. -// -// This file must be the last library any binary links against. -// Otherwise, the heap checker may not be able to run early enough to -// catalog all the global objects in your program. If this happens, -// and later in the program you allocate memory and have one of these -// "uncataloged" global objects point to it, the heap checker will -// consider that allocation to be a leak, even though it's not (since -// the allocated object is reachable from global data and hence "live"). - -#include // for abort() -#include - -// A dummy variable to refer from heap-checker.cc. This is to make -// sure this file is not optimized out by the linker. -bool heap_leak_checker_bcad_variable; - -extern void HeapLeakChecker_AfterDestructors(); // in heap-checker.cc - -// A helper class to ensure that some components of heap leak checking -// can happen before construction and after destruction -// of all global/static objects. -class HeapLeakCheckerGlobalPrePost { - public: - HeapLeakCheckerGlobalPrePost() { - if (count_ == 0) { - // The 'new int' will ensure that we have run an initial malloc - // hook, which will set up the heap checker via - // MallocHook_InitAtFirstAllocation_HeapLeakChecker. See malloc_hook.cc. - // This is done in this roundabout fashion in order to avoid self-deadlock - // if we directly called HeapLeakChecker_BeforeConstructors here. - delete new int; - // This needs to be called before the first allocation of an STL - // object, but after libc is done setting up threads (because it - // calls setenv, which requires a thread-aware errno). By - // putting it here, we hope it's the first bit of code executed - // after the libc global-constructor code. - MallocExtension::Initialize(); - } - ++count_; - } - ~HeapLeakCheckerGlobalPrePost() { - if (count_ <= 0) abort(); - --count_; - if (count_ == 0) HeapLeakChecker_AfterDestructors(); - } - private: - // Counter of constructions/destructions of objects of this class - // (just in case there are more than one of them). - static int count_; -}; - -int HeapLeakCheckerGlobalPrePost::count_ = 0; - -// The early-construction/late-destruction global object. -static const HeapLeakCheckerGlobalPrePost heap_leak_checker_global_pre_post; diff --git a/contrib/libtcmalloc/src/heap-checker.cc b/contrib/libtcmalloc/src/heap-checker.cc deleted file mode 100644 index 9c82dea08e4..00000000000 --- a/contrib/libtcmalloc/src/heap-checker.cc +++ /dev/null @@ -1,2388 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// All Rights Reserved. -// -// Author: Maxim Lifantsev -// - -#include "config.h" - -#include // for O_RDONLY (we use syscall to do actual reads) -#include -#include -#ifdef HAVE_UNISTD_H -#include -#endif -#ifdef HAVE_MMAP -#include -#endif -#ifdef HAVE_PTHREAD -#include -#endif -#include -#include -#include -#include - -#if defined(HAVE_LINUX_PTRACE_H) -#include -#endif -#ifdef HAVE_SYS_SYSCALL_H -#include -#endif -#if defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__MINGW32__) -#include -#include -#undef ERROR // windows defines these as macros, which can cause trouble -#undef max -#undef min -#endif - -#include -#include -#include -#include -#include -#include - -#include - -#include "base/basictypes.h" -#include "base/googleinit.h" -#include "base/logging.h" -#include -#include "base/commandlineflags.h" -#include "base/elfcore.h" // for i386_regs -#include "base/thread_lister.h" -#include "heap-profile-table.h" -#include "base/low_level_alloc.h" -#include "malloc_hook-inl.h" -#include -#include -#include "maybe_threads.h" -#include "memory_region_map.h" -#include "base/spinlock.h" -#include "base/sysinfo.h" -#include "base/stl_allocator.h" - -using std::string; -using std::basic_string; -using std::pair; -using std::map; -using std::set; -using std::vector; -using std::swap; -using std::make_pair; -using std::min; -using std::max; -using std::less; -using std::char_traits; - -// If current process is being ptrace()d, 'TracerPid' in /proc/self/status -// will be non-zero. -static bool IsDebuggerAttached(void) { // only works under linux, probably - char buf[256]; // TracerPid comes relatively earlier in status output - int fd = open("/proc/self/status", O_RDONLY); - if (fd == -1) { - return false; // Can't tell for sure. - } - const int len = read(fd, buf, sizeof(buf)); - bool rc = false; - if (len > 0) { - const char *const kTracerPid = "TracerPid:\t"; - buf[len - 1] = '\0'; - const char *p = strstr(buf, kTracerPid); - if (p != NULL) { - rc = (strncmp(p + strlen(kTracerPid), "0\n", 2) != 0); - } - } - close(fd); - return rc; -} - -// This is the default if you don't link in -lprofiler -extern "C" { -ATTRIBUTE_WEAK PERFTOOLS_DLL_DECL bool ProfilingIsEnabledForAllThreads(); -bool ProfilingIsEnabledForAllThreads() { return false; } -} - -//---------------------------------------------------------------------- -// Flags that control heap-checking -//---------------------------------------------------------------------- - -DEFINE_string(heap_check, - EnvToString("HEAPCHECK", ""), - "The heap leak checking to be done over the whole executable: " - "\"minimal\", \"normal\", \"strict\", " - "\"draconian\", \"as-is\", and \"local\" " - " or the empty string are the supported choices. " - "(See HeapLeakChecker_InternalInitStart for details.)"); - -DEFINE_bool(heap_check_report, true, "Obsolete"); - -DEFINE_bool(heap_check_before_constructors, - true, - "deprecated; pretty much always true now"); - -DEFINE_bool(heap_check_after_destructors, - EnvToBool("HEAP_CHECK_AFTER_DESTRUCTORS", false), - "If overall heap check is to end after global destructors " - "or right after all REGISTER_HEAPCHECK_CLEANUP's"); - -DEFINE_bool(heap_check_strict_check, true, "Obsolete"); - -DEFINE_bool(heap_check_ignore_global_live, - EnvToBool("HEAP_CHECK_IGNORE_GLOBAL_LIVE", true), - "If overall heap check is to ignore heap objects reachable " - "from the global data"); - -DEFINE_bool(heap_check_identify_leaks, - EnvToBool("HEAP_CHECK_IDENTIFY_LEAKS", false), - "If heap check should generate the addresses of the leaked " - "objects in the memory leak profiles. This may be useful " - "in tracking down leaks where only a small fraction of " - "objects allocated at the same stack trace are leaked."); - -DEFINE_bool(heap_check_ignore_thread_live, - EnvToBool("HEAP_CHECK_IGNORE_THREAD_LIVE", true), - "If set to true, objects reachable from thread stacks " - "and registers are not reported as leaks"); - -DEFINE_bool(heap_check_test_pointer_alignment, - EnvToBool("HEAP_CHECK_TEST_POINTER_ALIGNMENT", false), - "Set to true to check if the found leak can be due to " - "use of unaligned pointers"); - -// Alignment at which all pointers in memory are supposed to be located; -// use 1 if any alignment is ok. -// heap_check_test_pointer_alignment flag guides if we try the value of 1. -// The larger it can be, the lesser is the chance of missing real leaks. -static const size_t kPointerSourceAlignment = sizeof(void*); -DEFINE_int32(heap_check_pointer_source_alignment, - EnvToInt("HEAP_CHECK_POINTER_SOURCE_ALIGNMENT", - kPointerSourceAlignment), - "Alignment at which all pointers in memory are supposed to be " - "located. Use 1 if any alignment is ok."); - -// A reasonable default to handle pointers inside of typical class objects: -// Too low and we won't be able to traverse pointers to normally-used -// nested objects and base parts of multiple-inherited objects. -// Too high and it will both slow down leak checking (FindInsideAlloc -// in HaveOnHeapLocked will get slower when there are large on-heap objects) -// and make it probabilistically more likely to miss leaks -// of large-sized objects. -static const int64 kHeapCheckMaxPointerOffset = 1024; -DEFINE_int64(heap_check_max_pointer_offset, - EnvToInt("HEAP_CHECK_MAX_POINTER_OFFSET", - kHeapCheckMaxPointerOffset), - "Largest pointer offset for which we traverse " - "pointers going inside of heap allocated objects. " - "Set to -1 to use the actual largest heap object size."); - -DEFINE_bool(heap_check_run_under_gdb, - EnvToBool("HEAP_CHECK_RUN_UNDER_GDB", false), - "If false, turns off heap-checking library when running under gdb " - "(normally, set to 'true' only when debugging the heap-checker)"); - -DEFINE_int32(heap_check_delay_seconds, 0, - "Number of seconds to delay on-exit heap checking." - " If you set this flag," - " you may also want to set exit_timeout_seconds in order to" - " avoid exit timeouts.\n" - "NOTE: This flag is to be used only to help diagnose issues" - " where it is suspected that the heap checker is reporting" - " false leaks that will disappear if the heap checker delays" - " its checks. Report any such issues to the heap-checker" - " maintainer(s)."); - -//---------------------------------------------------------------------- - -DEFINE_string(heap_profile_pprof, - EnvToString("PPROF_PATH", "pprof"), - "OBSOLETE; not used"); - -DEFINE_string(heap_check_dump_directory, - EnvToString("HEAP_CHECK_DUMP_DIRECTORY", "/tmp"), - "Directory to put heap-checker leak dump information"); - - -//---------------------------------------------------------------------- -// HeapLeakChecker global data -//---------------------------------------------------------------------- - -// Global lock for all the global data of this module. -static SpinLock heap_checker_lock(SpinLock::LINKER_INITIALIZED); - -//---------------------------------------------------------------------- - -// Heap profile prefix for leak checking profiles. -// Gets assigned once when leak checking is turned on, then never modified. -static const string* profile_name_prefix = NULL; - -// Whole-program heap leak checker. -// Gets assigned once when leak checking is turned on, -// then main_heap_checker is never deleted. -static HeapLeakChecker* main_heap_checker = NULL; - -// Whether we will use main_heap_checker to do a check at program exit -// automatically. In any case user can ask for more checks on main_heap_checker -// via GlobalChecker(). -static bool do_main_heap_check = false; - -// The heap profile we use to collect info about the heap. -// This is created in HeapLeakChecker::BeforeConstructorsLocked -// together with setting heap_checker_on (below) to true -// and registering our new/delete malloc hooks; -// similarly all are unset in HeapLeakChecker::TurnItselfOffLocked. -static HeapProfileTable* heap_profile = NULL; - -// If we are doing (or going to do) any kind of heap-checking. -static bool heap_checker_on = false; - -// pid of the process that does whole-program heap leak checking -static pid_t heap_checker_pid = 0; - -// If we did heap profiling during global constructors execution -static bool constructor_heap_profiling = false; - -// RAW_VLOG level we dump key INFO messages at. If you want to turn -// off these messages, set the environment variable PERFTOOLS_VERBOSE=-1. -static const int heap_checker_info_level = 0; - -//---------------------------------------------------------------------- -// HeapLeakChecker's own memory allocator that is -// independent of the normal program allocator. -//---------------------------------------------------------------------- - -// Wrapper of LowLevelAlloc for STL_Allocator and direct use. -// We always access this class under held heap_checker_lock, -// this allows us to in particular protect the period when threads are stopped -// at random spots with TCMalloc_ListAllProcessThreads by heap_checker_lock, -// w/o worrying about the lock in LowLevelAlloc::Arena. -// We rely on the fact that we use an own arena with an own lock here. -class HeapLeakChecker::Allocator { - public: - static void Init() { - RAW_DCHECK(heap_checker_lock.IsHeld(), ""); - RAW_DCHECK(arena_ == NULL, ""); - arena_ = LowLevelAlloc::NewArena(0, LowLevelAlloc::DefaultArena()); - } - static void Shutdown() { - RAW_DCHECK(heap_checker_lock.IsHeld(), ""); - if (!LowLevelAlloc::DeleteArena(arena_) || alloc_count_ != 0) { - RAW_LOG(FATAL, "Internal heap checker leak of %d objects", alloc_count_); - } - } - static int alloc_count() { - RAW_DCHECK(heap_checker_lock.IsHeld(), ""); - return alloc_count_; - } - static void* Allocate(size_t n) { - RAW_DCHECK(arena_ && heap_checker_lock.IsHeld(), ""); - void* p = LowLevelAlloc::AllocWithArena(n, arena_); - if (p) alloc_count_ += 1; - return p; - } - static void Free(void* p) { - RAW_DCHECK(heap_checker_lock.IsHeld(), ""); - if (p) alloc_count_ -= 1; - LowLevelAlloc::Free(p); - } - static void Free(void* p, size_t /* n */) { - Free(p); - } - // destruct, free, and make *p to be NULL - template static void DeleteAndNull(T** p) { - (*p)->~T(); - Free(*p); - *p = NULL; - } - template static void DeleteAndNullIfNot(T** p) { - if (*p != NULL) DeleteAndNull(p); - } - private: - static LowLevelAlloc::Arena* arena_; - static int alloc_count_; -}; - -LowLevelAlloc::Arena* HeapLeakChecker::Allocator::arena_ = NULL; -int HeapLeakChecker::Allocator::alloc_count_ = 0; - -//---------------------------------------------------------------------- -// HeapLeakChecker live object tracking components -//---------------------------------------------------------------------- - -// Cases of live object placement we distinguish -enum ObjectPlacement { - MUST_BE_ON_HEAP, // Must point to a live object of the matching size in the - // heap_profile map of the heap when we get to it - IGNORED_ON_HEAP, // Is a live (ignored) object on heap - MAYBE_LIVE, // Is a piece of writable memory from /proc/self/maps - IN_GLOBAL_DATA, // Is part of global data region of the executable - THREAD_DATA, // Part of a thread stack and a thread descriptor with TLS - THREAD_REGISTERS, // Values in registers of some thread -}; - -// Information about an allocated object -struct AllocObject { - const void* ptr; // the object - uintptr_t size; // its size - ObjectPlacement place; // where ptr points to - - AllocObject(const void* p, size_t s, ObjectPlacement l) - : ptr(p), size(s), place(l) { } -}; - -// All objects (memory ranges) ignored via HeapLeakChecker::IgnoreObject -// Key is the object's address; value is its size. -typedef map, - STL_Allocator, - HeapLeakChecker::Allocator> - > IgnoredObjectsMap; -static IgnoredObjectsMap* ignored_objects = NULL; - -// All objects (memory ranges) that we consider to be the sources of pointers -// to live (not leaked) objects. -// At different times this holds (what can be reached from) global data regions -// and the objects we've been told to ignore. -// For any AllocObject::ptr "live_objects" is supposed to contain at most one -// record at any time. We maintain this by checking with the heap_profile map -// of the heap and removing the live heap objects we've handled from it. -// This vector is maintained as a stack and the frontier of reachable -// live heap objects in our flood traversal of them. -typedef vector - > LiveObjectsStack; -static LiveObjectsStack* live_objects = NULL; - -// A special string type that uses my allocator -typedef basic_string, - STL_Allocator - > HCL_string; - -// A placeholder to fill-in the starting values for live_objects -// for each library so we can keep the library-name association for logging. -typedef map, - STL_Allocator, - HeapLeakChecker::Allocator> - > LibraryLiveObjectsStacks; -static LibraryLiveObjectsStacks* library_live_objects = NULL; - -// Value stored in the map of disabled address ranges; -// its key is the end of the address range. -// We'll ignore allocations with a return address in a disabled range -// if the address occurs at 'max_depth' or less in the stack trace. -struct HeapLeakChecker::RangeValue { - uintptr_t start_address; // the start of the range - int max_depth; // the maximal stack depth to disable at -}; -typedef map, - STL_Allocator, - HeapLeakChecker::Allocator> - > DisabledRangeMap; -// The disabled program counter address ranges for profile dumping -// that are registered with HeapLeakChecker::DisableChecksFromToLocked. -static DisabledRangeMap* disabled_ranges = NULL; - -// Set of stack tops. -// These are used to consider live only appropriate chunks of the memory areas -// that are used for stacks (and maybe thread-specific data as well) -// so that we do not treat pointers from outdated stack frames as live. -typedef set, - STL_Allocator - > StackTopSet; -static StackTopSet* stack_tops = NULL; - -// A map of ranges of code addresses for the system libraries -// that can mmap/mremap/sbrk-allocate memory regions for stacks -// and thread-local storage that we want to consider as live global data. -// Maps from the end address to the start address. -typedef map, - STL_Allocator, - HeapLeakChecker::Allocator> - > GlobalRegionCallerRangeMap; -static GlobalRegionCallerRangeMap* global_region_caller_ranges = NULL; - -// TODO(maxim): make our big data structs into own modules - -// Disabler is implemented by keeping track of a per-thread count -// of active Disabler objects. Any objects allocated while the -// count > 0 are not reported. - -#ifdef HAVE_TLS - -static __thread int thread_disable_counter -// The "inital exec" model is faster than the default TLS model, at -// the cost you can't dlopen this library. But dlopen on heap-checker -// doesn't work anyway -- it must run before main -- so this is a good -// trade-off. -# ifdef HAVE___ATTRIBUTE__ - __attribute__ ((tls_model ("initial-exec"))) -# endif - ; -inline int get_thread_disable_counter() { - return thread_disable_counter; -} -inline void set_thread_disable_counter(int value) { - thread_disable_counter = value; -} - -#else // #ifdef HAVE_TLS - -static pthread_key_t thread_disable_counter_key; -static int main_thread_counter; // storage for use before main() -static bool use_main_thread_counter = true; - -// TODO(csilvers): this is called from NewHook, in the middle of malloc(). -// If perftools_pthread_getspecific calls malloc, that will lead to an -// infinite loop. I don't know how to fix that, so I hope it never happens! -inline int get_thread_disable_counter() { - if (use_main_thread_counter) // means we're running really early - return main_thread_counter; - void* p = perftools_pthread_getspecific(thread_disable_counter_key); - return (intptr_t)p; // kinda evil: store the counter directly in the void* -} - -inline void set_thread_disable_counter(int value) { - if (use_main_thread_counter) { // means we're running really early - main_thread_counter = value; - return; - } - intptr_t pointer_sized_value = value; - // kinda evil: store the counter directly in the void* - void* p = (void*)pointer_sized_value; - // NOTE: this may call malloc, which will call NewHook which will call - // get_thread_disable_counter() which will call pthread_getspecific(). I - // don't know if anything bad can happen if we call getspecific() in the - // middle of a setspecific() call. It seems to work ok in practice... - perftools_pthread_setspecific(thread_disable_counter_key, p); -} - -// The idea here is that this initializer will run pretty late: after -// pthreads have been totally set up. At this point we can call -// pthreads routines, so we set those up. -class InitThreadDisableCounter { - public: - InitThreadDisableCounter() { - perftools_pthread_key_create(&thread_disable_counter_key, NULL); - // Set up the main thread's value, which we have a special variable for. - void* p = (void*)main_thread_counter; // store the counter directly - perftools_pthread_setspecific(thread_disable_counter_key, p); - use_main_thread_counter = false; - } -}; -InitThreadDisableCounter init_thread_disable_counter; - -#endif // #ifdef HAVE_TLS - -HeapLeakChecker::Disabler::Disabler() { - // It is faster to unconditionally increment the thread-local - // counter than to check whether or not heap-checking is on - // in a thread-safe manner. - int counter = get_thread_disable_counter(); - set_thread_disable_counter(counter + 1); - RAW_VLOG(10, "Increasing thread disable counter to %d", counter + 1); -} - -HeapLeakChecker::Disabler::~Disabler() { - int counter = get_thread_disable_counter(); - RAW_DCHECK(counter > 0, ""); - if (counter > 0) { - set_thread_disable_counter(counter - 1); - RAW_VLOG(10, "Decreasing thread disable counter to %d", counter); - } else { - RAW_VLOG(0, "Thread disable counter underflow : %d", counter); - } -} - -//---------------------------------------------------------------------- - -// The size of the largest heap object allocated so far. -static size_t max_heap_object_size = 0; -// The possible range of addresses that can point -// into one of the elements of heap_objects. -static uintptr_t min_heap_address = uintptr_t(-1LL); -static uintptr_t max_heap_address = 0; - -//---------------------------------------------------------------------- - -// Simple casting helpers for uintptr_t and void*: -template -inline static const void* AsPtr(T addr) { - return reinterpret_cast(addr); -} -inline static uintptr_t AsInt(const void* ptr) { - return reinterpret_cast(ptr); -} - -//---------------------------------------------------------------------- - -// We've seen reports that strstr causes heap-checker crashes in some -// libc's (?): -// http://code.google.com/p/gperftools/issues/detail?id=263 -// It's simple enough to use our own. This is not in time-critical code. -static const char* hc_strstr(const char* s1, const char* s2) { - const size_t len = strlen(s2); - RAW_CHECK(len > 0, "Unexpected empty string passed to strstr()"); - for (const char* p = strchr(s1, *s2); p != NULL; p = strchr(p+1, *s2)) { - if (strncmp(p, s2, len) == 0) { - return p; - } - } - return NULL; -} - -//---------------------------------------------------------------------- - -// Our hooks for MallocHook -static void NewHook(const void* ptr, size_t size) { - if (ptr != NULL) { - const int counter = get_thread_disable_counter(); - const bool ignore = (counter > 0); - RAW_VLOG(16, "Recording Alloc: %p of %" PRIuS "; %d", ptr, size, - int(counter)); - - // Fetch the caller's stack trace before acquiring heap_checker_lock. - void* stack[HeapProfileTable::kMaxStackDepth]; - int depth = HeapProfileTable::GetCallerStackTrace(0, stack); - - { SpinLockHolder l(&heap_checker_lock); - if (size > max_heap_object_size) max_heap_object_size = size; - uintptr_t addr = AsInt(ptr); - if (addr < min_heap_address) min_heap_address = addr; - addr += size; - if (addr > max_heap_address) max_heap_address = addr; - if (heap_checker_on) { - heap_profile->RecordAlloc(ptr, size, depth, stack); - if (ignore) { - heap_profile->MarkAsIgnored(ptr); - } - } - } - RAW_VLOG(17, "Alloc Recorded: %p of %" PRIuS "", ptr, size); - } -} - -static void DeleteHook(const void* ptr) { - if (ptr != NULL) { - RAW_VLOG(16, "Recording Free %p", ptr); - { SpinLockHolder l(&heap_checker_lock); - if (heap_checker_on) heap_profile->RecordFree(ptr); - } - RAW_VLOG(17, "Free Recorded: %p", ptr); - } -} - -//---------------------------------------------------------------------- - -enum StackDirection { - GROWS_TOWARDS_HIGH_ADDRESSES, - GROWS_TOWARDS_LOW_ADDRESSES, - UNKNOWN_DIRECTION -}; - -// Determine which way the stack grows: - -static StackDirection ATTRIBUTE_NOINLINE GetStackDirection( - const uintptr_t *const ptr) { - uintptr_t x; - if (&x < ptr) - return GROWS_TOWARDS_LOW_ADDRESSES; - if (ptr < &x) - return GROWS_TOWARDS_HIGH_ADDRESSES; - - RAW_CHECK(0, ""); // Couldn't determine the stack direction. - - return UNKNOWN_DIRECTION; -} - -// Direction of stack growth (will initialize via GetStackDirection()) -static StackDirection stack_direction = UNKNOWN_DIRECTION; - -// This routine is called for every thread stack we know about to register it. -static void RegisterStackLocked(const void* top_ptr) { - RAW_DCHECK(heap_checker_lock.IsHeld(), ""); - RAW_DCHECK(MemoryRegionMap::LockIsHeld(), ""); - RAW_VLOG(10, "Thread stack at %p", top_ptr); - uintptr_t top = AsInt(top_ptr); - stack_tops->insert(top); // add for later use - - // make sure stack_direction is initialized - if (stack_direction == UNKNOWN_DIRECTION) { - stack_direction = GetStackDirection(&top); - } - - // Find memory region with this stack - MemoryRegionMap::Region region; - if (MemoryRegionMap::FindAndMarkStackRegion(top, ®ion)) { - // Make the proper portion of the stack live: - if (stack_direction == GROWS_TOWARDS_LOW_ADDRESSES) { - RAW_VLOG(11, "Live stack at %p of %" PRIuPTR " bytes", - top_ptr, region.end_addr - top); - live_objects->push_back(AllocObject(top_ptr, region.end_addr - top, - THREAD_DATA)); - } else { // GROWS_TOWARDS_HIGH_ADDRESSES - RAW_VLOG(11, "Live stack at %p of %" PRIuPTR " bytes", - AsPtr(region.start_addr), - top - region.start_addr); - live_objects->push_back(AllocObject(AsPtr(region.start_addr), - top - region.start_addr, - THREAD_DATA)); - } - // not in MemoryRegionMap, look in library_live_objects: - } else if (FLAGS_heap_check_ignore_global_live) { - for (LibraryLiveObjectsStacks::iterator lib = library_live_objects->begin(); - lib != library_live_objects->end(); ++lib) { - for (LiveObjectsStack::iterator span = lib->second.begin(); - span != lib->second.end(); ++span) { - uintptr_t start = AsInt(span->ptr); - uintptr_t end = start + span->size; - if (start <= top && top < end) { - RAW_VLOG(11, "Stack at %p is inside /proc/self/maps chunk %p..%p", - top_ptr, AsPtr(start), AsPtr(end)); - // Shrink start..end region by chopping away the memory regions in - // MemoryRegionMap that land in it to undo merging of regions - // in /proc/self/maps, so that we correctly identify what portion - // of start..end is actually the stack region. - uintptr_t stack_start = start; - uintptr_t stack_end = end; - // can optimize-away this loop, but it does not run often - RAW_DCHECK(MemoryRegionMap::LockIsHeld(), ""); - for (MemoryRegionMap::RegionIterator r = - MemoryRegionMap::BeginRegionLocked(); - r != MemoryRegionMap::EndRegionLocked(); ++r) { - if (top < r->start_addr && r->start_addr < stack_end) { - stack_end = r->start_addr; - } - if (stack_start < r->end_addr && r->end_addr <= top) { - stack_start = r->end_addr; - } - } - if (stack_start != start || stack_end != end) { - RAW_VLOG(11, "Stack at %p is actually inside memory chunk %p..%p", - top_ptr, AsPtr(stack_start), AsPtr(stack_end)); - } - // Make the proper portion of the stack live: - if (stack_direction == GROWS_TOWARDS_LOW_ADDRESSES) { - RAW_VLOG(11, "Live stack at %p of %" PRIuPTR " bytes", - top_ptr, stack_end - top); - live_objects->push_back( - AllocObject(top_ptr, stack_end - top, THREAD_DATA)); - } else { // GROWS_TOWARDS_HIGH_ADDRESSES - RAW_VLOG(11, "Live stack at %p of %" PRIuPTR " bytes", - AsPtr(stack_start), top - stack_start); - live_objects->push_back( - AllocObject(AsPtr(stack_start), top - stack_start, THREAD_DATA)); - } - lib->second.erase(span); // kill the rest of the region - // Put the non-stack part(s) of the region back: - if (stack_start != start) { - lib->second.push_back(AllocObject(AsPtr(start), stack_start - start, - MAYBE_LIVE)); - } - if (stack_end != end) { - lib->second.push_back(AllocObject(AsPtr(stack_end), end - stack_end, - MAYBE_LIVE)); - } - return; - } - } - } - RAW_LOG(ERROR, "Memory region for stack at %p not found. " - "Will likely report false leak positives.", top_ptr); - } -} - -// Iterator for heap allocation map data to make ignored objects "live" -// (i.e., treated as roots for the mark-and-sweep phase) -static void MakeIgnoredObjectsLiveCallbackLocked( - const void* ptr, const HeapProfileTable::AllocInfo& info) { - RAW_DCHECK(heap_checker_lock.IsHeld(), ""); - if (info.ignored) { - live_objects->push_back(AllocObject(ptr, info.object_size, - MUST_BE_ON_HEAP)); - } -} - -// Iterator for heap allocation map data to make objects allocated from -// disabled regions of code to be live. -static void MakeDisabledLiveCallbackLocked( - const void* ptr, const HeapProfileTable::AllocInfo& info) { - RAW_DCHECK(heap_checker_lock.IsHeld(), ""); - bool stack_disable = false; - bool range_disable = false; - for (int depth = 0; depth < info.stack_depth; depth++) { - uintptr_t addr = AsInt(info.call_stack[depth]); - if (disabled_ranges) { - DisabledRangeMap::const_iterator iter - = disabled_ranges->upper_bound(addr); - if (iter != disabled_ranges->end()) { - RAW_DCHECK(iter->first > addr, ""); - if (iter->second.start_address < addr && - iter->second.max_depth > depth) { - range_disable = true; // in range; dropping - break; - } - } - } - } - if (stack_disable || range_disable) { - uintptr_t start_address = AsInt(ptr); - uintptr_t end_address = start_address + info.object_size; - StackTopSet::const_iterator iter - = stack_tops->lower_bound(start_address); - if (iter != stack_tops->end()) { - RAW_DCHECK(*iter >= start_address, ""); - if (*iter < end_address) { - // We do not disable (treat as live) whole allocated regions - // if they are used to hold thread call stacks - // (i.e. when we find a stack inside). - // The reason is that we'll treat as live the currently used - // stack portions anyway (see RegisterStackLocked), - // and the rest of the region where the stack lives can well - // contain outdated stack variables which are not live anymore, - // hence should not be treated as such. - RAW_VLOG(11, "Not %s-disabling %" PRIuS " bytes at %p" - ": have stack inside: %p", - (stack_disable ? "stack" : "range"), - info.object_size, ptr, AsPtr(*iter)); - return; - } - } - RAW_VLOG(11, "%s-disabling %" PRIuS " bytes at %p", - (stack_disable ? "Stack" : "Range"), info.object_size, ptr); - live_objects->push_back(AllocObject(ptr, info.object_size, - MUST_BE_ON_HEAP)); - } -} - -static const char kUnnamedProcSelfMapEntry[] = "UNNAMED"; - -// This function takes some fields from a /proc/self/maps line: -// -// start_address start address of a memory region. -// end_address end address of a memory region -// permissions rwx + private/shared bit -// filename filename of the mapped file -// -// If the region is not writeable, then it cannot have any heap -// pointers in it, otherwise we record it as a candidate live region -// to get filtered later. -static void RecordGlobalDataLocked(uintptr_t start_address, - uintptr_t end_address, - const char* permissions, - const char* filename) { - RAW_DCHECK(heap_checker_lock.IsHeld(), ""); - // Ignore non-writeable regions. - if (strchr(permissions, 'w') == NULL) return; - if (filename == NULL || *filename == '\0') { - filename = kUnnamedProcSelfMapEntry; - } - RAW_VLOG(11, "Looking into %s: 0x%" PRIxPTR "..0x%" PRIxPTR, - filename, start_address, end_address); - (*library_live_objects)[filename]. - push_back(AllocObject(AsPtr(start_address), - end_address - start_address, - MAYBE_LIVE)); -} - -// See if 'library' from /proc/self/maps has base name 'library_base' -// i.e. contains it and has '.' or '-' after it. -static bool IsLibraryNamed(const char* library, const char* library_base) { - const char* p = hc_strstr(library, library_base); - size_t sz = strlen(library_base); - return p != NULL && (p[sz] == '.' || p[sz] == '-'); -} - -// static -void HeapLeakChecker::DisableLibraryAllocsLocked(const char* library, - uintptr_t start_address, - uintptr_t end_address) { - RAW_DCHECK(heap_checker_lock.IsHeld(), ""); - int depth = 0; - // TODO(maxim): maybe this should be extended to also use objdump - // and pick the text portion of the library more precisely. - if (IsLibraryNamed(library, "/libpthread") || - // libpthread has a lot of small "system" leaks we don't care about. - // In particular it allocates memory to store data supplied via - // pthread_setspecific (which can be the only pointer to a heap object). - IsLibraryNamed(library, "/libdl") || - // library loaders leak some "system" heap that we don't care about - IsLibraryNamed(library, "/libcrypto") || - // Sometimes libcrypto of OpenSSH is compiled with -fomit-frame-pointer - // (any library can be, of course, but this one often is because speed - // is so important for making crypto usable). We ignore all its - // allocations because we can't see the call stacks. We'd prefer - // to ignore allocations done in files/symbols that match - // "default_malloc_ex|default_realloc_ex" - // but that doesn't work when the end-result binary is stripped. - IsLibraryNamed(library, "/libjvm") || - // JVM has a lot of leaks we don't care about. - IsLibraryNamed(library, "/libzip") - // The JVM leaks java.util.zip.Inflater after loading classes. - ) { - depth = 1; // only disable allocation calls directly from the library code - } else if (IsLibraryNamed(library, "/ld") - // library loader leaks some "system" heap - // (e.g. thread-local storage) that we don't care about - ) { - depth = 2; // disable allocation calls directly from the library code - // and at depth 2 from it. - // We need depth 2 here solely because of a libc bug that - // forces us to jump through __memalign_hook and MemalignOverride hoops - // in tcmalloc.cc. - // Those buggy __libc_memalign() calls are in ld-linux.so and happen for - // thread-local storage allocations that we want to ignore here. - // We go with the depth-2 hack as a workaround for this libc bug: - // otherwise we'd need to extend MallocHook interface - // so that correct stack depth adjustment can be propagated from - // the exceptional case of MemalignOverride. - // Using depth 2 here should not mask real leaks because ld-linux.so - // does not call user code. - } - if (depth) { - RAW_VLOG(10, "Disabling allocations from %s at depth %d:", library, depth); - DisableChecksFromToLocked(AsPtr(start_address), AsPtr(end_address), depth); - if (IsLibraryNamed(library, "/libpthread") || - IsLibraryNamed(library, "/libdl") || - IsLibraryNamed(library, "/ld")) { - RAW_VLOG(10, "Global memory regions made by %s will be live data", - library); - if (global_region_caller_ranges == NULL) { - global_region_caller_ranges = - new(Allocator::Allocate(sizeof(GlobalRegionCallerRangeMap))) - GlobalRegionCallerRangeMap; - } - global_region_caller_ranges - ->insert(make_pair(end_address, start_address)); - } - } -} - -// static -HeapLeakChecker::ProcMapsResult HeapLeakChecker::UseProcMapsLocked( - ProcMapsTask proc_maps_task) { - RAW_DCHECK(heap_checker_lock.IsHeld(), ""); - // Need to provide own scratch memory to ProcMapsIterator: - ProcMapsIterator::Buffer buffer; - ProcMapsIterator it(0, &buffer); - if (!it.Valid()) { - int errsv = errno; - RAW_LOG(ERROR, "Could not open /proc/self/maps: errno=%d. " - "Libraries will not be handled correctly.", errsv); - return CANT_OPEN_PROC_MAPS; - } - uint64 start_address, end_address, file_offset; - int64 inode; - char *permissions, *filename; - bool saw_shared_lib = false; - bool saw_nonzero_inode = false; - bool saw_shared_lib_with_nonzero_inode = false; - while (it.Next(&start_address, &end_address, &permissions, - &file_offset, &inode, &filename)) { - if (start_address >= end_address) { - // Warn if a line we can be interested in is ill-formed: - if (inode != 0) { - RAW_LOG(ERROR, "Errors reading /proc/self/maps. " - "Some global memory regions will not " - "be handled correctly."); - } - // Silently skip other ill-formed lines: some are possible - // probably due to the interplay of how /proc/self/maps is updated - // while we read it in chunks in ProcMapsIterator and - // do things in this loop. - continue; - } - // Determine if any shared libraries are present (this is the same - // list of extensions as is found in pprof). We want to ignore - // 'fake' libraries with inode 0 when determining. However, some - // systems don't share inodes via /proc, so we turn off this check - // if we don't see any evidence that we're getting inode info. - if (inode != 0) { - saw_nonzero_inode = true; - } - if ((hc_strstr(filename, "lib") && hc_strstr(filename, ".so")) || - hc_strstr(filename, ".dll") || - // not all .dylib filenames start with lib. .dylib is big enough - // that we are unlikely to get false matches just checking that. - hc_strstr(filename, ".dylib") || hc_strstr(filename, ".bundle")) { - saw_shared_lib = true; - if (inode != 0) { - saw_shared_lib_with_nonzero_inode = true; - } - } - - switch (proc_maps_task) { - case DISABLE_LIBRARY_ALLOCS: - // All lines starting like - // "401dc000-4030f000 r??p 00132000 03:01 13991972 lib/bin" - // identify a data and code sections of a shared library or our binary - if (inode != 0 && strncmp(permissions, "r-xp", 4) == 0) { - DisableLibraryAllocsLocked(filename, start_address, end_address); - } - break; - case RECORD_GLOBAL_DATA: - RecordGlobalDataLocked(start_address, end_address, - permissions, filename); - break; - default: - RAW_CHECK(0, ""); - } - } - // If /proc/self/maps is reporting inodes properly (we saw a - // non-zero inode), then we only say we saw a shared lib if we saw a - // 'real' one, with a non-zero inode. - if (saw_nonzero_inode) { - saw_shared_lib = saw_shared_lib_with_nonzero_inode; - } - if (!saw_shared_lib) { - RAW_LOG(ERROR, "No shared libs detected. Will likely report false leak " - "positives for statically linked executables."); - return NO_SHARED_LIBS_IN_PROC_MAPS; - } - return PROC_MAPS_USED; -} - -// Total number and size of live objects dropped from the profile; -// (re)initialized in IgnoreAllLiveObjectsLocked. -static int64 live_objects_total; -static int64 live_bytes_total; - -// pid of the thread that is doing the current leak check -// (protected by our lock; IgnoreAllLiveObjectsLocked sets it) -static pid_t self_thread_pid = 0; - -// Status of our thread listing callback execution -// (protected by our lock; used from within IgnoreAllLiveObjectsLocked) -static enum { - CALLBACK_NOT_STARTED, - CALLBACK_STARTED, - CALLBACK_COMPLETED, -} thread_listing_status = CALLBACK_NOT_STARTED; - -// Ideally to avoid deadlocks this function should not result in any libc -// or other function calls that might need to lock a mutex: -// It is called when all threads of a process are stopped -// at arbitrary points thus potentially holding those locks. -// -// In practice we are calling some simple i/o and sprintf-type library functions -// for logging messages, but use only our own LowLevelAlloc::Arena allocator. -// -// This is known to be buggy: the library i/o function calls are able to cause -// deadlocks when they request a lock that a stopped thread happens to hold. -// This issue as far as we know have so far not resulted in any deadlocks -// in practice, so for now we are taking our chance that the deadlocks -// have insignificant frequency. -// -// If such deadlocks become a problem we should make the i/o calls -// into appropriately direct system calls (or eliminate them), -// in particular write() is not safe and vsnprintf() is potentially dangerous -// due to reliance on locale functions (these are called through RAW_LOG -// and in other ways). -// - -#if defined(HAVE_LINUX_PTRACE_H) && defined(HAVE_SYS_SYSCALL_H) && defined(DUMPER) -# if (defined(__i386__) || defined(__x86_64)) -# define THREAD_REGS i386_regs -# elif defined(__PPC__) -# define THREAD_REGS ppc_regs -# endif -#endif - -/*static*/ int HeapLeakChecker::IgnoreLiveThreadsLocked(void* parameter, - int num_threads, - pid_t* thread_pids, - va_list /*ap*/) { - RAW_DCHECK(heap_checker_lock.IsHeld(), ""); - thread_listing_status = CALLBACK_STARTED; - RAW_VLOG(11, "Found %d threads (from pid %d)", num_threads, getpid()); - - if (FLAGS_heap_check_ignore_global_live) { - UseProcMapsLocked(RECORD_GLOBAL_DATA); - } - - // We put the registers from other threads here - // to make pointers stored in them live. - vector > thread_registers; - - int failures = 0; - for (int i = 0; i < num_threads; ++i) { - // the leak checking thread itself is handled - // specially via self_thread_stack, not here: - if (thread_pids[i] == self_thread_pid) continue; - RAW_VLOG(11, "Handling thread with pid %d", thread_pids[i]); -#ifdef THREAD_REGS - THREAD_REGS thread_regs; -#define sys_ptrace(r, p, a, d) syscall(SYS_ptrace, (r), (p), (a), (d)) - // We use sys_ptrace to avoid thread locking - // because this is called from TCMalloc_ListAllProcessThreads - // when all but this thread are suspended. - if (sys_ptrace(PTRACE_GETREGS, thread_pids[i], NULL, &thread_regs) == 0) { - // Need to use SP to get all the data from the very last stack frame: - COMPILE_ASSERT(sizeof(thread_regs.SP) == sizeof(void*), - SP_register_does_not_look_like_a_pointer); - RegisterStackLocked(reinterpret_cast(thread_regs.SP)); - // Make registers live (just in case PTRACE_ATTACH resulted in some - // register pointers still being in the registers and not on the stack): - for (void** p = reinterpret_cast(&thread_regs); - p < reinterpret_cast(&thread_regs + 1); ++p) { - RAW_VLOG(12, "Thread register %p", *p); - thread_registers.push_back(*p); - } - } else { - failures += 1; - } -#else - failures += 1; -#endif - } - // Use all the collected thread (stack) liveness sources: - IgnoreLiveObjectsLocked("threads stack data", ""); - if (thread_registers.size()) { - // Make thread registers be live heap data sources. - // we rely here on the fact that vector is in one memory chunk: - RAW_VLOG(11, "Live registers at %p of %" PRIuS " bytes", - &thread_registers[0], thread_registers.size() * sizeof(void*)); - live_objects->push_back(AllocObject(&thread_registers[0], - thread_registers.size() * sizeof(void*), - THREAD_REGISTERS)); - IgnoreLiveObjectsLocked("threads register data", ""); - } - // Do all other liveness walking while all threads are stopped: - IgnoreNonThreadLiveObjectsLocked(); - // Can now resume the threads: - TCMalloc_ResumeAllProcessThreads(num_threads, thread_pids); - thread_listing_status = CALLBACK_COMPLETED; - return failures; -} - -// Stack top of the thread that is doing the current leak check -// (protected by our lock; IgnoreAllLiveObjectsLocked sets it) -static const void* self_thread_stack_top; - -// static -void HeapLeakChecker::IgnoreNonThreadLiveObjectsLocked() { - RAW_DCHECK(heap_checker_lock.IsHeld(), ""); - RAW_DCHECK(MemoryRegionMap::LockIsHeld(), ""); - RAW_VLOG(11, "Handling self thread with pid %d", self_thread_pid); - // Register our own stack: - - // Important that all stack ranges (including the one here) - // are known before we start looking at them - // in MakeDisabledLiveCallbackLocked: - RegisterStackLocked(self_thread_stack_top); - IgnoreLiveObjectsLocked("stack data", ""); - - // Make objects we were told to ignore live: - if (ignored_objects) { - for (IgnoredObjectsMap::const_iterator object = ignored_objects->begin(); - object != ignored_objects->end(); ++object) { - const void* ptr = AsPtr(object->first); - RAW_VLOG(11, "Ignored live object at %p of %" PRIuS " bytes", - ptr, object->second); - live_objects-> - push_back(AllocObject(ptr, object->second, MUST_BE_ON_HEAP)); - // we do this liveness check for ignored_objects before doing any - // live heap walking to make sure it does not fail needlessly: - size_t object_size; - if (!(heap_profile->FindAlloc(ptr, &object_size) && - object->second == object_size)) { - RAW_LOG(FATAL, "Object at %p of %" PRIuS " bytes from an" - " IgnoreObject() has disappeared", ptr, object->second); - } - } - IgnoreLiveObjectsLocked("ignored objects", ""); - } - - // Treat objects that were allocated when a Disabler was live as - // roots. I.e., if X was allocated while a Disabler was active, - // and Y is reachable from X, arrange that neither X nor Y are - // treated as leaks. - heap_profile->IterateAllocs(MakeIgnoredObjectsLiveCallbackLocked); - IgnoreLiveObjectsLocked("disabled objects", ""); - - // Make code-address-disabled objects live and ignored: - // This in particular makes all thread-specific data live - // because the basic data structure to hold pointers to thread-specific data - // is allocated from libpthreads and we have range-disabled that - // library code with UseProcMapsLocked(DISABLE_LIBRARY_ALLOCS); - // so now we declare all thread-specific data reachable from there as live. - heap_profile->IterateAllocs(MakeDisabledLiveCallbackLocked); - IgnoreLiveObjectsLocked("disabled code", ""); - - // Actually make global data live: - if (FLAGS_heap_check_ignore_global_live) { - bool have_null_region_callers = false; - for (LibraryLiveObjectsStacks::iterator l = library_live_objects->begin(); - l != library_live_objects->end(); ++l) { - RAW_CHECK(live_objects->empty(), ""); - // Process library_live_objects in l->second - // filtering them by MemoryRegionMap: - // It's safe to iterate over MemoryRegionMap - // w/o locks here as we are inside MemoryRegionMap::Lock(): - RAW_DCHECK(MemoryRegionMap::LockIsHeld(), ""); - // The only change to MemoryRegionMap possible in this loop - // is region addition as a result of allocating more memory - // for live_objects. This won't invalidate the RegionIterator - // or the intent of the loop. - // --see the comment by MemoryRegionMap::BeginRegionLocked(). - for (MemoryRegionMap::RegionIterator region = - MemoryRegionMap::BeginRegionLocked(); - region != MemoryRegionMap::EndRegionLocked(); ++region) { - // "region" from MemoryRegionMap is to be subtracted from - // (tentatively live) regions in l->second - // if it has a stack inside or it was allocated by - // a non-special caller (not one covered by a range - // in global_region_caller_ranges). - // This will in particular exclude all memory chunks used - // by the heap itself as well as what's been allocated with - // any allocator on top of mmap. - bool subtract = true; - if (!region->is_stack && global_region_caller_ranges) { - if (region->caller() == static_cast(NULL)) { - have_null_region_callers = true; - } else { - GlobalRegionCallerRangeMap::const_iterator iter - = global_region_caller_ranges->upper_bound(region->caller()); - if (iter != global_region_caller_ranges->end()) { - RAW_DCHECK(iter->first > region->caller(), ""); - if (iter->second < region->caller()) { // in special region - subtract = false; - } - } - } - } - if (subtract) { - // The loop puts the result of filtering l->second into live_objects: - for (LiveObjectsStack::const_iterator i = l->second.begin(); - i != l->second.end(); ++i) { - // subtract *region from *i - uintptr_t start = AsInt(i->ptr); - uintptr_t end = start + i->size; - if (region->start_addr <= start && end <= region->end_addr) { - // full deletion due to subsumption - } else if (start < region->start_addr && - region->end_addr < end) { // cutting-out split - live_objects->push_back(AllocObject(i->ptr, - region->start_addr - start, - IN_GLOBAL_DATA)); - live_objects->push_back(AllocObject(AsPtr(region->end_addr), - end - region->end_addr, - IN_GLOBAL_DATA)); - } else if (region->end_addr > start && - region->start_addr <= start) { // cut from start - live_objects->push_back(AllocObject(AsPtr(region->end_addr), - end - region->end_addr, - IN_GLOBAL_DATA)); - } else if (region->start_addr > start && - region->start_addr < end) { // cut from end - live_objects->push_back(AllocObject(i->ptr, - region->start_addr - start, - IN_GLOBAL_DATA)); - } else { // pass: no intersection - live_objects->push_back(AllocObject(i->ptr, i->size, - IN_GLOBAL_DATA)); - } - } - // Move live_objects back into l->second - // for filtering by the next region. - live_objects->swap(l->second); - live_objects->clear(); - } - } - // Now get and use live_objects from the final version of l->second: - if (VLOG_IS_ON(11)) { - for (LiveObjectsStack::const_iterator i = l->second.begin(); - i != l->second.end(); ++i) { - RAW_VLOG(11, "Library live region at %p of %" PRIuPTR " bytes", - i->ptr, i->size); - } - } - live_objects->swap(l->second); - IgnoreLiveObjectsLocked("in globals of\n ", l->first.c_str()); - } - if (have_null_region_callers) { - RAW_LOG(ERROR, "Have memory regions w/o callers: " - "might report false leaks"); - } - Allocator::DeleteAndNull(&library_live_objects); - } -} - -// Callback for TCMalloc_ListAllProcessThreads in IgnoreAllLiveObjectsLocked below -// to test/verify that we have just the one main thread, in which case -// we can do everything in that main thread, -// so that CPU profiler can collect all its samples. -// Returns the number of threads in the process. -static int IsOneThread(void* parameter, int num_threads, - pid_t* thread_pids, va_list ap) { - if (num_threads != 1) { - RAW_LOG(WARNING, "Have threads: Won't CPU-profile the bulk of leak " - "checking work happening in IgnoreLiveThreadsLocked!"); - } - TCMalloc_ResumeAllProcessThreads(num_threads, thread_pids); - return num_threads; -} - -// Dummy for IgnoreAllLiveObjectsLocked below. -// Making it global helps with compiler warnings. -static va_list dummy_ap; - -// static -void HeapLeakChecker::IgnoreAllLiveObjectsLocked(const void* self_stack_top) { - RAW_DCHECK(heap_checker_lock.IsHeld(), ""); - RAW_CHECK(live_objects == NULL, ""); - live_objects = new(Allocator::Allocate(sizeof(LiveObjectsStack))) - LiveObjectsStack; - stack_tops = new(Allocator::Allocate(sizeof(StackTopSet))) StackTopSet; - // reset the counts - live_objects_total = 0; - live_bytes_total = 0; - // Reduce max_heap_object_size to FLAGS_heap_check_max_pointer_offset - // for the time of leak check. - // FLAGS_heap_check_max_pointer_offset caps max_heap_object_size - // to manage reasonably low chances of random bytes - // appearing to be pointing into large actually leaked heap objects. - const size_t old_max_heap_object_size = max_heap_object_size; - max_heap_object_size = ( - FLAGS_heap_check_max_pointer_offset != -1 - ? min(size_t(FLAGS_heap_check_max_pointer_offset), max_heap_object_size) - : max_heap_object_size); - // Record global data as live: - if (FLAGS_heap_check_ignore_global_live) { - library_live_objects = - new(Allocator::Allocate(sizeof(LibraryLiveObjectsStacks))) - LibraryLiveObjectsStacks; - } - // Ignore all thread stacks: - thread_listing_status = CALLBACK_NOT_STARTED; - bool need_to_ignore_non_thread_objects = true; - self_thread_pid = getpid(); - self_thread_stack_top = self_stack_top; - if (FLAGS_heap_check_ignore_thread_live) { - // In case we are doing CPU profiling we'd like to do all the work - // in the main thread, not in the special thread created by - // TCMalloc_ListAllProcessThreads, so that CPU profiler can - // collect all its samples. The machinery of - // TCMalloc_ListAllProcessThreads conflicts with the CPU profiler - // by also relying on signals and ::sigaction. We can do this - // (run everything in the main thread) safely only if there's just - // the main thread itself in our process. This variable reflects - // these two conditions: - bool want_and_can_run_in_main_thread = - ProfilingIsEnabledForAllThreads() && - TCMalloc_ListAllProcessThreads(NULL, IsOneThread) == 1; - // When the normal path of TCMalloc_ListAllProcessThreads below is taken, - // we fully suspend the threads right here before any liveness checking - // and keep them suspended for the whole time of liveness checking - // inside of the IgnoreLiveThreadsLocked callback. - // (The threads can't (de)allocate due to lock on the delete hook but - // if not suspended they could still mess with the pointer - // graph while we walk it). - int r = want_and_can_run_in_main_thread - ? IgnoreLiveThreadsLocked(NULL, 1, &self_thread_pid, dummy_ap) - : TCMalloc_ListAllProcessThreads(NULL, IgnoreLiveThreadsLocked); - need_to_ignore_non_thread_objects = r < 0; - if (r < 0) { - RAW_LOG(WARNING, "Thread finding failed with %d errno=%d", r, errno); - if (thread_listing_status == CALLBACK_COMPLETED) { - RAW_LOG(INFO, "Thread finding callback " - "finished ok; hopefully everything is fine"); - need_to_ignore_non_thread_objects = false; - } else if (thread_listing_status == CALLBACK_STARTED) { - RAW_LOG(FATAL, "Thread finding callback was " - "interrupted or crashed; can't fix this"); - } else { // CALLBACK_NOT_STARTED - RAW_LOG(ERROR, "Could not find thread stacks. " - "Will likely report false leak positives."); - } - } else if (r != 0) { - RAW_LOG(ERROR, "Thread stacks not found for %d threads. " - "Will likely report false leak positives.", r); - } else { - RAW_VLOG(11, "Thread stacks appear to be found for all threads"); - } - } else { - RAW_LOG(WARNING, "Not looking for thread stacks; " - "objects reachable only from there " - "will be reported as leaks"); - } - // Do all other live data ignoring here if we did not do it - // within thread listing callback with all threads stopped. - if (need_to_ignore_non_thread_objects) { - if (FLAGS_heap_check_ignore_global_live) { - UseProcMapsLocked(RECORD_GLOBAL_DATA); - } - IgnoreNonThreadLiveObjectsLocked(); - } - if (live_objects_total) { - RAW_VLOG(10, "Ignoring %" PRId64 " reachable objects of %" PRId64 " bytes", - live_objects_total, live_bytes_total); - } - // Free these: we made them here and heap_profile never saw them - Allocator::DeleteAndNull(&live_objects); - Allocator::DeleteAndNull(&stack_tops); - max_heap_object_size = old_max_heap_object_size; // reset this var -} - -// Alignment at which we should consider pointer positions -// in IgnoreLiveObjectsLocked. Will normally use the value of -// FLAGS_heap_check_pointer_source_alignment. -static size_t pointer_source_alignment = kPointerSourceAlignment; -// Global lock for HeapLeakChecker::DoNoLeaks -// to protect pointer_source_alignment. -static SpinLock alignment_checker_lock(SpinLock::LINKER_INITIALIZED); - -// This function changes the live bits in the heap_profile-table's state: -// we only record the live objects to be skipped. -// -// When checking if a byte sequence points to a heap object we use -// HeapProfileTable::FindInsideAlloc to handle both pointers to -// the start and inside of heap-allocated objects. -// The "inside" case needs to be checked to support -// at least the following relatively common cases: -// - C++ arrays allocated with new FooClass[size] for classes -// with destructors have their size recorded in a sizeof(int) field -// before the place normal pointers point to. -// - basic_string<>-s for e.g. the C++ library of gcc 3.4 -// have the meta-info in basic_string<...>::_Rep recorded -// before the place normal pointers point to. -// - Multiple-inherited objects have their pointers when cast to -// different base classes pointing inside of the actually -// allocated object. -// - Sometimes reachability pointers point to member objects of heap objects, -// and then those member objects point to the full heap object. -// - Third party UnicodeString: it stores a 32-bit refcount -// (in both 32-bit and 64-bit binaries) as the first uint32 -// in the allocated memory and a normal pointer points at -// the second uint32 behind the refcount. -// By finding these additional objects here -// we slightly increase the chance to mistake random memory bytes -// for a pointer and miss a leak in a particular run of a binary. -// -/*static*/ void HeapLeakChecker::IgnoreLiveObjectsLocked(const char* name, - const char* name2) { - RAW_DCHECK(heap_checker_lock.IsHeld(), ""); - int64 live_object_count = 0; - int64 live_byte_count = 0; - while (!live_objects->empty()) { - const char* object = - reinterpret_cast(live_objects->back().ptr); - size_t size = live_objects->back().size; - const ObjectPlacement place = live_objects->back().place; - live_objects->pop_back(); - if (place == MUST_BE_ON_HEAP && heap_profile->MarkAsLive(object)) { - live_object_count += 1; - live_byte_count += size; - } - RAW_VLOG(13, "Looking for heap pointers in %p of %" PRIuS " bytes", - object, size); - const char* const whole_object = object; - size_t const whole_size = size; - // Try interpretting any byte sequence in object,size as a heap pointer: - const size_t remainder = AsInt(object) % pointer_source_alignment; - if (remainder) { - object += pointer_source_alignment - remainder; - if (size >= pointer_source_alignment - remainder) { - size -= pointer_source_alignment - remainder; - } else { - size = 0; - } - } - if (size < sizeof(void*)) continue; - -#ifdef NO_FRAME_POINTER - // Frame pointer omission requires us to use libunwind, which uses direct - // mmap and munmap system calls, and that needs special handling. - if (name2 == kUnnamedProcSelfMapEntry) { - static const uintptr_t page_mask = ~(getpagesize() - 1); - const uintptr_t addr = reinterpret_cast(object); - if ((addr & page_mask) == 0 && (size & page_mask) == 0) { - // This is an object we slurped from /proc/self/maps. - // It may or may not be readable at this point. - // - // In case all the above conditions made a mistake, and the object is - // not related to libunwind, we also verify that it's not readable - // before ignoring it. - if (msync(const_cast(object), size, MS_ASYNC) != 0) { - // Skip unreadable object, so we don't crash trying to sweep it. - RAW_VLOG(0, "Ignoring inaccessible object [%p, %p) " - "(msync error %d (%s))", - object, object + size, errno, strerror(errno)); - continue; - } - } - } -#endif - - const char* const max_object = object + size - sizeof(void*); - while (object <= max_object) { - // potentially unaligned load: - const uintptr_t addr = *reinterpret_cast(object); - // Do fast check before the more expensive HaveOnHeapLocked lookup: - // this code runs for all memory words that are potentially pointers: - const bool can_be_on_heap = - // Order tests by the likelyhood of the test failing in 64/32 bit modes. - // Yes, this matters: we either lose 5..6% speed in 32 bit mode - // (which is already slower) or by a factor of 1.5..1.91 in 64 bit mode. - // After the alignment test got dropped the above performance figures - // must have changed; might need to revisit this. -#if defined(__x86_64__) - addr <= max_heap_address && // <= is for 0-sized object with max addr - min_heap_address <= addr; -#else - min_heap_address <= addr && - addr <= max_heap_address; // <= is for 0-sized object with max addr -#endif - if (can_be_on_heap) { - const void* ptr = reinterpret_cast(addr); - // Too expensive (inner loop): manually uncomment when debugging: - // RAW_VLOG(17, "Trying pointer to %p at %p", ptr, object); - size_t object_size; - if (HaveOnHeapLocked(&ptr, &object_size) && - heap_profile->MarkAsLive(ptr)) { - // We take the (hopefully low) risk here of encountering by accident - // a byte sequence in memory that matches an address of - // a heap object which is in fact leaked. - // I.e. in very rare and probably not repeatable/lasting cases - // we might miss some real heap memory leaks. - RAW_VLOG(14, "Found pointer to %p of %" PRIuS " bytes at %p " - "inside %p of size %" PRIuS "", - ptr, object_size, object, whole_object, whole_size); - if (VLOG_IS_ON(15)) { - // log call stacks to help debug how come something is not a leak - HeapProfileTable::AllocInfo alloc; - if (!heap_profile->FindAllocDetails(ptr, &alloc)) { - RAW_LOG(FATAL, "FindAllocDetails failed on ptr %p", ptr); - } - RAW_LOG(INFO, "New live %p object's alloc stack:", ptr); - for (int i = 0; i < alloc.stack_depth; ++i) { - RAW_LOG(INFO, " @ %p", alloc.call_stack[i]); - } - } - live_object_count += 1; - live_byte_count += object_size; - live_objects->push_back(AllocObject(ptr, object_size, - IGNORED_ON_HEAP)); - } - } - object += pointer_source_alignment; - } - } - live_objects_total += live_object_count; - live_bytes_total += live_byte_count; - if (live_object_count) { - RAW_VLOG(10, "Removed %" PRId64 " live heap objects of %" PRId64 " bytes: %s%s", - live_object_count, live_byte_count, name, name2); - } -} - -//---------------------------------------------------------------------- -// HeapLeakChecker leak check disabling components -//---------------------------------------------------------------------- - -// static -void HeapLeakChecker::DisableChecksIn(const char* pattern) { - RAW_LOG(WARNING, "DisableChecksIn(%s) is ignored", pattern); -} - -// static -void HeapLeakChecker::DoIgnoreObject(const void* ptr) { - SpinLockHolder l(&heap_checker_lock); - if (!heap_checker_on) return; - size_t object_size; - if (!HaveOnHeapLocked(&ptr, &object_size)) { - RAW_LOG(ERROR, "No live heap object at %p to ignore", ptr); - } else { - RAW_VLOG(10, "Going to ignore live object at %p of %" PRIuS " bytes", - ptr, object_size); - if (ignored_objects == NULL) { - ignored_objects = new(Allocator::Allocate(sizeof(IgnoredObjectsMap))) - IgnoredObjectsMap; - } - if (!ignored_objects->insert(make_pair(AsInt(ptr), object_size)).second) { - RAW_LOG(WARNING, "Object at %p is already being ignored", ptr); - } - } -} - -// static -void HeapLeakChecker::UnIgnoreObject(const void* ptr) { - SpinLockHolder l(&heap_checker_lock); - if (!heap_checker_on) return; - size_t object_size; - if (!HaveOnHeapLocked(&ptr, &object_size)) { - RAW_LOG(FATAL, "No live heap object at %p to un-ignore", ptr); - } else { - bool found = false; - if (ignored_objects) { - IgnoredObjectsMap::iterator object = ignored_objects->find(AsInt(ptr)); - if (object != ignored_objects->end() && object_size == object->second) { - ignored_objects->erase(object); - found = true; - RAW_VLOG(10, "Now not going to ignore live object " - "at %p of %" PRIuS " bytes", ptr, object_size); - } - } - if (!found) RAW_LOG(FATAL, "Object at %p has not been ignored", ptr); - } -} - -//---------------------------------------------------------------------- -// HeapLeakChecker non-static functions -//---------------------------------------------------------------------- - -char* HeapLeakChecker::MakeProfileNameLocked() { - RAW_DCHECK(lock_->IsHeld(), ""); - RAW_DCHECK(heap_checker_lock.IsHeld(), ""); - const int len = profile_name_prefix->size() + strlen(name_) + 5 + - strlen(HeapProfileTable::kFileExt) + 1; - char* file_name = reinterpret_cast(Allocator::Allocate(len)); - snprintf(file_name, len, "%s.%s-end%s", - profile_name_prefix->c_str(), name_, - HeapProfileTable::kFileExt); - return file_name; -} - -void HeapLeakChecker::Create(const char *name, bool make_start_snapshot) { - SpinLockHolder l(lock_); - name_ = NULL; // checker is inactive - start_snapshot_ = NULL; - has_checked_ = false; - inuse_bytes_increase_ = 0; - inuse_allocs_increase_ = 0; - keep_profiles_ = false; - char* n = new char[strlen(name) + 1]; // do this before we lock - IgnoreObject(n); // otherwise it might be treated as live due to our stack - { // Heap activity in other threads is paused for this whole scope. - SpinLockHolder al(&alignment_checker_lock); - SpinLockHolder hl(&heap_checker_lock); - MemoryRegionMap::LockHolder ml; - if (heap_checker_on && profile_name_prefix != NULL) { - RAW_DCHECK(strchr(name, '/') == NULL, "must be a simple name"); - memcpy(n, name, strlen(name) + 1); - name_ = n; // checker is active - if (make_start_snapshot) { - start_snapshot_ = heap_profile->TakeSnapshot(); - } - - const HeapProfileTable::Stats& t = heap_profile->total(); - const size_t start_inuse_bytes = t.alloc_size - t.free_size; - const size_t start_inuse_allocs = t.allocs - t.frees; - RAW_VLOG(10, "Start check \"%s\" profile: %" PRIuS " bytes " - "in %" PRIuS " objects", - name_, start_inuse_bytes, start_inuse_allocs); - } else { - RAW_LOG(WARNING, "Heap checker is not active, " - "hence checker \"%s\" will do nothing!", name); - RAW_LOG(WARNING, "To activate set the HEAPCHECK environment variable.\n"); - } - } - if (name_ == NULL) { - UnIgnoreObject(n); - delete[] n; // must be done after we unlock - } -} - -HeapLeakChecker::HeapLeakChecker(const char *name) : lock_(new SpinLock) { - RAW_DCHECK(strcmp(name, "_main_") != 0, "_main_ is reserved"); - Create(name, true/*create start_snapshot_*/); -} - -HeapLeakChecker::HeapLeakChecker() : lock_(new SpinLock) { - if (FLAGS_heap_check_before_constructors) { - // We want to check for leaks of objects allocated during global - // constructors (i.e., objects allocated already). So we do not - // create a baseline snapshot and hence check for leaks of objects - // that may have already been created. - Create("_main_", false); - } else { - // We want to ignore leaks of objects allocated during global - // constructors (i.e., objects allocated already). So we snapshot - // the current heap contents and use them as a baseline that is - // not reported by the leak checker. - Create("_main_", true); - } -} - -ssize_t HeapLeakChecker::BytesLeaked() const { - SpinLockHolder l(lock_); - if (!has_checked_) { - RAW_LOG(FATAL, "*NoLeaks|SameHeap must execute before this call"); - } - return inuse_bytes_increase_; -} - -ssize_t HeapLeakChecker::ObjectsLeaked() const { - SpinLockHolder l(lock_); - if (!has_checked_) { - RAW_LOG(FATAL, "*NoLeaks|SameHeap must execute before this call"); - } - return inuse_allocs_increase_; -} - -// Save pid of main thread for using in naming dump files -static int32 main_thread_pid = getpid(); -#ifdef HAVE_PROGRAM_INVOCATION_NAME -#ifdef __UCLIBC__ -extern const char* program_invocation_name; -extern const char* program_invocation_short_name; -#else -extern char* program_invocation_name; -extern char* program_invocation_short_name; -#endif -static const char* invocation_name() { return program_invocation_short_name; } -static string invocation_path() { return program_invocation_name; } -#else -static const char* invocation_name() { return ""; } -static string invocation_path() { return ""; } -#endif - -// Prints commands that users can run to get more information -// about the reported leaks. -static void SuggestPprofCommand(const char* pprof_file_arg) { - // Extra help information to print for the user when the test is - // being run in a way where the straightforward pprof command will - // not suffice. - string extra_help; - - // Common header info to print for remote runs - const string remote_header = - "This program is being executed remotely and therefore the pprof\n" - "command printed above will not work. Either run this program\n" - "locally, or adjust the pprof command as follows to allow it to\n" - "work on your local machine:\n"; - - // Extra command for fetching remote data - string fetch_cmd; - - RAW_LOG(WARNING, - "\n\n" - "If the preceding stack traces are not enough to find " - "the leaks, try running THIS shell command:\n\n" - "%s%s %s \"%s\" --inuse_objects --lines --heapcheck " - " --edgefraction=1e-10 --nodefraction=1e-10 --gv\n" - "\n" - "%s" - "If you are still puzzled about why the leaks are " - "there, try rerunning this program with " - "HEAP_CHECK_TEST_POINTER_ALIGNMENT=1 and/or with " - "HEAP_CHECK_MAX_POINTER_OFFSET=-1\n" - "If the leak report occurs in a small fraction of runs, " - "try running with TCMALLOC_MAX_FREE_QUEUE_SIZE of few hundred MB " - "or with TCMALLOC_RECLAIM_MEMORY=false, " // only works for debugalloc - "it might help find leaks more repeatably\n", - fetch_cmd.c_str(), - "pprof", // works as long as pprof is on your path - invocation_path().c_str(), - pprof_file_arg, - extra_help.c_str() - ); -} - -bool HeapLeakChecker::DoNoLeaks(ShouldSymbolize should_symbolize) { - SpinLockHolder l(lock_); - // The locking also helps us keep the messages - // for the two checks close together. - SpinLockHolder al(&alignment_checker_lock); - - // thread-safe: protected by alignment_checker_lock - static bool have_disabled_hooks_for_symbolize = false; - // Once we've checked for leaks and symbolized the results once, it's - // not safe to do it again. This is because in order to symbolize - // safely, we had to disable all the malloc hooks here, so we no - // longer can be confident we've collected all the data we need. - if (have_disabled_hooks_for_symbolize) { - RAW_LOG(FATAL, "Must not call heap leak checker manually after " - " program-exit's automatic check."); - } - - HeapProfileTable::Snapshot* leaks = NULL; - char* pprof_file = NULL; - - { - // Heap activity in other threads is paused during this function - // (i.e. until we got all profile difference info). - SpinLockHolder hl(&heap_checker_lock); - if (heap_checker_on == false) { - if (name_ != NULL) { // leak checking enabled when created the checker - RAW_LOG(WARNING, "Heap leak checker got turned off after checker " - "\"%s\" has been created, no leak check is being done for it!", - name_); - } - return true; - } - - // Update global_region_caller_ranges. They may need to change since - // e.g. initialization because shared libraries might have been loaded or - // unloaded. - Allocator::DeleteAndNullIfNot(&global_region_caller_ranges); - ProcMapsResult pm_result = UseProcMapsLocked(DISABLE_LIBRARY_ALLOCS); - RAW_CHECK(pm_result == PROC_MAPS_USED, ""); - - // Keep track of number of internally allocated objects so we - // can detect leaks in the heap-leak-checket itself - const int initial_allocs = Allocator::alloc_count(); - - if (name_ == NULL) { - RAW_LOG(FATAL, "Heap leak checker must not be turned on " - "after construction of a HeapLeakChecker"); - } - - MemoryRegionMap::LockHolder ml; - int a_local_var; // Use our stack ptr to make stack data live: - - // Make the heap profile, other threads are locked out. - HeapProfileTable::Snapshot* base = - reinterpret_cast(start_snapshot_); - RAW_DCHECK(FLAGS_heap_check_pointer_source_alignment > 0, ""); - pointer_source_alignment = FLAGS_heap_check_pointer_source_alignment; - IgnoreAllLiveObjectsLocked(&a_local_var); - leaks = heap_profile->NonLiveSnapshot(base); - - inuse_bytes_increase_ = static_cast(leaks->total().alloc_size); - inuse_allocs_increase_ = static_cast(leaks->total().allocs); - if (leaks->Empty()) { - heap_profile->ReleaseSnapshot(leaks); - leaks = NULL; - - // We can only check for internal leaks along the no-user-leak - // path since in the leak path we temporarily release - // heap_checker_lock and another thread can come in and disturb - // allocation counts. - if (Allocator::alloc_count() != initial_allocs) { - RAW_LOG(FATAL, "Internal HeapChecker leak of %d objects ; %d -> %d", - Allocator::alloc_count() - initial_allocs, - initial_allocs, Allocator::alloc_count()); - } - } else if (FLAGS_heap_check_test_pointer_alignment) { - if (pointer_source_alignment == 1) { - RAW_LOG(WARNING, "--heap_check_test_pointer_alignment has no effect: " - "--heap_check_pointer_source_alignment was already set to 1"); - } else { - // Try with reduced pointer aligment - pointer_source_alignment = 1; - IgnoreAllLiveObjectsLocked(&a_local_var); - HeapProfileTable::Snapshot* leaks_wo_align = - heap_profile->NonLiveSnapshot(base); - pointer_source_alignment = FLAGS_heap_check_pointer_source_alignment; - if (leaks_wo_align->Empty()) { - RAW_LOG(WARNING, "Found no leaks without pointer alignment: " - "something might be placing pointers at " - "unaligned addresses! This needs to be fixed."); - } else { - RAW_LOG(INFO, "Found leaks without pointer alignment as well: " - "unaligned pointers must not be the cause of leaks."); - RAW_LOG(INFO, "--heap_check_test_pointer_alignment did not help " - "to diagnose the leaks."); - } - heap_profile->ReleaseSnapshot(leaks_wo_align); - } - } - - if (leaks != NULL) { - pprof_file = MakeProfileNameLocked(); - } - } - - has_checked_ = true; - if (leaks == NULL) { - if (FLAGS_heap_check_max_pointer_offset == -1) { - RAW_LOG(WARNING, - "Found no leaks without max_pointer_offset restriction: " - "it's possible that the default value of " - "heap_check_max_pointer_offset flag is too low. " - "Do you use pointers with larger than that offsets " - "pointing in the middle of heap-allocated objects?"); - } - const HeapProfileTable::Stats& stats = heap_profile->total(); - RAW_VLOG(heap_checker_info_level, - "No leaks found for check \"%s\" " - "(but no 100%% guarantee that there aren't any): " - "found %" PRId64 " reachable heap objects of %" PRId64 " bytes", - name_, - int64(stats.allocs - stats.frees), - int64(stats.alloc_size - stats.free_size)); - } else { - if (should_symbolize == SYMBOLIZE) { - // To turn addresses into symbols, we need to fork, which is a - // problem if both parent and child end up trying to call the - // same malloc-hooks we've set up, at the same time. To avoid - // trouble, we turn off the hooks before symbolizing. Note that - // this makes it unsafe to ever leak-report again! Luckily, we - // typically only want to report once in a program's run, at the - // very end. - if (MallocHook::GetNewHook() == NewHook) - MallocHook::SetNewHook(NULL); - if (MallocHook::GetDeleteHook() == DeleteHook) - MallocHook::SetDeleteHook(NULL); - MemoryRegionMap::Shutdown(); - // Make sure all the hooks really got unset: - RAW_CHECK(MallocHook::GetNewHook() == NULL, ""); - RAW_CHECK(MallocHook::GetDeleteHook() == NULL, ""); - RAW_CHECK(MallocHook::GetMmapHook() == NULL, ""); - RAW_CHECK(MallocHook::GetSbrkHook() == NULL, ""); - have_disabled_hooks_for_symbolize = true; - leaks->ReportLeaks(name_, pprof_file, true); // true = should_symbolize - } else { - leaks->ReportLeaks(name_, pprof_file, false); - } - if (FLAGS_heap_check_identify_leaks) { - leaks->ReportIndividualObjects(); - } - - SuggestPprofCommand(pprof_file); - - { - SpinLockHolder hl(&heap_checker_lock); - heap_profile->ReleaseSnapshot(leaks); - Allocator::Free(pprof_file); - } - } - - return (leaks == NULL); -} - -HeapLeakChecker::~HeapLeakChecker() { - if (name_ != NULL) { // had leak checking enabled when created the checker - if (!has_checked_) { - RAW_LOG(FATAL, "Some *NoLeaks|SameHeap method" - " must be called on any created HeapLeakChecker"); - } - - // Deallocate any snapshot taken at start - if (start_snapshot_ != NULL) { - SpinLockHolder l(&heap_checker_lock); - heap_profile->ReleaseSnapshot( - reinterpret_cast(start_snapshot_)); - } - - UnIgnoreObject(name_); - delete[] name_; - name_ = NULL; - } - delete lock_; -} - -//---------------------------------------------------------------------- -// HeapLeakChecker overall heap check components -//---------------------------------------------------------------------- - -// static -bool HeapLeakChecker::IsActive() { - SpinLockHolder l(&heap_checker_lock); - return heap_checker_on; -} - -vector* HeapCleaner::heap_cleanups_ = NULL; - -// When a HeapCleaner object is intialized, add its function to the static list -// of cleaners to be run before leaks checking. -HeapCleaner::HeapCleaner(void_function f) { - if (heap_cleanups_ == NULL) - heap_cleanups_ = new vector; - heap_cleanups_->push_back(f); -} - -// Run all of the cleanup functions and delete the vector. -void HeapCleaner::RunHeapCleanups() { - if (!heap_cleanups_) - return; - for (int i = 0; i < heap_cleanups_->size(); i++) { - void (*f)(void) = (*heap_cleanups_)[i]; - f(); - } - delete heap_cleanups_; - heap_cleanups_ = NULL; -} - -// Program exit heap cleanup registered as a module object destructor. -// Will not get executed when we crash on a signal. -// -void HeapLeakChecker_RunHeapCleanups() { - if (FLAGS_heap_check == "local") // don't check heap in this mode - return; - { SpinLockHolder l(&heap_checker_lock); - // can get here (via forks?) with other pids - if (heap_checker_pid != getpid()) return; - } - HeapCleaner::RunHeapCleanups(); - if (!FLAGS_heap_check_after_destructors) HeapLeakChecker::DoMainHeapCheck(); -} - -static bool internal_init_start_has_run = false; - -// Called exactly once, before main() (but hopefully just before). -// This picks a good unique name for the dumped leak checking heap profiles. -// -// Because we crash when InternalInitStart is called more than once, -// it's fine that we hold heap_checker_lock only around pieces of -// this function: this is still enough for thread-safety w.r.t. other functions -// of this module. -// We can't hold heap_checker_lock throughout because it would deadlock -// on a memory allocation since our new/delete hooks can be on. -// -void HeapLeakChecker_InternalInitStart() { - { SpinLockHolder l(&heap_checker_lock); - RAW_CHECK(!internal_init_start_has_run, - "Heap-check constructor called twice. Perhaps you both linked" - " in the heap checker, and also used LD_PRELOAD to load it?"); - internal_init_start_has_run = true; - -#ifdef ADDRESS_SANITIZER - // AddressSanitizer's custom malloc conflicts with HeapChecker. - FLAGS_heap_check = ""; -#endif - - if (FLAGS_heap_check.empty()) { - // turns out we do not need checking in the end; can stop profiling - HeapLeakChecker::TurnItselfOffLocked(); - return; - } else if (RunningOnValgrind()) { - // There is no point in trying -- we'll just fail. - RAW_LOG(WARNING, "Can't run under Valgrind; will turn itself off"); - HeapLeakChecker::TurnItselfOffLocked(); - return; - } - } - - // Changing this to false can be useful when debugging heap-checker itself: - if (!FLAGS_heap_check_run_under_gdb && IsDebuggerAttached()) { - RAW_LOG(WARNING, "Someone is ptrace()ing us; will turn itself off"); - SpinLockHolder l(&heap_checker_lock); - HeapLeakChecker::TurnItselfOffLocked(); - return; - } - - { SpinLockHolder l(&heap_checker_lock); - if (!constructor_heap_profiling) { - RAW_LOG(FATAL, "Can not start so late. You have to enable heap checking " - "with HEAPCHECK=."); - } - } - - // Set all flags - RAW_DCHECK(FLAGS_heap_check_pointer_source_alignment > 0, ""); - if (FLAGS_heap_check == "minimal") { - // The least we can check. - FLAGS_heap_check_before_constructors = false; // from after main - // (ignore more) - FLAGS_heap_check_after_destructors = false; // to after cleanup - // (most data is live) - FLAGS_heap_check_ignore_thread_live = true; // ignore all live - FLAGS_heap_check_ignore_global_live = true; // ignore all live - } else if (FLAGS_heap_check == "normal") { - // Faster than 'minimal' and not much stricter. - FLAGS_heap_check_before_constructors = true; // from no profile (fast) - FLAGS_heap_check_after_destructors = false; // to after cleanup - // (most data is live) - FLAGS_heap_check_ignore_thread_live = true; // ignore all live - FLAGS_heap_check_ignore_global_live = true; // ignore all live - } else if (FLAGS_heap_check == "strict") { - // A bit stricter than 'normal': global destructors must fully clean up - // after themselves if they are present. - FLAGS_heap_check_before_constructors = true; // from no profile (fast) - FLAGS_heap_check_after_destructors = true; // to after destructors - // (less data live) - FLAGS_heap_check_ignore_thread_live = true; // ignore all live - FLAGS_heap_check_ignore_global_live = true; // ignore all live - } else if (FLAGS_heap_check == "draconian") { - // Drop not very portable and not very exact live heap flooding. - FLAGS_heap_check_before_constructors = true; // from no profile (fast) - FLAGS_heap_check_after_destructors = true; // to after destructors - // (need them) - FLAGS_heap_check_ignore_thread_live = false; // no live flood (stricter) - FLAGS_heap_check_ignore_global_live = false; // no live flood (stricter) - } else if (FLAGS_heap_check == "as-is") { - // do nothing: use other flags as is - } else if (FLAGS_heap_check == "local") { - // do nothing - } else { - RAW_LOG(FATAL, "Unsupported heap_check flag: %s", - FLAGS_heap_check.c_str()); - } - // FreeBSD doesn't seem to honor atexit execution order: - // http://code.google.com/p/gperftools/issues/detail?id=375 - // Since heap-checking before destructors depends on atexit running - // at the right time, on FreeBSD we always check after, even in the - // less strict modes. This just means FreeBSD is always a bit - // stricter in its checking than other OSes. - // This now appears to be the case in other OSes as well; - // so always check afterwards. - FLAGS_heap_check_after_destructors = true; - - { SpinLockHolder l(&heap_checker_lock); - RAW_DCHECK(heap_checker_pid == getpid(), ""); - heap_checker_on = true; - RAW_DCHECK(heap_profile, ""); - HeapLeakChecker::ProcMapsResult pm_result = HeapLeakChecker::UseProcMapsLocked(HeapLeakChecker::DISABLE_LIBRARY_ALLOCS); - // might neeed to do this more than once - // if one later dynamically loads libraries that we want disabled - if (pm_result != HeapLeakChecker::PROC_MAPS_USED) { // can't function - HeapLeakChecker::TurnItselfOffLocked(); - return; - } - } - - // make a good place and name for heap profile leak dumps - string* profile_prefix = - new string(FLAGS_heap_check_dump_directory + "/" + invocation_name()); - - // Finalize prefix for dumping leak checking profiles. - const int32 our_pid = getpid(); // safest to call getpid() outside lock - { SpinLockHolder l(&heap_checker_lock); - // main_thread_pid might still be 0 if this function is being called before - // global constructors. In that case, our pid *is* the main pid. - if (main_thread_pid == 0) - main_thread_pid = our_pid; - } - char pid_buf[15]; - snprintf(pid_buf, sizeof(pid_buf), ".%d", main_thread_pid); - *profile_prefix += pid_buf; - { SpinLockHolder l(&heap_checker_lock); - RAW_DCHECK(profile_name_prefix == NULL, ""); - profile_name_prefix = profile_prefix; - } - - // Make sure new/delete hooks are installed properly - // and heap profiler is indeed able to keep track - // of the objects being allocated. - // We test this to make sure we are indeed checking for leaks. - char* test_str = new char[5]; - size_t size; - { SpinLockHolder l(&heap_checker_lock); - RAW_CHECK(heap_profile->FindAlloc(test_str, &size), - "our own new/delete not linked?"); - } - delete[] test_str; - { SpinLockHolder l(&heap_checker_lock); - // This check can fail when it should not if another thread allocates - // into this same spot right this moment, - // which is unlikely since this code runs in InitGoogle. - RAW_CHECK(!heap_profile->FindAlloc(test_str, &size), - "our own new/delete not linked?"); - } - // If we crash in the above code, it probably means that - // "nm | grep new" will show that tcmalloc's new/delete - // implementation did not get linked-in into this binary - // (i.e. nm will list __builtin_new and __builtin_vec_new as undefined). - // If this happens, it is a BUILD bug to be fixed. - - RAW_VLOG(heap_checker_info_level, - "WARNING: Perftools heap leak checker is active " - "-- Performance may suffer"); - - if (FLAGS_heap_check != "local") { - HeapLeakChecker* main_hc = new HeapLeakChecker(); - SpinLockHolder l(&heap_checker_lock); - RAW_DCHECK(main_heap_checker == NULL, - "Repeated creation of main_heap_checker"); - main_heap_checker = main_hc; - do_main_heap_check = true; - } - - { SpinLockHolder l(&heap_checker_lock); - RAW_CHECK(heap_checker_on && constructor_heap_profiling, - "Leak checking is expected to be fully turned on now"); - } - - // For binaries built in debug mode, this will set release queue of - // debugallocation.cc to 100M to make it less likely for real leaks to - // be hidden due to reuse of heap memory object addresses. - // Running a test with --malloc_reclaim_memory=0 would help find leaks even - // better, but the test might run out of memory as a result. - // The scenario is that a heap object at address X is allocated and freed, - // but some other data-structure still retains a pointer to X. - // Then the same heap memory is used for another object, which is leaked, - // but the leak is not noticed due to the pointer to the original object at X. - // TODO(csilvers): support this in some manner. -#if 0 - SetCommandLineOptionWithMode("max_free_queue_size", "104857600", // 100M - SET_FLAG_IF_DEFAULT); -#endif -} - -// We want this to run early as well, but not so early as -// ::BeforeConstructors (we want flag assignments to have already -// happened, for instance). Initializer-registration does the trick. -REGISTER_MODULE_INITIALIZER(init_start, HeapLeakChecker_InternalInitStart()); -REGISTER_MODULE_DESTRUCTOR(init_start, HeapLeakChecker_RunHeapCleanups()); - -// static -bool HeapLeakChecker::NoGlobalLeaksMaybeSymbolize( - ShouldSymbolize should_symbolize) { - // we never delete or change main_heap_checker once it's set: - HeapLeakChecker* main_hc = GlobalChecker(); - if (main_hc) { - RAW_VLOG(10, "Checking for whole-program memory leaks"); - return main_hc->DoNoLeaks(should_symbolize); - } - return true; -} - -// static -bool HeapLeakChecker::DoMainHeapCheck() { - if (FLAGS_heap_check_delay_seconds > 0) { - sleep(FLAGS_heap_check_delay_seconds); - } - { SpinLockHolder l(&heap_checker_lock); - if (!do_main_heap_check) return false; - RAW_DCHECK(heap_checker_pid == getpid(), ""); - do_main_heap_check = false; // will do it now; no need to do it more - } - - // The program is over, so it's safe to symbolize addresses (which - // requires a fork) because no serious work is expected to be done - // after this. Symbolizing is really useful -- knowing what - // function has a leak is better than knowing just an address -- - // and while we can only safely symbolize once in a program run, - // now is the time (after all, there's no "later" that would be better). - if (!NoGlobalLeaksMaybeSymbolize(SYMBOLIZE)) { - if (FLAGS_heap_check_identify_leaks) { - RAW_LOG(FATAL, "Whole-program memory leaks found."); - } - RAW_LOG(ERROR, "Exiting with error code (instead of crashing) " - "because of whole-program memory leaks"); - _exit(1); // we don't want to call atexit() routines! - } - return true; -} - -// static -HeapLeakChecker* HeapLeakChecker::GlobalChecker() { - SpinLockHolder l(&heap_checker_lock); - return main_heap_checker; -} - -// static -bool HeapLeakChecker::NoGlobalLeaks() { - // symbolizing requires a fork, which isn't safe to do in general. - return NoGlobalLeaksMaybeSymbolize(DO_NOT_SYMBOLIZE); -} - -// static -void HeapLeakChecker::CancelGlobalCheck() { - SpinLockHolder l(&heap_checker_lock); - if (do_main_heap_check) { - RAW_VLOG(heap_checker_info_level, - "Canceling the automatic at-exit whole-program memory leak check"); - do_main_heap_check = false; - } -} - -// static -void HeapLeakChecker::BeforeConstructorsLocked() { - RAW_DCHECK(heap_checker_lock.IsHeld(), ""); - RAW_CHECK(!constructor_heap_profiling, - "BeforeConstructorsLocked called multiple times"); -#ifdef ADDRESS_SANITIZER - // AddressSanitizer's custom malloc conflicts with HeapChecker. - return; -#endif - // Set hooks early to crash if 'new' gets called before we make heap_profile, - // and make sure no other hooks existed: - RAW_CHECK(MallocHook::AddNewHook(&NewHook), ""); - RAW_CHECK(MallocHook::AddDeleteHook(&DeleteHook), ""); - constructor_heap_profiling = true; - MemoryRegionMap::Init(1, /* use_buckets */ false); - // Set up MemoryRegionMap with (at least) one caller stack frame to record - // (important that it's done before HeapProfileTable creation below). - Allocator::Init(); - RAW_CHECK(heap_profile == NULL, ""); - heap_profile = new(Allocator::Allocate(sizeof(HeapProfileTable))) - HeapProfileTable(&Allocator::Allocate, &Allocator::Free, - /* profile_mmap */ false); - RAW_VLOG(10, "Starting tracking the heap"); - heap_checker_on = true; -} - -// static -void HeapLeakChecker::TurnItselfOffLocked() { - RAW_DCHECK(heap_checker_lock.IsHeld(), ""); - // Set FLAGS_heap_check to "", for users who test for it - if (!FLAGS_heap_check.empty()) // be a noop in the common case - FLAGS_heap_check.clear(); // because clear() could allocate memory - if (constructor_heap_profiling) { - RAW_CHECK(heap_checker_on, ""); - RAW_VLOG(heap_checker_info_level, "Turning perftools heap leak checking off"); - heap_checker_on = false; - // Unset our hooks checking they were set: - RAW_CHECK(MallocHook::RemoveNewHook(&NewHook), ""); - RAW_CHECK(MallocHook::RemoveDeleteHook(&DeleteHook), ""); - Allocator::DeleteAndNull(&heap_profile); - // free our optional global data: - Allocator::DeleteAndNullIfNot(&ignored_objects); - Allocator::DeleteAndNullIfNot(&disabled_ranges); - Allocator::DeleteAndNullIfNot(&global_region_caller_ranges); - Allocator::Shutdown(); - MemoryRegionMap::Shutdown(); - } - RAW_CHECK(!heap_checker_on, ""); -} - -extern bool heap_leak_checker_bcad_variable; // in heap-checker-bcad.cc - -static bool has_called_before_constructors = false; - -// TODO(maxim): inline this function with -// MallocHook_InitAtFirstAllocation_HeapLeakChecker, and also rename -// HeapLeakChecker::BeforeConstructorsLocked. -void HeapLeakChecker_BeforeConstructors() { - SpinLockHolder l(&heap_checker_lock); - // We can be called from several places: the first mmap/sbrk/alloc call - // or the first global c-tor from heap-checker-bcad.cc: - // Do not re-execute initialization: - if (has_called_before_constructors) return; - has_called_before_constructors = true; - - heap_checker_pid = getpid(); // set it always - heap_leak_checker_bcad_variable = true; - // just to reference it, so that heap-checker-bcad.o is linked in - - // This function can be called *very* early, before the normal - // global-constructor that sets FLAGS_verbose. Set it manually now, - // so the RAW_LOG messages here are controllable. - const char* verbose_str = GetenvBeforeMain("PERFTOOLS_VERBOSE"); - if (verbose_str && atoi(verbose_str)) { // different than the default of 0? - FLAGS_verbose = atoi(verbose_str); - } - - bool need_heap_check = true; - // The user indicates a desire for heap-checking via the HEAPCHECK - // environment variable. If it's not set, there's no way to do - // heap-checking. - if (!GetenvBeforeMain("HEAPCHECK")) { - need_heap_check = false; - } -#ifdef HAVE_GETEUID - if (need_heap_check && getuid() != geteuid()) { - // heap-checker writes out files. Thus, for security reasons, we don't - // recognize the env. var. to turn on heap-checking if we're setuid. - RAW_LOG(WARNING, ("HeapChecker: ignoring HEAPCHECK because " - "program seems to be setuid\n")); - need_heap_check = false; - } -#endif - if (need_heap_check) { - HeapLeakChecker::BeforeConstructorsLocked(); - } -} - -// This function overrides the weak function defined in malloc_hook.cc and -// called by one of the initial malloc hooks (malloc_hook.cc) when the very -// first memory allocation or an mmap/sbrk happens. This ensures that -// HeapLeakChecker is initialized and installs all its hooks early enough to -// track absolutely all memory allocations and all memory region acquisitions -// via mmap and sbrk. -extern "C" void MallocHook_InitAtFirstAllocation_HeapLeakChecker() { - HeapLeakChecker_BeforeConstructors(); -} - -// This function is executed after all global object destructors run. -void HeapLeakChecker_AfterDestructors() { - { SpinLockHolder l(&heap_checker_lock); - // can get here (via forks?) with other pids - if (heap_checker_pid != getpid()) return; - } - if (FLAGS_heap_check_after_destructors) { - if (HeapLeakChecker::DoMainHeapCheck()) { - const struct timespec sleep_time = { 0, 500000000 }; // 500 ms - nanosleep(&sleep_time, NULL); - // Need this hack to wait for other pthreads to exit. - // Otherwise tcmalloc find errors - // on a free() call from pthreads. - } - } - SpinLockHolder l(&heap_checker_lock); - RAW_CHECK(!do_main_heap_check, "should have done it"); -} - -//---------------------------------------------------------------------- -// HeapLeakChecker disabling helpers -//---------------------------------------------------------------------- - -// These functions are at the end of the file to prevent their inlining: - -// static -void HeapLeakChecker::DisableChecksFromToLocked(const void* start_address, - const void* end_address, - int max_depth) { - RAW_DCHECK(heap_checker_lock.IsHeld(), ""); - RAW_DCHECK(start_address < end_address, ""); - if (disabled_ranges == NULL) { - disabled_ranges = new(Allocator::Allocate(sizeof(DisabledRangeMap))) - DisabledRangeMap; - } - RangeValue value; - value.start_address = AsInt(start_address); - value.max_depth = max_depth; - if (disabled_ranges->insert(make_pair(AsInt(end_address), value)).second) { - RAW_VLOG(10, "Disabling leak checking in stack traces " - "under frame addresses between %p..%p", - start_address, end_address); - } else { // check that this is just a verbatim repetition - RangeValue const& val = disabled_ranges->find(AsInt(end_address))->second; - if (val.max_depth != value.max_depth || - val.start_address != value.start_address) { - RAW_LOG(FATAL, "Two DisableChecksToHereFrom calls conflict: " - "(%p, %p, %d) vs. (%p, %p, %d)", - AsPtr(val.start_address), end_address, val.max_depth, - start_address, end_address, max_depth); - } - } -} - -// static -inline bool HeapLeakChecker::HaveOnHeapLocked(const void** ptr, - size_t* object_size) { - // Commented-out because HaveOnHeapLocked is very performance-critical: - // RAW_DCHECK(heap_checker_lock.IsHeld(), ""); - const uintptr_t addr = AsInt(*ptr); - if (heap_profile->FindInsideAlloc( - *ptr, max_heap_object_size, ptr, object_size)) { - RAW_VLOG(16, "Got pointer into %p at +%" PRIuPTR " offset", - *ptr, addr - AsInt(*ptr)); - return true; - } - return false; -} - -// static -const void* HeapLeakChecker::GetAllocCaller(void* ptr) { - // this is used only in the unittest, so the heavy checks are fine - HeapProfileTable::AllocInfo info; - { SpinLockHolder l(&heap_checker_lock); - RAW_CHECK(heap_profile->FindAllocDetails(ptr, &info), ""); - } - RAW_CHECK(info.stack_depth >= 1, ""); - return info.call_stack[0]; -} diff --git a/contrib/libtcmalloc/src/heap-profile-stats.h b/contrib/libtcmalloc/src/heap-profile-stats.h deleted file mode 100644 index ae45d5883fa..00000000000 --- a/contrib/libtcmalloc/src/heap-profile-stats.h +++ /dev/null @@ -1,78 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2013, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// This file defines structs to accumulate memory allocation and deallocation -// counts. These structs are commonly used for malloc (in HeapProfileTable) -// and mmap (in MemoryRegionMap). - -// A bucket is data structure for heap profiling to store a pair of a stack -// trace and counts of (de)allocation. Buckets are stored in a hash table -// which is declared as "HeapProfileBucket**". -// -// A hash value is computed from a stack trace. Collision in the hash table -// is resolved by separate chaining with linked lists. The links in the list -// are implemented with the member "HeapProfileBucket* next". -// -// A structure of a hash table HeapProfileBucket** bucket_table would be like: -// bucket_table[0] => NULL -// bucket_table[1] => HeapProfileBucket() => HeapProfileBucket() => NULL -// ... -// bucket_table[i] => HeapProfileBucket() => NULL -// ... -// bucket_table[n] => HeapProfileBucket() => NULL - -#ifndef HEAP_PROFILE_STATS_H_ -#define HEAP_PROFILE_STATS_H_ - -struct HeapProfileStats { - // Returns true if the two HeapProfileStats are semantically equal. - bool Equivalent(const HeapProfileStats& other) const { - return allocs - frees == other.allocs - other.frees && - alloc_size - free_size == other.alloc_size - other.free_size; - } - - int32 allocs; // Number of allocation calls. - int32 frees; // Number of free calls. - int64 alloc_size; // Total size of all allocated objects so far. - int64 free_size; // Total size of all freed objects so far. -}; - -// Allocation and deallocation statistics per each stack trace. -struct HeapProfileBucket : public HeapProfileStats { - // Longest stack trace we record. - static const int kMaxStackDepth = 32; - - uintptr_t hash; // Hash value of the stack trace. - int depth; // Depth of stack trace. - const void** stack; // Stack trace. - HeapProfileBucket* next; // Next entry in hash-table. -}; - -#endif // HEAP_PROFILE_STATS_H_ diff --git a/contrib/libtcmalloc/src/heap-profile-table.cc b/contrib/libtcmalloc/src/heap-profile-table.cc deleted file mode 100644 index 7486468c056..00000000000 --- a/contrib/libtcmalloc/src/heap-profile-table.cc +++ /dev/null @@ -1,631 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2006, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat -// Maxim Lifantsev (refactoring) -// - -#include - -#ifdef HAVE_UNISTD_H -#include // for write() -#endif -#include // for open() -#ifdef HAVE_GLOB_H -#include -#ifndef GLOB_NOMATCH // true on some old cygwins -# define GLOB_NOMATCH 0 -#endif -#endif -#ifdef HAVE_INTTYPES_H -#include // for PRIxPTR -#endif -#ifdef HAVE_POLL_H -#include -#endif -#include -#include -#include -#include -#include // for sort(), equal(), and copy() - -#include "heap-profile-table.h" - -#include "base/logging.h" -#include "raw_printer.h" -#include "symbolize.h" -#include -#include -#include "memory_region_map.h" -#include "base/commandlineflags.h" -#include "base/logging.h" // for the RawFD I/O commands -#include "base/sysinfo.h" - -using std::sort; -using std::equal; -using std::copy; -using std::string; -using std::map; - -using tcmalloc::FillProcSelfMaps; // from sysinfo.h -using tcmalloc::DumpProcSelfMaps; // from sysinfo.h - -//---------------------------------------------------------------------- - -DEFINE_bool(cleanup_old_heap_profiles, - EnvToBool("HEAP_PROFILE_CLEANUP", true), - "At initialization time, delete old heap profiles."); - -DEFINE_int32(heap_check_max_leaks, - EnvToInt("HEAP_CHECK_MAX_LEAKS", 20), - "The maximum number of leak reports to print."); - -//---------------------------------------------------------------------- - -// header of the dumped heap profile -static const char kProfileHeader[] = "heap profile: "; -static const char kProcSelfMapsHeader[] = "\nMAPPED_LIBRARIES:\n"; - -//---------------------------------------------------------------------- - -const char HeapProfileTable::kFileExt[] = ".heap"; - -//---------------------------------------------------------------------- - -static const int kHashTableSize = 179999; // Size for bucket_table_. -/*static*/ const int HeapProfileTable::kMaxStackDepth; - -//---------------------------------------------------------------------- - -// We strip out different number of stack frames in debug mode -// because less inlining happens in that case -#ifdef NDEBUG -static const int kStripFrames = 2; -#else -static const int kStripFrames = 3; -#endif - -// For sorting Stats or Buckets by in-use space -static bool ByAllocatedSpace(HeapProfileTable::Stats* a, - HeapProfileTable::Stats* b) { - // Return true iff "a" has more allocated space than "b" - return (a->alloc_size - a->free_size) > (b->alloc_size - b->free_size); -} - -//---------------------------------------------------------------------- - -HeapProfileTable::HeapProfileTable(Allocator alloc, - DeAllocator dealloc, - bool profile_mmap) - : alloc_(alloc), - dealloc_(dealloc), - profile_mmap_(profile_mmap), - bucket_table_(NULL), - num_buckets_(0), - address_map_(NULL) { - // Make a hash table for buckets. - const int table_bytes = kHashTableSize * sizeof(*bucket_table_); - bucket_table_ = static_cast(alloc_(table_bytes)); - memset(bucket_table_, 0, table_bytes); - - // Make an allocation map. - address_map_ = - new(alloc_(sizeof(AllocationMap))) AllocationMap(alloc_, dealloc_); - - // Initialize. - memset(&total_, 0, sizeof(total_)); - num_buckets_ = 0; -} - -HeapProfileTable::~HeapProfileTable() { - // Free the allocation map. - address_map_->~AllocationMap(); - dealloc_(address_map_); - address_map_ = NULL; - - // Free the hash table. - for (int i = 0; i < kHashTableSize; i++) { - for (Bucket* curr = bucket_table_[i]; curr != 0; /**/) { - Bucket* bucket = curr; - curr = curr->next; - dealloc_(bucket->stack); - dealloc_(bucket); - } - } - dealloc_(bucket_table_); - bucket_table_ = NULL; -} - -HeapProfileTable::Bucket* HeapProfileTable::GetBucket(int depth, - const void* const key[]) { - // Make hash-value - uintptr_t h = 0; - for (int i = 0; i < depth; i++) { - h += reinterpret_cast(key[i]); - h += h << 10; - h ^= h >> 6; - } - h += h << 3; - h ^= h >> 11; - - // Lookup stack trace in table - unsigned int buck = ((unsigned int) h) % kHashTableSize; - for (Bucket* b = bucket_table_[buck]; b != 0; b = b->next) { - if ((b->hash == h) && - (b->depth == depth) && - equal(key, key + depth, b->stack)) { - return b; - } - } - - // Create new bucket - const size_t key_size = sizeof(key[0]) * depth; - const void** kcopy = reinterpret_cast(alloc_(key_size)); - copy(key, key + depth, kcopy); - Bucket* b = reinterpret_cast(alloc_(sizeof(Bucket))); - memset(b, 0, sizeof(*b)); - b->hash = h; - b->depth = depth; - b->stack = kcopy; - b->next = bucket_table_[buck]; - bucket_table_[buck] = b; - num_buckets_++; - return b; -} - -int HeapProfileTable::GetCallerStackTrace( - int skip_count, void* stack[kMaxStackDepth]) { - return MallocHook::GetCallerStackTrace( - stack, kMaxStackDepth, kStripFrames + skip_count + 1); -} - -void HeapProfileTable::RecordAlloc( - const void* ptr, size_t bytes, int stack_depth, - const void* const call_stack[]) { - Bucket* b = GetBucket(stack_depth, call_stack); - b->allocs++; - b->alloc_size += bytes; - total_.allocs++; - total_.alloc_size += bytes; - - AllocValue v; - v.set_bucket(b); // also did set_live(false); set_ignore(false) - v.bytes = bytes; - address_map_->Insert(ptr, v); -} - -void HeapProfileTable::RecordFree(const void* ptr) { - AllocValue v; - if (address_map_->FindAndRemove(ptr, &v)) { - Bucket* b = v.bucket(); - b->frees++; - b->free_size += v.bytes; - total_.frees++; - total_.free_size += v.bytes; - } -} - -bool HeapProfileTable::FindAlloc(const void* ptr, size_t* object_size) const { - const AllocValue* alloc_value = address_map_->Find(ptr); - if (alloc_value != NULL) *object_size = alloc_value->bytes; - return alloc_value != NULL; -} - -bool HeapProfileTable::FindAllocDetails(const void* ptr, - AllocInfo* info) const { - const AllocValue* alloc_value = address_map_->Find(ptr); - if (alloc_value != NULL) { - info->object_size = alloc_value->bytes; - info->call_stack = alloc_value->bucket()->stack; - info->stack_depth = alloc_value->bucket()->depth; - } - return alloc_value != NULL; -} - -bool HeapProfileTable::FindInsideAlloc(const void* ptr, - size_t max_size, - const void** object_ptr, - size_t* object_size) const { - const AllocValue* alloc_value = - address_map_->FindInside(&AllocValueSize, max_size, ptr, object_ptr); - if (alloc_value != NULL) *object_size = alloc_value->bytes; - return alloc_value != NULL; -} - -bool HeapProfileTable::MarkAsLive(const void* ptr) { - AllocValue* alloc = address_map_->FindMutable(ptr); - if (alloc && !alloc->live()) { - alloc->set_live(true); - return true; - } - return false; -} - -void HeapProfileTable::MarkAsIgnored(const void* ptr) { - AllocValue* alloc = address_map_->FindMutable(ptr); - if (alloc) { - alloc->set_ignore(true); - } -} - -// We'd be happier using snprintfer, but we don't to reduce dependencies. -int HeapProfileTable::UnparseBucket(const Bucket& b, - char* buf, int buflen, int bufsize, - const char* extra, - Stats* profile_stats) { - if (profile_stats != NULL) { - profile_stats->allocs += b.allocs; - profile_stats->alloc_size += b.alloc_size; - profile_stats->frees += b.frees; - profile_stats->free_size += b.free_size; - } - int printed = - snprintf(buf + buflen, bufsize - buflen, "%6d: %8" PRId64 " [%6d: %8" PRId64 "] @%s", - b.allocs - b.frees, - b.alloc_size - b.free_size, - b.allocs, - b.alloc_size, - extra); - // If it looks like the snprintf failed, ignore the fact we printed anything - if (printed < 0 || printed >= bufsize - buflen) return buflen; - buflen += printed; - for (int d = 0; d < b.depth; d++) { - printed = snprintf(buf + buflen, bufsize - buflen, " 0x%08" PRIxPTR, - reinterpret_cast(b.stack[d])); - if (printed < 0 || printed >= bufsize - buflen) return buflen; - buflen += printed; - } - printed = snprintf(buf + buflen, bufsize - buflen, "\n"); - if (printed < 0 || printed >= bufsize - buflen) return buflen; - buflen += printed; - return buflen; -} - -HeapProfileTable::Bucket** -HeapProfileTable::MakeSortedBucketList() const { - Bucket** list = static_cast(alloc_(sizeof(Bucket) * num_buckets_)); - - int bucket_count = 0; - for (int i = 0; i < kHashTableSize; i++) { - for (Bucket* curr = bucket_table_[i]; curr != 0; curr = curr->next) { - list[bucket_count++] = curr; - } - } - RAW_DCHECK(bucket_count == num_buckets_, ""); - - sort(list, list + num_buckets_, ByAllocatedSpace); - - return list; -} - -void HeapProfileTable::IterateOrderedAllocContexts( - AllocContextIterator callback) const { - Bucket** list = MakeSortedBucketList(); - AllocContextInfo info; - for (int i = 0; i < num_buckets_; ++i) { - *static_cast(&info) = *static_cast(list[i]); - info.stack_depth = list[i]->depth; - info.call_stack = list[i]->stack; - callback(info); - } - dealloc_(list); -} - -int HeapProfileTable::FillOrderedProfile(char buf[], int size) const { - Bucket** list = MakeSortedBucketList(); - - // Our file format is "bucket, bucket, ..., bucket, proc_self_maps_info". - // In the cases buf is too small, we'd rather leave out the last - // buckets than leave out the /proc/self/maps info. To ensure that, - // we actually print the /proc/self/maps info first, then move it to - // the end of the buffer, then write the bucket info into whatever - // is remaining, and then move the maps info one last time to close - // any gaps. Whew! - int map_length = snprintf(buf, size, "%s", kProcSelfMapsHeader); - if (map_length < 0 || map_length >= size) { - dealloc_(list); - return 0; - } - bool dummy; // "wrote_all" -- did /proc/self/maps fit in its entirety? - map_length += FillProcSelfMaps(buf + map_length, size - map_length, &dummy); - RAW_DCHECK(map_length <= size, ""); - char* const map_start = buf + size - map_length; // move to end - memmove(map_start, buf, map_length); - size -= map_length; - - Stats stats; - memset(&stats, 0, sizeof(stats)); - int bucket_length = snprintf(buf, size, "%s", kProfileHeader); - if (bucket_length < 0 || bucket_length >= size) { - dealloc_(list); - return 0; - } - bucket_length = UnparseBucket(total_, buf, bucket_length, size, - " heapprofile", &stats); - - // Dump the mmap list first. - if (profile_mmap_) { - BufferArgs buffer(buf, bucket_length, size); - MemoryRegionMap::IterateBuckets(DumpBucketIterator, &buffer); - bucket_length = buffer.buflen; - } - - for (int i = 0; i < num_buckets_; i++) { - bucket_length = UnparseBucket(*list[i], buf, bucket_length, size, "", - &stats); - } - RAW_DCHECK(bucket_length < size, ""); - - dealloc_(list); - - RAW_DCHECK(buf + bucket_length <= map_start, ""); - memmove(buf + bucket_length, map_start, map_length); // close the gap - - return bucket_length + map_length; -} - -// static -void HeapProfileTable::DumpBucketIterator(const Bucket* bucket, - BufferArgs* args) { - args->buflen = UnparseBucket(*bucket, args->buf, args->buflen, args->bufsize, - "", NULL); -} - -inline -void HeapProfileTable::DumpNonLiveIterator(const void* ptr, AllocValue* v, - const DumpArgs& args) { - if (v->live()) { - v->set_live(false); - return; - } - if (v->ignore()) { - return; - } - Bucket b; - memset(&b, 0, sizeof(b)); - b.allocs = 1; - b.alloc_size = v->bytes; - b.depth = v->bucket()->depth; - b.stack = v->bucket()->stack; - char buf[1024]; - int len = UnparseBucket(b, buf, 0, sizeof(buf), "", args.profile_stats); - RawWrite(args.fd, buf, len); -} - -// Callback from NonLiveSnapshot; adds entry to arg->dest -// if not the entry is not live and is not present in arg->base. -void HeapProfileTable::AddIfNonLive(const void* ptr, AllocValue* v, - AddNonLiveArgs* arg) { - if (v->live()) { - v->set_live(false); - } else { - if (arg->base != NULL && arg->base->map_.Find(ptr) != NULL) { - // Present in arg->base, so do not save - } else { - arg->dest->Add(ptr, *v); - } - } -} - -bool HeapProfileTable::WriteProfile(const char* file_name, - const Bucket& total, - AllocationMap* allocations) { - RAW_VLOG(1, "Dumping non-live heap profile to %s", file_name); - RawFD fd = RawOpenForWriting(file_name); - if (fd != kIllegalRawFD) { - RawWrite(fd, kProfileHeader, strlen(kProfileHeader)); - char buf[512]; - int len = UnparseBucket(total, buf, 0, sizeof(buf), " heapprofile", - NULL); - RawWrite(fd, buf, len); - const DumpArgs args(fd, NULL); - allocations->Iterate(DumpNonLiveIterator, args); - RawWrite(fd, kProcSelfMapsHeader, strlen(kProcSelfMapsHeader)); - DumpProcSelfMaps(fd); - RawClose(fd); - return true; - } else { - RAW_LOG(ERROR, "Failed dumping filtered heap profile to %s", file_name); - return false; - } -} - -void HeapProfileTable::CleanupOldProfiles(const char* prefix) { - if (!FLAGS_cleanup_old_heap_profiles) - return; - string pattern = string(prefix) + ".*" + kFileExt; -#if defined(HAVE_GLOB_H) - glob_t g; - const int r = glob(pattern.c_str(), GLOB_ERR, NULL, &g); - if (r == 0 || r == GLOB_NOMATCH) { - const int prefix_length = strlen(prefix); - for (int i = 0; i < g.gl_pathc; i++) { - const char* fname = g.gl_pathv[i]; - if ((strlen(fname) >= prefix_length) && - (memcmp(fname, prefix, prefix_length) == 0)) { - RAW_VLOG(1, "Removing old heap profile %s", fname); - unlink(fname); - } - } - } - globfree(&g); -#else /* HAVE_GLOB_H */ - RAW_LOG(WARNING, "Unable to remove old heap profiles (can't run glob())"); -#endif -} - -HeapProfileTable::Snapshot* HeapProfileTable::TakeSnapshot() { - Snapshot* s = new (alloc_(sizeof(Snapshot))) Snapshot(alloc_, dealloc_); - address_map_->Iterate(AddToSnapshot, s); - return s; -} - -void HeapProfileTable::ReleaseSnapshot(Snapshot* s) { - s->~Snapshot(); - dealloc_(s); -} - -// Callback from TakeSnapshot; adds a single entry to snapshot -void HeapProfileTable::AddToSnapshot(const void* ptr, AllocValue* v, - Snapshot* snapshot) { - snapshot->Add(ptr, *v); -} - -HeapProfileTable::Snapshot* HeapProfileTable::NonLiveSnapshot( - Snapshot* base) { - RAW_VLOG(2, "NonLiveSnapshot input: %d %d\n", - int(total_.allocs - total_.frees), - int(total_.alloc_size - total_.free_size)); - - Snapshot* s = new (alloc_(sizeof(Snapshot))) Snapshot(alloc_, dealloc_); - AddNonLiveArgs args; - args.dest = s; - args.base = base; - address_map_->Iterate(AddIfNonLive, &args); - RAW_VLOG(2, "NonLiveSnapshot output: %d %d\n", - int(s->total_.allocs - s->total_.frees), - int(s->total_.alloc_size - s->total_.free_size)); - return s; -} - -// Information kept per unique bucket seen -struct HeapProfileTable::Snapshot::Entry { - int count; - int bytes; - Bucket* bucket; - Entry() : count(0), bytes(0) { } - - // Order by decreasing bytes - bool operator<(const Entry& x) const { - return this->bytes > x.bytes; - } -}; - -// State used to generate leak report. We keep a mapping from Bucket pointer -// the collected stats for that bucket. -struct HeapProfileTable::Snapshot::ReportState { - map buckets_; -}; - -// Callback from ReportLeaks; updates ReportState. -void HeapProfileTable::Snapshot::ReportCallback(const void* ptr, - AllocValue* v, - ReportState* state) { - Entry* e = &state->buckets_[v->bucket()]; // Creates empty Entry first time - e->bucket = v->bucket(); - e->count++; - e->bytes += v->bytes; -} - -void HeapProfileTable::Snapshot::ReportLeaks(const char* checker_name, - const char* filename, - bool should_symbolize) { - // This is only used by the heap leak checker, but is intimately - // tied to the allocation map that belongs in this module and is - // therefore placed here. - RAW_LOG(ERROR, "Leak check %s detected leaks of %" PRIuS " bytes " - "in %" PRIuS " objects", - checker_name, - size_t(total_.alloc_size), - size_t(total_.allocs)); - - // Group objects by Bucket - ReportState state; - map_.Iterate(&ReportCallback, &state); - - // Sort buckets by decreasing leaked size - const int n = state.buckets_.size(); - Entry* entries = new Entry[n]; - int dst = 0; - for (map::const_iterator iter = state.buckets_.begin(); - iter != state.buckets_.end(); - ++iter) { - entries[dst++] = iter->second; - } - sort(entries, entries + n); - - // Report a bounded number of leaks to keep the leak report from - // growing too long. - const int to_report = - (FLAGS_heap_check_max_leaks > 0 && - n > FLAGS_heap_check_max_leaks) ? FLAGS_heap_check_max_leaks : n; - RAW_LOG(ERROR, "The %d largest leaks:", to_report); - - // Print - SymbolTable symbolization_table; - for (int i = 0; i < to_report; i++) { - const Entry& e = entries[i]; - for (int j = 0; j < e.bucket->depth; j++) { - symbolization_table.Add(e.bucket->stack[j]); - } - } - static const int kBufSize = 2<<10; - char buffer[kBufSize]; - if (should_symbolize) - symbolization_table.Symbolize(); - for (int i = 0; i < to_report; i++) { - const Entry& e = entries[i]; - base::RawPrinter printer(buffer, kBufSize); - printer.Printf("Leak of %d bytes in %d objects allocated from:\n", - e.bytes, e.count); - for (int j = 0; j < e.bucket->depth; j++) { - const void* pc = e.bucket->stack[j]; - printer.Printf("\t@ %" PRIxPTR " %s\n", - reinterpret_cast(pc), symbolization_table.GetSymbol(pc)); - } - RAW_LOG(ERROR, "%s", buffer); - } - - if (to_report < n) { - RAW_LOG(ERROR, "Skipping leaks numbered %d..%d", - to_report, n-1); - } - delete[] entries; - - // TODO: Dump the sorted Entry list instead of dumping raw data? - // (should be much shorter) - if (!HeapProfileTable::WriteProfile(filename, total_, &map_)) { - RAW_LOG(ERROR, "Could not write pprof profile to %s", filename); - } -} - -void HeapProfileTable::Snapshot::ReportObject(const void* ptr, - AllocValue* v, - char* unused) { - // Perhaps also log the allocation stack trace (unsymbolized) - // on this line in case somebody finds it useful. - RAW_LOG(ERROR, "leaked %" PRIuS " byte object %p", v->bytes, ptr); -} - -void HeapProfileTable::Snapshot::ReportIndividualObjects() { - char unused; - map_.Iterate(ReportObject, &unused); -} diff --git a/contrib/libtcmalloc/src/heap-profile-table.h b/contrib/libtcmalloc/src/heap-profile-table.h deleted file mode 100644 index 3c6284741af..00000000000 --- a/contrib/libtcmalloc/src/heap-profile-table.h +++ /dev/null @@ -1,399 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2006, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat -// Maxim Lifantsev (refactoring) -// - -#ifndef BASE_HEAP_PROFILE_TABLE_H_ -#define BASE_HEAP_PROFILE_TABLE_H_ - -#include "addressmap-inl.h" -#include "base/basictypes.h" -#include "base/logging.h" // for RawFD -#include "heap-profile-stats.h" - -// Table to maintain a heap profile data inside, -// i.e. the set of currently active heap memory allocations. -// thread-unsafe and non-reentrant code: -// each instance object must be used by one thread -// at a time w/o self-recursion. -// -// TODO(maxim): add a unittest for this class. -class HeapProfileTable { - public: - - // Extension to be used for heap pforile files. - static const char kFileExt[]; - - // Longest stack trace we record. - static const int kMaxStackDepth = 32; - - // data types ---------------------------- - - // Profile stats. - typedef HeapProfileStats Stats; - - // Info we can return about an allocation. - struct AllocInfo { - size_t object_size; // size of the allocation - const void* const* call_stack; // call stack that made the allocation call - int stack_depth; // depth of call_stack - bool live; - bool ignored; - }; - - // Info we return about an allocation context. - // An allocation context is a unique caller stack trace - // of an allocation operation. - struct AllocContextInfo : public Stats { - int stack_depth; // Depth of stack trace - const void* const* call_stack; // Stack trace - }; - - // Memory (de)allocator interface we'll use. - typedef void* (*Allocator)(size_t size); - typedef void (*DeAllocator)(void* ptr); - - // interface --------------------------- - - HeapProfileTable(Allocator alloc, DeAllocator dealloc, bool profile_mmap); - ~HeapProfileTable(); - - // Collect the stack trace for the function that asked to do the - // allocation for passing to RecordAlloc() below. - // - // The stack trace is stored in 'stack'. The stack depth is returned. - // - // 'skip_count' gives the number of stack frames between this call - // and the memory allocation function. - static int GetCallerStackTrace(int skip_count, void* stack[kMaxStackDepth]); - - // Record an allocation at 'ptr' of 'bytes' bytes. 'stack_depth' - // and 'call_stack' identifying the function that requested the - // allocation. They can be generated using GetCallerStackTrace() above. - void RecordAlloc(const void* ptr, size_t bytes, - int stack_depth, const void* const call_stack[]); - - // Record the deallocation of memory at 'ptr'. - void RecordFree(const void* ptr); - - // Return true iff we have recorded an allocation at 'ptr'. - // If yes, fill *object_size with the allocation byte size. - bool FindAlloc(const void* ptr, size_t* object_size) const; - // Same as FindAlloc, but fills all of *info. - bool FindAllocDetails(const void* ptr, AllocInfo* info) const; - - // Return true iff "ptr" points into a recorded allocation - // If yes, fill *object_ptr with the actual allocation address - // and *object_size with the allocation byte size. - // max_size specifies largest currently possible allocation size. - bool FindInsideAlloc(const void* ptr, size_t max_size, - const void** object_ptr, size_t* object_size) const; - - // If "ptr" points to a recorded allocation and it's not marked as live - // mark it as live and return true. Else return false. - // All allocations start as non-live. - bool MarkAsLive(const void* ptr); - - // If "ptr" points to a recorded allocation, mark it as "ignored". - // Ignored objects are treated like other objects, except that they - // are skipped in heap checking reports. - void MarkAsIgnored(const void* ptr); - - // Return current total (de)allocation statistics. It doesn't contain - // mmap'ed regions. - const Stats& total() const { return total_; } - - // Allocation data iteration callback: gets passed object pointer and - // fully-filled AllocInfo. - typedef void (*AllocIterator)(const void* ptr, const AllocInfo& info); - - // Iterate over the allocation profile data calling "callback" - // for every allocation. - void IterateAllocs(AllocIterator callback) const { - address_map_->Iterate(MapArgsAllocIterator, callback); - } - - // Allocation context profile data iteration callback - typedef void (*AllocContextIterator)(const AllocContextInfo& info); - - // Iterate over the allocation context profile data calling "callback" - // for every allocation context. Allocation contexts are ordered by the - // size of allocated space. - void IterateOrderedAllocContexts(AllocContextIterator callback) const; - - // Fill profile data into buffer 'buf' of size 'size' - // and return the actual size occupied by the dump in 'buf'. - // The profile buckets are dumped in the decreasing order - // of currently allocated bytes. - // We do not provision for 0-terminating 'buf'. - int FillOrderedProfile(char buf[], int size) const; - - // Cleanup any old profile files matching prefix + ".*" + kFileExt. - static void CleanupOldProfiles(const char* prefix); - - // Return a snapshot of the current contents of *this. - // Caller must call ReleaseSnapshot() on result when no longer needed. - // The result is only valid while this exists and until - // the snapshot is discarded by calling ReleaseSnapshot(). - class Snapshot; - Snapshot* TakeSnapshot(); - - // Release a previously taken snapshot. snapshot must not - // be used after this call. - void ReleaseSnapshot(Snapshot* snapshot); - - // Return a snapshot of every non-live, non-ignored object in *this. - // If "base" is non-NULL, skip any objects present in "base". - // As a side-effect, clears the "live" bit on every live object in *this. - // Caller must call ReleaseSnapshot() on result when no longer needed. - Snapshot* NonLiveSnapshot(Snapshot* base); - - private: - - // data types ---------------------------- - - // Hash table bucket to hold (de)allocation stats - // for a given allocation call stack trace. - typedef HeapProfileBucket Bucket; - - // Info stored in the address map - struct AllocValue { - // Access to the stack-trace bucket - Bucket* bucket() const { - return reinterpret_cast(bucket_rep & ~uintptr_t(kMask)); - } - // This also does set_live(false). - void set_bucket(Bucket* b) { bucket_rep = reinterpret_cast(b); } - size_t bytes; // Number of bytes in this allocation - - // Access to the allocation liveness flag (for leak checking) - bool live() const { return bucket_rep & kLive; } - void set_live(bool l) { - bucket_rep = (bucket_rep & ~uintptr_t(kLive)) | (l ? kLive : 0); - } - - // Should this allocation be ignored if it looks like a leak? - bool ignore() const { return bucket_rep & kIgnore; } - void set_ignore(bool r) { - bucket_rep = (bucket_rep & ~uintptr_t(kIgnore)) | (r ? kIgnore : 0); - } - - private: - // We store a few bits in the bottom bits of bucket_rep. - // (Alignment is at least four, so we have at least two bits.) - static const int kLive = 1; - static const int kIgnore = 2; - static const int kMask = kLive | kIgnore; - - uintptr_t bucket_rep; - }; - - // helper for FindInsideAlloc - static size_t AllocValueSize(const AllocValue& v) { return v.bytes; } - - typedef AddressMap AllocationMap; - - // Arguments that need to be passed DumpBucketIterator callback below. - struct BufferArgs { - BufferArgs(char* buf_arg, int buflen_arg, int bufsize_arg) - : buf(buf_arg), - buflen(buflen_arg), - bufsize(bufsize_arg) { - } - - char* buf; - int buflen; - int bufsize; - - DISALLOW_COPY_AND_ASSIGN(BufferArgs); - }; - - // Arguments that need to be passed DumpNonLiveIterator callback below. - struct DumpArgs { - DumpArgs(RawFD fd_arg, Stats* profile_stats_arg) - : fd(fd_arg), - profile_stats(profile_stats_arg) { - } - - RawFD fd; // file to write to - Stats* profile_stats; // stats to update (may be NULL) - }; - - // helpers ---------------------------- - - // Unparse bucket b and print its portion of profile dump into buf. - // We return the amount of space in buf that we use. We start printing - // at buf + buflen, and promise not to go beyond buf + bufsize. - // We do not provision for 0-terminating 'buf'. - // - // If profile_stats is non-NULL, we update *profile_stats by - // counting bucket b. - // - // "extra" is appended to the unparsed bucket. Typically it is empty, - // but may be set to something like " heapprofile" for the total - // bucket to indicate the type of the profile. - static int UnparseBucket(const Bucket& b, - char* buf, int buflen, int bufsize, - const char* extra, - Stats* profile_stats); - - // Get the bucket for the caller stack trace 'key' of depth 'depth' - // creating the bucket if needed. - Bucket* GetBucket(int depth, const void* const key[]); - - // Helper for IterateAllocs to do callback signature conversion - // from AllocationMap::Iterate to AllocIterator. - static void MapArgsAllocIterator(const void* ptr, AllocValue* v, - AllocIterator callback) { - AllocInfo info; - info.object_size = v->bytes; - info.call_stack = v->bucket()->stack; - info.stack_depth = v->bucket()->depth; - info.live = v->live(); - info.ignored = v->ignore(); - callback(ptr, info); - } - - // Helper to dump a bucket. - inline static void DumpBucketIterator(const Bucket* bucket, - BufferArgs* args); - - // Helper for DumpNonLiveProfile to do object-granularity - // heap profile dumping. It gets passed to AllocationMap::Iterate. - inline static void DumpNonLiveIterator(const void* ptr, AllocValue* v, - const DumpArgs& args); - - // Helper for IterateOrderedAllocContexts and FillOrderedProfile. - // Creates a sorted list of Buckets whose length is num_buckets_. - // The caller is responsible for deallocating the returned list. - Bucket** MakeSortedBucketList() const; - - // Helper for TakeSnapshot. Saves object to snapshot. - static void AddToSnapshot(const void* ptr, AllocValue* v, Snapshot* s); - - // Arguments passed to AddIfNonLive - struct AddNonLiveArgs { - Snapshot* dest; - Snapshot* base; - }; - - // Helper for NonLiveSnapshot. Adds the object to the destination - // snapshot if it is non-live. - static void AddIfNonLive(const void* ptr, AllocValue* v, - AddNonLiveArgs* arg); - - // Write contents of "*allocations" as a heap profile to - // "file_name". "total" must contain the total of all entries in - // "*allocations". - static bool WriteProfile(const char* file_name, - const Bucket& total, - AllocationMap* allocations); - - // data ---------------------------- - - // Memory (de)allocator that we use. - Allocator alloc_; - DeAllocator dealloc_; - - // Overall profile stats; we use only the Stats part, - // but make it a Bucket to pass to UnparseBucket. - Bucket total_; - - bool profile_mmap_; - - // Bucket hash table for malloc. - // We hand-craft one instead of using one of the pre-written - // ones because we do not want to use malloc when operating on the table. - // It is only few lines of code, so no big deal. - Bucket** bucket_table_; - int num_buckets_; - - // Map of all currently allocated objects and mapped regions we know about. - AllocationMap* address_map_; - - DISALLOW_COPY_AND_ASSIGN(HeapProfileTable); -}; - -class HeapProfileTable::Snapshot { - public: - const Stats& total() const { return total_; } - - // Report anything in this snapshot as a leak. - // May use new/delete for temporary storage. - // If should_symbolize is true, will fork (which is not threadsafe) - // to turn addresses into symbol names. Set to false for maximum safety. - // Also writes a heap profile to "filename" that contains - // all of the objects in this snapshot. - void ReportLeaks(const char* checker_name, const char* filename, - bool should_symbolize); - - // Report the addresses of all leaked objects. - // May use new/delete for temporary storage. - void ReportIndividualObjects(); - - bool Empty() const { - return (total_.allocs == 0) && (total_.alloc_size == 0); - } - - private: - friend class HeapProfileTable; - - // Total count/size are stored in a Bucket so we can reuse UnparseBucket - Bucket total_; - - // We share the Buckets managed by the parent table, but have our - // own object->bucket map. - AllocationMap map_; - - Snapshot(Allocator alloc, DeAllocator dealloc) : map_(alloc, dealloc) { - memset(&total_, 0, sizeof(total_)); - } - - // Callback used to populate a Snapshot object with entries found - // in another allocation map. - inline void Add(const void* ptr, const AllocValue& v) { - map_.Insert(ptr, v); - total_.allocs++; - total_.alloc_size += v.bytes; - } - - // Helpers for sorting and generating leak reports - struct Entry; - struct ReportState; - static void ReportCallback(const void* ptr, AllocValue* v, ReportState*); - static void ReportObject(const void* ptr, AllocValue* v, char*); - - DISALLOW_COPY_AND_ASSIGN(Snapshot); -}; - -#endif // BASE_HEAP_PROFILE_TABLE_H_ diff --git a/contrib/libtcmalloc/src/internal_logging.cc b/contrib/libtcmalloc/src/internal_logging.cc deleted file mode 100644 index 3b2d0cb80f9..00000000000 --- a/contrib/libtcmalloc/src/internal_logging.cc +++ /dev/null @@ -1,192 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Sanjay Ghemawat - -#include "config.h" -#include "internal_logging.h" -#include // for va_end, va_start -#include // for vsnprintf, va_list, etc -#include // for abort -#include // for strlen, memcpy -#ifdef HAVE_UNISTD_H -#include // for write() -#endif - -#include -#include "base/logging.h" // for perftools_vsnprintf -#include "base/spinlock.h" // for SpinLockHolder, SpinLock - -// Variables for storing crash output. Allocated statically since we -// may not be able to heap-allocate while crashing. -static SpinLock crash_lock(base::LINKER_INITIALIZED); -static bool crashed = false; -static const int kStatsBufferSize = 16 << 10; -static char stats_buffer[kStatsBufferSize] = { 0 }; - -namespace tcmalloc { - -static void WriteMessage(const char* msg, int length) { - write(STDERR_FILENO, msg, length); -} - -void (*log_message_writer)(const char* msg, int length) = WriteMessage; - - -class Logger { - public: - bool Add(const LogItem& item); - bool AddStr(const char* str, int n); - bool AddNum(uint64_t num, int base); // base must be 10 or 16. - - static const int kBufSize = 200; - char* p_; - char* end_; - char buf_[kBufSize]; -}; - -void Log(LogMode mode, const char* filename, int line, - LogItem a, LogItem b, LogItem c, LogItem d) { - Logger state; - state.p_ = state.buf_; - state.end_ = state.buf_ + sizeof(state.buf_); - state.AddStr(filename, strlen(filename)) - && state.AddStr(":", 1) - && state.AddNum(line, 10) - && state.AddStr("]", 1) - && state.Add(a) - && state.Add(b) - && state.Add(c) - && state.Add(d); - - // Teminate with newline - if (state.p_ >= state.end_) { - state.p_ = state.end_ - 1; - } - *state.p_ = '\n'; - state.p_++; - - int msglen = state.p_ - state.buf_; - if (mode == kLog) { - (*log_message_writer)(state.buf_, msglen); - return; - } - - bool first_crash = false; - { - SpinLockHolder l(&crash_lock); - if (!crashed) { - crashed = true; - first_crash = true; - } - } - - (*log_message_writer)(state.buf_, msglen); - if (first_crash && mode == kCrashWithStats) { - MallocExtension::instance()->GetStats(stats_buffer, kStatsBufferSize); - (*log_message_writer)(stats_buffer, strlen(stats_buffer)); - } - - abort(); -} - -bool Logger::Add(const LogItem& item) { - // Separate items with spaces - if (p_ < end_) { - *p_ = ' '; - p_++; - } - - switch (item.tag_) { - case LogItem::kStr: - return AddStr(item.u_.str, strlen(item.u_.str)); - case LogItem::kUnsigned: - return AddNum(item.u_.unum, 10); - case LogItem::kSigned: - if (item.u_.snum < 0) { - // The cast to uint64_t is intentionally before the negation - // so that we do not attempt to negate -2^63. - return AddStr("-", 1) - && AddNum(- static_cast(item.u_.snum), 10); - } else { - return AddNum(static_cast(item.u_.snum), 10); - } - case LogItem::kPtr: - return AddStr("0x", 2) - && AddNum(reinterpret_cast(item.u_.ptr), 16); - default: - return false; - } -} - -bool Logger::AddStr(const char* str, int n) { - if (end_ - p_ < n) { - return false; - } else { - memcpy(p_, str, n); - p_ += n; - return true; - } -} - -bool Logger::AddNum(uint64_t num, int base) { - static const char kDigits[] = "0123456789abcdef"; - char space[22]; // more than enough for 2^64 in smallest supported base (10) - char* end = space + sizeof(space); - char* pos = end; - do { - pos--; - *pos = kDigits[num % base]; - num /= base; - } while (num > 0 && pos > space); - return AddStr(pos, end - pos); -} - -} // end tcmalloc namespace - -void TCMalloc_Printer::printf(const char* format, ...) { - if (left_ > 0) { - va_list ap; - va_start(ap, format); - const int r = perftools_vsnprintf(buf_, left_, format, ap); - va_end(ap); - if (r < 0) { - // Perhaps an old glibc that returns -1 on truncation? - left_ = 0; - } else if (r > left_) { - // Truncation - left_ = 0; - } else { - left_ -= r; - buf_ += r; - } - } -} diff --git a/contrib/libtcmalloc/src/internal_logging.h b/contrib/libtcmalloc/src/internal_logging.h deleted file mode 100644 index c6363894911..00000000000 --- a/contrib/libtcmalloc/src/internal_logging.h +++ /dev/null @@ -1,144 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat -// -// Internal logging and related utility routines. - -#ifndef TCMALLOC_INTERNAL_LOGGING_H_ -#define TCMALLOC_INTERNAL_LOGGING_H_ - -#include "config.h" -#include // for size_t -#if defined HAVE_STDINT_H -#include -#elif defined HAVE_INTTYPES_H -#include -#else -#include -#endif - -//------------------------------------------------------------------- -// Utility routines -//------------------------------------------------------------------- - -// Safe logging helper: we write directly to the stderr file -// descriptor and avoid FILE buffering because that may invoke -// malloc(). -// -// Example: -// Log(kLog, __FILE__, __LINE__, "error", bytes); - -namespace tcmalloc { -enum LogMode { - kLog, // Just print the message - kCrash, // Print the message and crash - kCrashWithStats // Print the message, some stats, and crash -}; - -class Logger; - -// A LogItem holds any of the argument types that can be passed to Log() -class LogItem { - public: - LogItem() : tag_(kEnd) { } - LogItem(const char* v) : tag_(kStr) { u_.str = v; } - LogItem(int v) : tag_(kSigned) { u_.snum = v; } - LogItem(long v) : tag_(kSigned) { u_.snum = v; } - LogItem(long long v) : tag_(kSigned) { u_.snum = v; } - LogItem(unsigned int v) : tag_(kUnsigned) { u_.unum = v; } - LogItem(unsigned long v) : tag_(kUnsigned) { u_.unum = v; } - LogItem(unsigned long long v) : tag_(kUnsigned) { u_.unum = v; } - LogItem(const void* v) : tag_(kPtr) { u_.ptr = v; } - private: - friend class Logger; - enum Tag { - kStr, - kSigned, - kUnsigned, - kPtr, - kEnd - }; - Tag tag_; - union { - const char* str; - const void* ptr; - int64_t snum; - uint64_t unum; - } u_; -}; - -extern PERFTOOLS_DLL_DECL void Log(LogMode mode, const char* filename, int line, - LogItem a, LogItem b = LogItem(), - LogItem c = LogItem(), LogItem d = LogItem()); - -// Tests can override this function to collect logging messages. -extern PERFTOOLS_DLL_DECL void (*log_message_writer)(const char* msg, int length); - -} // end tcmalloc namespace - -// Like assert(), but executed even in NDEBUG mode -#undef CHECK_CONDITION -#define CHECK_CONDITION(cond) \ -do { \ - if (!(cond)) { \ - ::tcmalloc::Log(::tcmalloc::kCrash, __FILE__, __LINE__, #cond); \ - } \ -} while (0) - -// Our own version of assert() so we can avoid hanging by trying to do -// all kinds of goofy printing while holding the malloc lock. -#ifndef NDEBUG -#define ASSERT(cond) CHECK_CONDITION(cond) -#else -#define ASSERT(cond) ((void) 0) -#endif - -// Print into buffer -class TCMalloc_Printer { - private: - char* buf_; // Where should we write next - int left_; // Space left in buffer (including space for \0) - - public: - // REQUIRES: "length > 0" - TCMalloc_Printer(char* buf, int length) : buf_(buf), left_(length) { - buf[0] = '\0'; - } - - void printf(const char* format, ...) -#ifdef HAVE___ATTRIBUTE__ - __attribute__ ((__format__ (__printf__, 2, 3))) -#endif -; -}; - -#endif // TCMALLOC_INTERNAL_LOGGING_H_ diff --git a/contrib/libtcmalloc/src/libc_override.h b/contrib/libtcmalloc/src/libc_override.h deleted file mode 100644 index 0dbabb2d169..00000000000 --- a/contrib/libtcmalloc/src/libc_override.h +++ /dev/null @@ -1,91 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2011, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Craig Silverstein -// -// This .h file imports the code that causes tcmalloc to override libc -// versions of malloc/free/new/delete/etc. That is, it provides the -// logic that makes it so calls to malloc(10) go through tcmalloc, -// rather than the default (libc) malloc. -// -// This file also provides a method: ReplaceSystemAlloc(), that every -// libc_override_*.h file it #includes is required to provide. This -// is called when first setting up tcmalloc -- that is, when a global -// constructor in tcmalloc.cc is executed -- to do any initialization -// work that may be required for this OS. (Note we cannot entirely -// control when tcmalloc is initialized, and the system may do some -// mallocs and frees before this routine is called.) It may be a -// noop. -// -// Every libc has its own way of doing this, and sometimes the compiler -// matters too, so we have a different file for each libc, and often -// for different compilers and OS's. - -#ifndef TCMALLOC_LIBC_OVERRIDE_INL_H_ -#define TCMALLOC_LIBC_OVERRIDE_INL_H_ - -#include "config.h" -#ifdef HAVE_FEATURES_H -#include // for __GLIBC__ -#endif -#include - -static void ReplaceSystemAlloc(); // defined in the .h files below - -// For windows, there are two ways to get tcmalloc. If we're -// patching, then src/windows/patch_function.cc will do the necessary -// overriding here. Otherwise, we doing the 'redefine' trick, where -// we remove malloc/new/etc from mscvcrt.dll, and just need to define -// them now. -#if defined(_WIN32) && defined(WIN32_DO_PATCHING) -void PatchWindowsFunctions(); // in src/windows/patch_function.cc -static void ReplaceSystemAlloc() { PatchWindowsFunctions(); } - -#elif defined(_WIN32) && !defined(WIN32_DO_PATCHING) -#include "libc_override_redefine.h" - -#elif defined(__APPLE__) -#include "libc_override_osx.h" - -#elif defined(__GLIBC__) -#include "libc_override_glibc.h" - -// Not all gcc systems necessarily support weak symbols, but all the -// ones I know of do, so for now just assume they all do. -#elif defined(__GNUC__) -#include "libc_override_gcc_and_weak.h" - -#else -#error Need to add support for your libc/OS here - -#endif - -#endif // TCMALLOC_LIBC_OVERRIDE_INL_H_ diff --git a/contrib/libtcmalloc/src/libc_override_gcc_and_weak.h b/contrib/libtcmalloc/src/libc_override_gcc_and_weak.h deleted file mode 100644 index 323d615d0ac..00000000000 --- a/contrib/libtcmalloc/src/libc_override_gcc_and_weak.h +++ /dev/null @@ -1,172 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2011, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Craig Silverstein -// -// Used to override malloc routines on systems that define the -// memory allocation routines to be weak symbols in their libc -// (almost all unix-based systems are like this), on gcc, which -// suppports the 'alias' attribute. - -#ifndef TCMALLOC_LIBC_OVERRIDE_GCC_AND_WEAK_INL_H_ -#define TCMALLOC_LIBC_OVERRIDE_GCC_AND_WEAK_INL_H_ - -#ifdef HAVE_SYS_CDEFS_H -#include // for __THROW -#endif -#include - -#include "getenv_safe.h" // TCMallocGetenvSafe -#include "base/commandlineflags.h" - -#ifndef __THROW // I guess we're not on a glibc-like system -# define __THROW // __THROW is just an optimization, so ok to make it "" -#endif - -#ifndef __GNUC__ -# error libc_override_gcc_and_weak.h is for gcc distributions only. -#endif - -#define ALIAS(tc_fn) __attribute__ ((alias (#tc_fn), used)) - -void* operator new(size_t size) - ALIAS(tc_new); -void operator delete(void* p) noexcept - ALIAS(tc_delete); -void* operator new[](size_t size) - ALIAS(tc_newarray); -void operator delete[](void* p) noexcept - ALIAS(tc_deletearray); -void* operator new(size_t size, const std::nothrow_t& nt) noexcept - ALIAS(tc_new_nothrow); -void* operator new[](size_t size, const std::nothrow_t& nt) noexcept - ALIAS(tc_newarray_nothrow); -void operator delete(void* p, const std::nothrow_t& nt) noexcept - ALIAS(tc_delete_nothrow); -void operator delete[](void* p, const std::nothrow_t& nt) noexcept - ALIAS(tc_deletearray_nothrow); - -#if defined(ENABLE_SIZED_DELETE) - -void operator delete(void *p, size_t size) throw() - ALIAS(tc_delete_sized); -void operator delete[](void *p, size_t size) throw() - ALIAS(tc_deletearray_sized); - -#elif defined(ENABLE_DYNAMIC_SIZED_DELETE) && \ - (__GNUC__ * 100 + __GNUC_MINOR__) >= 405 - -static void delegate_sized_delete(void *p, size_t s) throw() { - (operator delete)(p); -} - -static void delegate_sized_deletearray(void *p, size_t s) throw() { - (operator delete[])(p); -} - -extern "C" __attribute__((weak)) -int tcmalloc_sized_delete_enabled(void); - -static bool sized_delete_enabled(void) { - if (tcmalloc_sized_delete_enabled != 0) { - return !!tcmalloc_sized_delete_enabled(); - } - - const char *flag = TCMallocGetenvSafe("TCMALLOC_ENABLE_SIZED_DELETE"); - return tcmalloc::commandlineflags::StringToBool(flag, false); -} - -extern "C" { - -static void *resolve_delete_sized(void) { - if (sized_delete_enabled()) { - return reinterpret_cast(tc_delete_sized); - } - return reinterpret_cast(delegate_sized_delete); -} - -static void *resolve_deletearray_sized(void) { - if (sized_delete_enabled()) { - return reinterpret_cast(tc_deletearray_sized); - } - return reinterpret_cast(delegate_sized_deletearray); -} - -} - -void operator delete(void *p, size_t size) throw() - __attribute__((ifunc("resolve_delete_sized"))); -void operator delete[](void *p, size_t size) throw() - __attribute__((ifunc("resolve_deletearray_sized"))); - -#else /* !ENABLE_SIZED_DELETE && !ENABLE_DYN_SIZED_DELETE */ - -void operator delete(void *p, size_t size) throw() - ALIAS(tc_delete); -void operator delete[](void *p, size_t size) throw() - ALIAS(tc_deletearray); - -#endif /* !ENABLE_SIZED_DELETE && !ENABLE_DYN_SIZED_DELETE */ - -extern "C" { - void* malloc(size_t size) __THROW ALIAS(tc_malloc); - void free(void* ptr) __THROW ALIAS(tc_free); - void* realloc(void* ptr, size_t size) __THROW ALIAS(tc_realloc); - void* calloc(size_t n, size_t size) __THROW ALIAS(tc_calloc); - void cfree(void* ptr) __THROW ALIAS(tc_cfree); - void* memalign(size_t align, size_t s) __THROW ALIAS(tc_memalign); - void* valloc(size_t size) __THROW ALIAS(tc_valloc); - void* pvalloc(size_t size) __THROW ALIAS(tc_pvalloc); - int posix_memalign(void** r, size_t a, size_t s) __THROW - ALIAS(tc_posix_memalign); -#ifndef __UCLIBC__ - void malloc_stats(void) __THROW ALIAS(tc_malloc_stats); -#endif - int mallopt(int cmd, int value) __THROW ALIAS(tc_mallopt); -#ifdef HAVE_STRUCT_MALLINFO - struct mallinfo mallinfo(void) __THROW ALIAS(tc_mallinfo); -#endif - size_t malloc_size(void* p) __THROW ALIAS(tc_malloc_size); -#if defined(__ANDROID__) - size_t malloc_usable_size(const void* p) __THROW - ALIAS(tc_malloc_size); -#else - size_t malloc_usable_size(void* p) __THROW ALIAS(tc_malloc_size); -#endif -} // extern "C" - -#undef ALIAS - -// No need to do anything at tcmalloc-registration time: we do it all -// via overriding weak symbols (at link time). -static void ReplaceSystemAlloc() { } - -#endif // TCMALLOC_LIBC_OVERRIDE_GCC_AND_WEAK_INL_H_ diff --git a/contrib/libtcmalloc/src/libc_override_glibc.h b/contrib/libtcmalloc/src/libc_override_glibc.h deleted file mode 100644 index cc17df315c0..00000000000 --- a/contrib/libtcmalloc/src/libc_override_glibc.h +++ /dev/null @@ -1,92 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2011, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Craig Silverstein -// -// Used to override malloc routines on systems that are using glibc. - -#ifndef TCMALLOC_LIBC_OVERRIDE_GLIBC_INL_H_ -#define TCMALLOC_LIBC_OVERRIDE_GLIBC_INL_H_ - -#include "config.h" -#include // for __GLIBC__ -#include - -#ifndef __GLIBC__ -# error libc_override_glibc.h is for glibc distributions only. -#endif - -// In glibc, the memory-allocation methods are weak symbols, so we can -// just override them with our own. If we're using gcc, we can use -// __attribute__((alias)) to do the overriding easily (exception: -// Mach-O, which doesn't support aliases). Otherwise we have to use a -// function call. -#if !defined(__GNUC__) || defined(__MACH__) - -// This also defines ReplaceSystemAlloc(). -# include "libc_override_redefine.h" // defines functions malloc()/etc - -#else // #if !defined(__GNUC__) || defined(__MACH__) - -// If we get here, we're a gcc system, so do all the overriding we do -// with gcc. This does the overriding of all the 'normal' memory -// allocation. This also defines ReplaceSystemAlloc(). -# include "libc_override_gcc_and_weak.h" - -// We also have to do some glibc-specific overriding. Some library -// routines on RedHat 9 allocate memory using malloc() and free it -// using __libc_free() (or vice-versa). Since we provide our own -// implementations of malloc/free, we need to make sure that the -// __libc_XXX variants (defined as part of glibc) also point to the -// same implementations. Since it only matters for redhat, we -// do it inside the gcc #ifdef, since redhat uses gcc. -// TODO(csilvers): only do this if we detect we're an old enough glibc? - -#define ALIAS(tc_fn) __attribute__ ((alias (#tc_fn))) -extern "C" { - void* __libc_malloc(size_t size) ALIAS(tc_malloc); - void __libc_free(void* ptr) ALIAS(tc_free); - void* __libc_realloc(void* ptr, size_t size) ALIAS(tc_realloc); - void* __libc_calloc(size_t n, size_t size) ALIAS(tc_calloc); - void __libc_cfree(void* ptr) ALIAS(tc_cfree); - void* __libc_memalign(size_t align, size_t s) ALIAS(tc_memalign); - void* __libc_valloc(size_t size) ALIAS(tc_valloc); - void* __libc_pvalloc(size_t size) ALIAS(tc_pvalloc); - int __posix_memalign(void** r, size_t a, size_t s) ALIAS(tc_posix_memalign); -} // extern "C" -#undef ALIAS - -#endif // #if defined(__GNUC__) && !defined(__MACH__) - -// No need to write ReplaceSystemAlloc(); one of the #includes above -// did it for us. - -#endif // TCMALLOC_LIBC_OVERRIDE_GLIBC_INL_H_ diff --git a/contrib/libtcmalloc/src/libc_override_osx.h b/contrib/libtcmalloc/src/libc_override_osx.h deleted file mode 100644 index afd57d1560a..00000000000 --- a/contrib/libtcmalloc/src/libc_override_osx.h +++ /dev/null @@ -1,308 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2011, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Craig Silverstein -// -// Used to override malloc routines on OS X systems. We use the -// malloc-zone functionality built into OS X to register our malloc -// routine. -// -// 1) We used to use the normal 'override weak libc malloc/etc' -// technique for OS X. This is not optimal because mach does not -// support the 'alias' attribute, so we had to have forwarding -// functions. It also does not work very well with OS X shared -// libraries (dylibs) -- in general, the shared libs don't use -// tcmalloc unless run with the DYLD_FORCE_FLAT_NAMESPACE envvar. -// -// 2) Another approach would be to use an interposition array: -// static const interpose_t interposers[] __attribute__((section("__DATA, __interpose"))) = { -// { (void *)tc_malloc, (void *)malloc }, -// { (void *)tc_free, (void *)free }, -// }; -// This requires the user to set the DYLD_INSERT_LIBRARIES envvar, so -// is not much better. -// -// 3) Registering a new malloc zone avoids all these issues: -// http://www.opensource.apple.com/source/Libc/Libc-583/include/malloc/malloc.h -// http://www.opensource.apple.com/source/Libc/Libc-583/gen/malloc.c -// If we make tcmalloc the default malloc zone (undocumented but -// possible) then all new allocs use it, even those in shared -// libraries. Allocs done before tcmalloc was installed, or in libs -// that aren't using tcmalloc for some reason, will correctly go -// through the malloc-zone interface when free-ing, and will pick up -// the libc free rather than tcmalloc free. So it should "never" -// cause a crash (famous last words). -// -// 4) The routines one must define for one's own malloc have changed -// between OS X versions. This requires some hoops on our part, but -// is only really annoying when it comes to posix_memalign. The right -// behavior there depends on what OS version tcmalloc was compiled on, -// but also what OS version the program is running on. For now, we -// punt and don't implement our own posix_memalign. Apps that really -// care can use tc_posix_memalign directly. - -#ifndef TCMALLOC_LIBC_OVERRIDE_OSX_INL_H_ -#define TCMALLOC_LIBC_OVERRIDE_OSX_INL_H_ - -#include "config.h" -#ifdef HAVE_FEATURES_H -#include -#endif -#include - -#if !defined(__APPLE__) -# error libc_override_glibc-osx.h is for OS X distributions only. -#endif - -#include -#include - -namespace tcmalloc { - void CentralCacheLockAll(); - void CentralCacheUnlockAll(); -} - -// from AvailabilityMacros.h -#if defined(MAC_OS_X_VERSION_10_6) && \ - MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 -extern "C" { - // This function is only available on 10.6 (and later) but the - // LibSystem headers do not use AvailabilityMacros.h to handle weak - // importing automatically. This prototype is a copy of the one in - // with the WEAK_IMPORT_ATTRBIUTE added. - extern malloc_zone_t *malloc_default_purgeable_zone(void) - WEAK_IMPORT_ATTRIBUTE; -} -#endif - -// We need to provide wrappers around all the libc functions. -namespace { -size_t mz_size(malloc_zone_t* zone, const void* ptr) { - if (MallocExtension::instance()->GetOwnership(ptr) != MallocExtension::kOwned) - return 0; // malloc_zone semantics: return 0 if we don't own the memory - - // TODO(csilvers): change this method to take a const void*, one day. - return MallocExtension::instance()->GetAllocatedSize(const_cast(ptr)); -} - -void* mz_malloc(malloc_zone_t* zone, size_t size) { - return tc_malloc(size); -} - -void* mz_calloc(malloc_zone_t* zone, size_t num_items, size_t size) { - return tc_calloc(num_items, size); -} - -void* mz_valloc(malloc_zone_t* zone, size_t size) { - return tc_valloc(size); -} - -void mz_free(malloc_zone_t* zone, void* ptr) { - return tc_free(ptr); -} - -void* mz_realloc(malloc_zone_t* zone, void* ptr, size_t size) { - return tc_realloc(ptr, size); -} - -void* mz_memalign(malloc_zone_t* zone, size_t align, size_t size) { - return tc_memalign(align, size); -} - -void mz_destroy(malloc_zone_t* zone) { - // A no-op -- we will not be destroyed! -} - -// malloc_introspection callbacks. I'm not clear on what all of these do. -kern_return_t mi_enumerator(task_t task, void *, - unsigned type_mask, vm_address_t zone_address, - memory_reader_t reader, - vm_range_recorder_t recorder) { - // Should enumerate all the pointers we have. Seems like a lot of work. - return KERN_FAILURE; -} - -size_t mi_good_size(malloc_zone_t *zone, size_t size) { - // I think it's always safe to return size, but we maybe could do better. - return size; -} - -boolean_t mi_check(malloc_zone_t *zone) { - return MallocExtension::instance()->VerifyAllMemory(); -} - -void mi_print(malloc_zone_t *zone, boolean_t verbose) { - int bufsize = 8192; - if (verbose) - bufsize = 102400; // I picked this size arbitrarily - char* buffer = new char[bufsize]; - MallocExtension::instance()->GetStats(buffer, bufsize); - fprintf(stdout, "%s", buffer); - delete[] buffer; -} - -void mi_log(malloc_zone_t *zone, void *address) { - // I don't think we support anything like this -} - -void mi_force_lock(malloc_zone_t *zone) { - tcmalloc::CentralCacheLockAll(); -} - -void mi_force_unlock(malloc_zone_t *zone) { - tcmalloc::CentralCacheUnlockAll(); -} - -void mi_statistics(malloc_zone_t *zone, malloc_statistics_t *stats) { - // TODO(csilvers): figure out how to fill these out - stats->blocks_in_use = 0; - stats->size_in_use = 0; - stats->max_size_in_use = 0; - stats->size_allocated = 0; -} - -boolean_t mi_zone_locked(malloc_zone_t *zone) { - return false; // Hopefully unneeded by us! -} - -} // unnamed namespace - -// OS X doesn't have pvalloc, cfree, malloc_statc, etc, so we can just -// define our own. :-) OS X supplies posix_memalign in some versions -// but not others, either strongly or weakly linked, in a way that's -// difficult enough to code to correctly, that I just don't try to -// support either memalign() or posix_memalign(). If you need them -// and are willing to code to tcmalloc, you can use tc_posix_memalign(). -extern "C" { - void cfree(void* p) { tc_cfree(p); } - void* pvalloc(size_t s) { return tc_pvalloc(s); } - void malloc_stats(void) { tc_malloc_stats(); } - int mallopt(int cmd, int v) { return tc_mallopt(cmd, v); } - // No struct mallinfo on OS X, so don't define mallinfo(). - // An alias for malloc_size(), which OS X defines. - size_t malloc_usable_size(void* p) { return tc_malloc_size(p); } -} // extern "C" - -static malloc_zone_t *get_default_zone() { - malloc_zone_t **zones = NULL; - unsigned int num_zones = 0; - - /* - * On OSX 10.12, malloc_default_zone returns a special zone that is not - * present in the list of registered zones. That zone uses a "lite zone" - * if one is present (apparently enabled when malloc stack logging is - * enabled), or the first registered zone otherwise. In practice this - * means unless malloc stack logging is enabled, the first registered - * zone is the default. - * So get the list of zones to get the first one, instead of relying on - * malloc_default_zone. - */ - if (KERN_SUCCESS != malloc_get_all_zones(0, NULL, (vm_address_t**) &zones, - &num_zones)) { - /* Reset the value in case the failure happened after it was set. */ - num_zones = 0; - } - - if (num_zones) - return zones[0]; - - return malloc_default_zone(); -} - - -static void ReplaceSystemAlloc() { - static malloc_introspection_t tcmalloc_introspection; - memset(&tcmalloc_introspection, 0, sizeof(tcmalloc_introspection)); - - tcmalloc_introspection.enumerator = &mi_enumerator; - tcmalloc_introspection.good_size = &mi_good_size; - tcmalloc_introspection.check = &mi_check; - tcmalloc_introspection.print = &mi_print; - tcmalloc_introspection.log = &mi_log; - tcmalloc_introspection.force_lock = &mi_force_lock; - tcmalloc_introspection.force_unlock = &mi_force_unlock; - - static malloc_zone_t tcmalloc_zone; - memset(&tcmalloc_zone, 0, sizeof(malloc_zone_t)); - - // Start with a version 4 zone which is used for OS X 10.4 and 10.5. - tcmalloc_zone.version = 4; - tcmalloc_zone.zone_name = "tcmalloc"; - tcmalloc_zone.size = &mz_size; - tcmalloc_zone.malloc = &mz_malloc; - tcmalloc_zone.calloc = &mz_calloc; - tcmalloc_zone.valloc = &mz_valloc; - tcmalloc_zone.free = &mz_free; - tcmalloc_zone.realloc = &mz_realloc; - tcmalloc_zone.destroy = &mz_destroy; - tcmalloc_zone.batch_malloc = NULL; - tcmalloc_zone.batch_free = NULL; - tcmalloc_zone.introspect = &tcmalloc_introspection; - - // from AvailabilityMacros.h -#if defined(MAC_OS_X_VERSION_10_6) && \ - MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 - // Switch to version 6 on OSX 10.6 to support memalign. - tcmalloc_zone.version = 6; - tcmalloc_zone.free_definite_size = NULL; - tcmalloc_zone.memalign = &mz_memalign; - tcmalloc_introspection.zone_locked = &mi_zone_locked; - - // Request the default purgable zone to force its creation. The - // current default zone is registered with the purgable zone for - // doing tiny and small allocs. Sadly, it assumes that the default - // zone is the szone implementation from OS X and will crash if it - // isn't. By creating the zone now, this will be true and changing - // the default zone won't cause a problem. This only needs to - // happen when actually running on OS X 10.6 and higher (note the - // ifdef above only checks if we were *compiled* with 10.6 or - // higher; at runtime we have to check if this symbol is defined.) - if (malloc_default_purgeable_zone) { - malloc_default_purgeable_zone(); - } -#endif - - // Register the tcmalloc zone. At this point, it will not be the - // default zone. - malloc_zone_register(&tcmalloc_zone); - - // Unregister and reregister the default zone. Unregistering swaps - // the specified zone with the last one registered which for the - // default zone makes the more recently registered zone the default - // zone. The default zone is then re-registered to ensure that - // allocations made from it earlier will be handled correctly. - // Things are not guaranteed to work that way, but it's how they work now. - malloc_zone_t *default_zone = get_default_zone(); - malloc_zone_unregister(default_zone); - malloc_zone_register(default_zone); -} - -#endif // TCMALLOC_LIBC_OVERRIDE_OSX_INL_H_ diff --git a/contrib/libtcmalloc/src/libc_override_redefine.h b/contrib/libtcmalloc/src/libc_override_redefine.h deleted file mode 100644 index 72679ef38b8..00000000000 --- a/contrib/libtcmalloc/src/libc_override_redefine.h +++ /dev/null @@ -1,92 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2011, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Craig Silverstein -// -// Used on systems that don't have their own definition of -// malloc/new/etc. (Typically this will be a windows msvcrt.dll that -// has been edited to remove the definitions.) We can just define our -// own as normal functions. -// -// This should also work on systems were all the malloc routines are -// defined as weak symbols, and there's no support for aliasing. - -#ifndef TCMALLOC_LIBC_OVERRIDE_REDEFINE_H_ -#define TCMALLOC_LIBC_OVERRIDE_REDEFINE_H_ - -void* operator new(size_t size) { return tc_new(size); } -void operator delete(void* p) throw() { tc_delete(p); } -void* operator new[](size_t size) { return tc_newarray(size); } -void operator delete[](void* p) throw() { tc_deletearray(p); } -void* operator new(size_t size, const std::nothrow_t& nt) throw() { - return tc_new_nothrow(size, nt); -} -void* operator new[](size_t size, const std::nothrow_t& nt) throw() { - return tc_newarray_nothrow(size, nt); -} -void operator delete(void* ptr, const std::nothrow_t& nt) throw() { - return tc_delete_nothrow(ptr, nt); -} -void operator delete[](void* ptr, const std::nothrow_t& nt) throw() { - return tc_deletearray_nothrow(ptr, nt); -} - -#ifdef ENABLE_SIZED_DELETE -void operator delete(void* p, size_t s) throw() { tc_delete_sized(p, s); } -void operator delete[](void* p, size_t s) throw(){ tc_deletearray_sized(p); } -#endif - -extern "C" { - void* malloc(size_t s) { return tc_malloc(s); } - void free(void* p) { tc_free(p); } - void* realloc(void* p, size_t s) { return tc_realloc(p, s); } - void* calloc(size_t n, size_t s) { return tc_calloc(n, s); } - void cfree(void* p) { tc_cfree(p); } - void* memalign(size_t a, size_t s) { return tc_memalign(a, s); } - void* valloc(size_t s) { return tc_valloc(s); } - void* pvalloc(size_t s) { return tc_pvalloc(s); } - int posix_memalign(void** r, size_t a, size_t s) { - return tc_posix_memalign(r, a, s); - } - void malloc_stats(void) { tc_malloc_stats(); } - int mallopt(int cmd, int v) { return tc_mallopt(cmd, v); } -#ifdef HAVE_STRUCT_MALLINFO - struct mallinfo mallinfo(void) { return tc_mallinfo(); } -#endif - size_t malloc_size(void* p) { return tc_malloc_size(p); } - size_t malloc_usable_size(void* p) { return tc_malloc_size(p); } -} // extern "C" - -// No need to do anything at tcmalloc-registration time: we do it all -// via overriding weak symbols (at link time). -static void ReplaceSystemAlloc() { } - -#endif // TCMALLOC_LIBC_OVERRIDE_REDEFINE_H_ diff --git a/contrib/libtcmalloc/src/linked_list.h b/contrib/libtcmalloc/src/linked_list.h deleted file mode 100644 index 66a07410760..00000000000 --- a/contrib/libtcmalloc/src/linked_list.h +++ /dev/null @@ -1,103 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat -// -// Some very basic linked list functions for dealing with using void * as -// storage. - -#ifndef TCMALLOC_LINKED_LIST_H_ -#define TCMALLOC_LINKED_LIST_H_ - -#include - -namespace tcmalloc { - -inline void *SLL_Next(void *t) { - return *(reinterpret_cast(t)); -} - -inline void SLL_SetNext(void *t, void *n) { - *(reinterpret_cast(t)) = n; -} - -inline void SLL_Push(void **list, void *element) { - SLL_SetNext(element, *list); - *list = element; -} - -inline void *SLL_Pop(void **list) { - void *result = *list; - *list = SLL_Next(*list); - return result; -} - -// Remove N elements from a linked list to which head points. head will be -// modified to point to the new head. start and end will point to the first -// and last nodes of the range. Note that end will point to NULL after this -// function is called. -inline void SLL_PopRange(void **head, int N, void **start, void **end) { - if (N == 0) { - *start = NULL; - *end = NULL; - return; - } - - void *tmp = *head; - for (int i = 1; i < N; ++i) { - tmp = SLL_Next(tmp); - } - - *start = *head; - *end = tmp; - *head = SLL_Next(tmp); - // Unlink range from list. - SLL_SetNext(tmp, NULL); -} - -inline void SLL_PushRange(void **head, void *start, void *end) { - if (!start) return; - SLL_SetNext(end, *head); - *head = start; -} - -inline size_t SLL_Size(void *head) { - int count = 0; - while (head) { - count++; - head = SLL_Next(head); - } - return count; -} - -} // namespace tcmalloc - -#endif // TCMALLOC_LINKED_LIST_H_ diff --git a/contrib/libtcmalloc/src/malloc_extension.cc b/contrib/libtcmalloc/src/malloc_extension.cc deleted file mode 100644 index 13a06f4dbe8..00000000000 --- a/contrib/libtcmalloc/src/malloc_extension.cc +++ /dev/null @@ -1,388 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat - -#include "config.h" -#include -#include -#include -#if defined HAVE_STDINT_H -#include -#elif defined HAVE_INTTYPES_H -#include -#else -#include -#endif -#include -#include "base/dynamic_annotations.h" -#include "base/sysinfo.h" // for FillProcSelfMaps -#ifndef NO_HEAP_CHECK -#include "gperftools/heap-checker.h" -#endif -#include "gperftools/malloc_extension.h" -#include "gperftools/malloc_extension_c.h" -#include "maybe_threads.h" -#include "base/googleinit.h" - -using STL_NAMESPACE::string; -using STL_NAMESPACE::vector; - -static void DumpAddressMap(string* result) { - *result += "\nMAPPED_LIBRARIES:\n"; - // We keep doubling until we get a fit - const size_t old_resultlen = result->size(); - for (int amap_size = 10240; amap_size < 10000000; amap_size *= 2) { - result->resize(old_resultlen + amap_size); - bool wrote_all = false; - const int bytes_written = - tcmalloc::FillProcSelfMaps(&((*result)[old_resultlen]), amap_size, - &wrote_all); - if (wrote_all) { // we fit! - (*result)[old_resultlen + bytes_written] = '\0'; - result->resize(old_resultlen + bytes_written); - return; - } - } - result->reserve(old_resultlen); // just don't print anything -} - -// Note: this routine is meant to be called before threads are spawned. -void MallocExtension::Initialize() { - static bool initialize_called = false; - - if (initialize_called) return; - initialize_called = true; - -#ifdef __GLIBC__ - // GNU libc++ versions 3.3 and 3.4 obey the environment variables - // GLIBCPP_FORCE_NEW and GLIBCXX_FORCE_NEW respectively. Setting - // one of these variables forces the STL default allocator to call - // new() or delete() for each allocation or deletion. Otherwise - // the STL allocator tries to avoid the high cost of doing - // allocations by pooling memory internally. However, tcmalloc - // does allocations really fast, especially for the types of small - // items one sees in STL, so it's better off just using us. - // TODO: control whether we do this via an environment variable? - setenv("GLIBCPP_FORCE_NEW", "1", false /* no overwrite*/); - setenv("GLIBCXX_FORCE_NEW", "1", false /* no overwrite*/); - - // Now we need to make the setenv 'stick', which it may not do since - // the env is flakey before main() is called. But luckily stl only - // looks at this env var the first time it tries to do an alloc, and - // caches what it finds. So we just cause an stl alloc here. - string dummy("I need to be allocated"); - dummy += "!"; // so the definition of dummy isn't optimized out -#endif /* __GLIBC__ */ -} - -// SysAllocator implementation -SysAllocator::~SysAllocator() {} - -// Default implementation -- does nothing -MallocExtension::~MallocExtension() { } -bool MallocExtension::VerifyAllMemory() { return true; } -bool MallocExtension::VerifyNewMemory(const void* p) { return true; } -bool MallocExtension::VerifyArrayNewMemory(const void* p) { return true; } -bool MallocExtension::VerifyMallocMemory(const void* p) { return true; } - -bool MallocExtension::GetNumericProperty(const char* property, size_t* value) { - return false; -} - -bool MallocExtension::SetNumericProperty(const char* property, size_t value) { - return false; -} - -void MallocExtension::GetStats(char* buffer, int length) { - assert(length > 0); - buffer[0] = '\0'; -} - -bool MallocExtension::MallocMemoryStats(int* blocks, size_t* total, - int histogram[kMallocHistogramSize]) { - *blocks = 0; - *total = 0; - memset(histogram, 0, sizeof(*histogram) * kMallocHistogramSize); - return true; -} - -void** MallocExtension::ReadStackTraces(int* sample_period) { - return NULL; -} - -void** MallocExtension::ReadHeapGrowthStackTraces() { - return NULL; -} - -void MallocExtension::MarkThreadIdle() { - // Default implementation does nothing -} - -void MallocExtension::MarkThreadBusy() { - // Default implementation does nothing -} - -SysAllocator* MallocExtension::GetSystemAllocator() { - return NULL; -} - -void MallocExtension::SetSystemAllocator(SysAllocator *a) { - // Default implementation does nothing -} - -void MallocExtension::ReleaseToSystem(size_t num_bytes) { - // Default implementation does nothing -} - -void MallocExtension::ReleaseFreeMemory() { - ReleaseToSystem(static_cast(-1)); // SIZE_T_MAX -} - -void MallocExtension::SetMemoryReleaseRate(double rate) { - // Default implementation does nothing -} - -double MallocExtension::GetMemoryReleaseRate() { - return -1.0; -} - -size_t MallocExtension::GetEstimatedAllocatedSize(size_t size) { - return size; -} - -size_t MallocExtension::GetAllocatedSize(const void* p) { - assert(GetOwnership(p) != kNotOwned); - return 0; -} - -MallocExtension::Ownership MallocExtension::GetOwnership(const void* p) { - return kUnknownOwnership; -} - -void MallocExtension::GetFreeListSizes( - vector* v) { - v->clear(); -} - -size_t MallocExtension::GetThreadCacheSize() { - return 0; -} - -void MallocExtension::MarkThreadTemporarilyIdle() { - // Default implementation does nothing -} - -// The current malloc extension object. - -static MallocExtension* current_instance; - -static void InitModule() { - if (current_instance != NULL) { - return; - } - current_instance = new MallocExtension; -#ifndef NO_HEAP_CHECK - HeapLeakChecker::IgnoreObject(current_instance); -#endif -} - -REGISTER_MODULE_INITIALIZER(malloc_extension_init, InitModule()) - -MallocExtension* MallocExtension::instance() { - InitModule(); - return current_instance; -} - -void MallocExtension::Register(MallocExtension* implementation) { - InitModule(); - // When running under valgrind, our custom malloc is replaced with - // valgrind's one and malloc extensions will not work. (Note: - // callers should be responsible for checking that they are the - // malloc that is really being run, before calling Register. This - // is just here as an extra sanity check.) - if (!RunningOnValgrind()) { - current_instance = implementation; - } -} - -// ----------------------------------------------------------------------- -// Heap sampling support -// ----------------------------------------------------------------------- - -namespace { - -// Accessors -uintptr_t Count(void** entry) { - return reinterpret_cast(entry[0]); -} -uintptr_t Size(void** entry) { - return reinterpret_cast(entry[1]); -} -uintptr_t Depth(void** entry) { - return reinterpret_cast(entry[2]); -} -void* PC(void** entry, int i) { - return entry[3+i]; -} - -void PrintCountAndSize(MallocExtensionWriter* writer, - uintptr_t count, uintptr_t size) { - char buf[100]; - snprintf(buf, sizeof(buf), - "%6" PRIu64 ": %8" PRIu64 " [%6" PRIu64 ": %8" PRIu64 "] @", - static_cast(count), - static_cast(size), - static_cast(count), - static_cast(size)); - writer->append(buf, strlen(buf)); -} - -void PrintHeader(MallocExtensionWriter* writer, - const char* label, void** entries) { - // Compute the total count and total size - uintptr_t total_count = 0; - uintptr_t total_size = 0; - for (void** entry = entries; Count(entry) != 0; entry += 3 + Depth(entry)) { - total_count += Count(entry); - total_size += Size(entry); - } - - const char* const kTitle = "heap profile: "; - writer->append(kTitle, strlen(kTitle)); - PrintCountAndSize(writer, total_count, total_size); - writer->append(" ", 1); - writer->append(label, strlen(label)); - writer->append("\n", 1); -} - -void PrintStackEntry(MallocExtensionWriter* writer, void** entry) { - PrintCountAndSize(writer, Count(entry), Size(entry)); - - for (int i = 0; i < Depth(entry); i++) { - char buf[32]; - snprintf(buf, sizeof(buf), " %p", PC(entry, i)); - writer->append(buf, strlen(buf)); - } - writer->append("\n", 1); -} - -} - -void MallocExtension::GetHeapSample(MallocExtensionWriter* writer) { - int sample_period = 0; - void** entries = ReadStackTraces(&sample_period); - if (entries == NULL) { - const char* const kErrorMsg = - "This malloc implementation does not support sampling.\n" - "As of 2005/01/26, only tcmalloc supports sampling, and\n" - "you are probably running a binary that does not use\n" - "tcmalloc.\n"; - writer->append(kErrorMsg, strlen(kErrorMsg)); - return; - } - - char label[32]; - sprintf(label, "heap_v2/%d", sample_period); - PrintHeader(writer, label, entries); - for (void** entry = entries; Count(entry) != 0; entry += 3 + Depth(entry)) { - PrintStackEntry(writer, entry); - } - delete[] entries; - - DumpAddressMap(writer); -} - -void MallocExtension::GetHeapGrowthStacks(MallocExtensionWriter* writer) { - void** entries = ReadHeapGrowthStackTraces(); - if (entries == NULL) { - const char* const kErrorMsg = - "This malloc implementation does not support " - "ReadHeapGrowthStackTraces().\n" - "As of 2005/09/27, only tcmalloc supports this, and you\n" - "are probably running a binary that does not use tcmalloc.\n"; - writer->append(kErrorMsg, strlen(kErrorMsg)); - return; - } - - // Do not canonicalize the stack entries, so that we get a - // time-ordered list of stack traces, which may be useful if the - // client wants to focus on the latest stack traces. - PrintHeader(writer, "growth", entries); - for (void** entry = entries; Count(entry) != 0; entry += 3 + Depth(entry)) { - PrintStackEntry(writer, entry); - } - delete[] entries; - - DumpAddressMap(writer); -} - -void MallocExtension::Ranges(void* arg, RangeFunction func) { - // No callbacks by default -} - -// These are C shims that work on the current instance. - -#define C_SHIM(fn, retval, paramlist, arglist) \ - extern "C" PERFTOOLS_DLL_DECL retval MallocExtension_##fn paramlist { \ - return MallocExtension::instance()->fn arglist; \ - } - -C_SHIM(VerifyAllMemory, int, (void), ()); -C_SHIM(VerifyNewMemory, int, (const void* p), (p)); -C_SHIM(VerifyArrayNewMemory, int, (const void* p), (p)); -C_SHIM(VerifyMallocMemory, int, (const void* p), (p)); -C_SHIM(MallocMemoryStats, int, - (int* blocks, size_t* total, int histogram[kMallocHistogramSize]), - (blocks, total, histogram)); - -C_SHIM(GetStats, void, - (char* buffer, int buffer_length), (buffer, buffer_length)); -C_SHIM(GetNumericProperty, int, - (const char* property, size_t* value), (property, value)); -C_SHIM(SetNumericProperty, int, - (const char* property, size_t value), (property, value)); - -C_SHIM(MarkThreadIdle, void, (void), ()); -C_SHIM(MarkThreadBusy, void, (void), ()); -C_SHIM(ReleaseFreeMemory, void, (void), ()); -C_SHIM(ReleaseToSystem, void, (size_t num_bytes), (num_bytes)); -C_SHIM(GetEstimatedAllocatedSize, size_t, (size_t size), (size)); -C_SHIM(GetAllocatedSize, size_t, (const void* p), (p)); -C_SHIM(GetThreadCacheSize, size_t, (void), ()); -C_SHIM(MarkThreadTemporarilyIdle, void, (void), ()); - -// Can't use the shim here because of the need to translate the enums. -extern "C" -MallocExtension_Ownership MallocExtension_GetOwnership(const void* p) { - return static_cast( - MallocExtension::instance()->GetOwnership(p)); -} diff --git a/contrib/libtcmalloc/src/malloc_hook-inl.h b/contrib/libtcmalloc/src/malloc_hook-inl.h deleted file mode 100644 index dbf4d46ed47..00000000000 --- a/contrib/libtcmalloc/src/malloc_hook-inl.h +++ /dev/null @@ -1,249 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat -// -// This has the implementation details of malloc_hook that are needed -// to use malloc-hook inside the tcmalloc system. It does not hold -// any of the client-facing calls that are used to add new hooks. - -#ifndef _MALLOC_HOOK_INL_H_ -#define _MALLOC_HOOK_INL_H_ - -#include -#include -#include "base/atomicops.h" -#include "base/basictypes.h" -#include - -#include "common.h" // for UNLIKELY - -namespace base { namespace internal { - -// Capacity of 8 means that HookList is 9 words. -static const int kHookListCapacity = 8; -// last entry is reserved for deprecated "singular" hooks. So we have -// 7 "normal" hooks per list -static const int kHookListMaxValues = 7; -static const int kHookListSingularIdx = 7; - -// HookList: a class that provides synchronized insertions and removals and -// lockless traversal. Most of the implementation is in malloc_hook.cc. -template -struct PERFTOOLS_DLL_DECL HookList { - COMPILE_ASSERT(sizeof(T) <= sizeof(AtomicWord), T_should_fit_in_AtomicWord); - - // Adds value to the list. Note that duplicates are allowed. Thread-safe and - // blocking (acquires hooklist_spinlock). Returns true on success; false - // otherwise (failures include invalid value and no space left). - bool Add(T value); - - void FixupPrivEndLocked(); - - // Removes the first entry matching value from the list. Thread-safe and - // blocking (acquires hooklist_spinlock). Returns true on success; false - // otherwise (failures include invalid value and no value found). - bool Remove(T value); - - // Store up to n values of the list in output_array, and return the number of - // elements stored. Thread-safe and non-blocking. This is fast (one memory - // access) if the list is empty. - int Traverse(T* output_array, int n) const; - - // Fast inline implementation for fast path of Invoke*Hook. - bool empty() const { - return base::subtle::NoBarrier_Load(&priv_end) == 0; - } - - // Used purely to handle deprecated singular hooks - T GetSingular() const { - const AtomicWord *place = &priv_data[kHookListSingularIdx]; - return bit_cast(base::subtle::NoBarrier_Load(place)); - } - - T ExchangeSingular(T new_val); - - // This internal data is not private so that the class is an aggregate and can - // be initialized by the linker. Don't access this directly. Use the - // INIT_HOOK_LIST macro in malloc_hook.cc. - - // One more than the index of the last valid element in priv_data. During - // 'Remove' this may be past the last valid element in priv_data, but - // subsequent values will be 0. - // - // Index kHookListCapacity-1 is reserved as 'deprecated' single hook pointer - AtomicWord priv_end; - AtomicWord priv_data[kHookListCapacity]; -}; - -ATTRIBUTE_VISIBILITY_HIDDEN extern HookList new_hooks_; -ATTRIBUTE_VISIBILITY_HIDDEN extern HookList delete_hooks_; -ATTRIBUTE_VISIBILITY_HIDDEN extern HookList premmap_hooks_; -ATTRIBUTE_VISIBILITY_HIDDEN extern HookList mmap_hooks_; -ATTRIBUTE_VISIBILITY_HIDDEN extern HookList mmap_replacement_; -ATTRIBUTE_VISIBILITY_HIDDEN extern HookList munmap_hooks_; -ATTRIBUTE_VISIBILITY_HIDDEN extern HookList munmap_replacement_; -ATTRIBUTE_VISIBILITY_HIDDEN extern HookList mremap_hooks_; -ATTRIBUTE_VISIBILITY_HIDDEN extern HookList presbrk_hooks_; -ATTRIBUTE_VISIBILITY_HIDDEN extern HookList sbrk_hooks_; - -} } // namespace base::internal - -// The following method is DEPRECATED -inline MallocHook::NewHook MallocHook::GetNewHook() { - return base::internal::new_hooks_.GetSingular(); -} - -inline void MallocHook::InvokeNewHook(const void* p, size_t s) { - if (UNLIKELY(!base::internal::new_hooks_.empty())) { - InvokeNewHookSlow(p, s); - } -} - -// The following method is DEPRECATED -inline MallocHook::DeleteHook MallocHook::GetDeleteHook() { - return base::internal::delete_hooks_.GetSingular(); -} - -inline void MallocHook::InvokeDeleteHook(const void* p) { - if (UNLIKELY(!base::internal::delete_hooks_.empty())) { - InvokeDeleteHookSlow(p); - } -} - -// The following method is DEPRECATED -inline MallocHook::PreMmapHook MallocHook::GetPreMmapHook() { - return base::internal::premmap_hooks_.GetSingular(); -} - -inline void MallocHook::InvokePreMmapHook(const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset) { - if (!base::internal::premmap_hooks_.empty()) { - InvokePreMmapHookSlow(start, size, protection, flags, fd, offset); - } -} - -// The following method is DEPRECATED -inline MallocHook::MmapHook MallocHook::GetMmapHook() { - return base::internal::mmap_hooks_.GetSingular(); -} - -inline void MallocHook::InvokeMmapHook(const void* result, - const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset) { - if (!base::internal::mmap_hooks_.empty()) { - InvokeMmapHookSlow(result, start, size, protection, flags, fd, offset); - } -} - -inline bool MallocHook::InvokeMmapReplacement(const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset, - void** result) { - if (!base::internal::mmap_replacement_.empty()) { - return InvokeMmapReplacementSlow(start, size, - protection, flags, - fd, offset, - result); - } - return false; -} - -// The following method is DEPRECATED -inline MallocHook::MunmapHook MallocHook::GetMunmapHook() { - return base::internal::munmap_hooks_.GetSingular(); -} - -inline void MallocHook::InvokeMunmapHook(const void* p, size_t size) { - if (!base::internal::munmap_hooks_.empty()) { - InvokeMunmapHookSlow(p, size); - } -} - -inline bool MallocHook::InvokeMunmapReplacement( - const void* p, size_t size, int* result) { - if (!base::internal::mmap_replacement_.empty()) { - return InvokeMunmapReplacementSlow(p, size, result); - } - return false; -} - -// The following method is DEPRECATED -inline MallocHook::MremapHook MallocHook::GetMremapHook() { - return base::internal::mremap_hooks_.GetSingular(); -} - -inline void MallocHook::InvokeMremapHook(const void* result, - const void* old_addr, - size_t old_size, - size_t new_size, - int flags, - const void* new_addr) { - if (!base::internal::mremap_hooks_.empty()) { - InvokeMremapHookSlow(result, old_addr, old_size, new_size, flags, new_addr); - } -} - -// The following method is DEPRECATED -inline MallocHook::PreSbrkHook MallocHook::GetPreSbrkHook() { - return base::internal::presbrk_hooks_.GetSingular(); -} - -inline void MallocHook::InvokePreSbrkHook(ptrdiff_t increment) { - if (!base::internal::presbrk_hooks_.empty() && increment != 0) { - InvokePreSbrkHookSlow(increment); - } -} - -// The following method is DEPRECATED -inline MallocHook::SbrkHook MallocHook::GetSbrkHook() { - return base::internal::sbrk_hooks_.GetSingular(); -} - -inline void MallocHook::InvokeSbrkHook(const void* result, - ptrdiff_t increment) { - if (!base::internal::sbrk_hooks_.empty() && increment != 0) { - InvokeSbrkHookSlow(result, increment); - } -} - -#endif /* _MALLOC_HOOK_INL_H_ */ diff --git a/contrib/libtcmalloc/src/malloc_hook.cc b/contrib/libtcmalloc/src/malloc_hook.cc deleted file mode 100644 index f87da8abbeb..00000000000 --- a/contrib/libtcmalloc/src/malloc_hook.cc +++ /dev/null @@ -1,703 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat - -#include "config.h" - -// Disable the glibc prototype of mremap(), as older versions of the -// system headers define this function with only four arguments, -// whereas newer versions allow an optional fifth argument: -#ifdef HAVE_MMAP -# define mremap glibc_mremap -# include -# undef mremap -#endif - -#include -#ifdef HAVE_STDINT_H -#include -#endif -#include -#include "base/logging.h" -#include "base/spinlock.h" -#include "maybe_emergency_malloc.h" -#include "maybe_threads.h" -#include "malloc_hook-inl.h" -#include - -// This #ifdef should almost never be set. Set NO_TCMALLOC_SAMPLES if -// you're porting to a system where you really can't get a stacktrace. -#ifdef NO_TCMALLOC_SAMPLES - // We use #define so code compiles even if you #include stacktrace.h somehow. -# define GetStackTrace(stack, depth, skip) (0) -#else -# include -#endif - -// __THROW is defined in glibc systems. It means, counter-intuitively, -// "This function will never throw an exception." It's an optional -// optimization tool, but we may need to use it to match glibc prototypes. -#ifndef __THROW // I guess we're not on a glibc system -# define __THROW // __THROW is just an optimization, so ok to make it "" -#endif - -using std::copy; - - -// Declaration of default weak initialization function, that can be overridden -// by linking-in a strong definition (as heap-checker.cc does). This is -// extern "C" so that it doesn't trigger gold's --detect-odr-violations warning, -// which only looks at C++ symbols. -// -// This function is declared here as weak, and defined later, rather than a more -// straightforward simple weak definition, as a workround for an icc compiler -// issue ((Intel reference 290819). This issue causes icc to resolve weak -// symbols too early, at compile rather than link time. By declaring it (weak) -// here, then defining it below after its use, we can avoid the problem. -extern "C" { -ATTRIBUTE_WEAK void MallocHook_InitAtFirstAllocation_HeapLeakChecker(); -} - -namespace { - -void RemoveInitialHooksAndCallInitializers(); // below. - -pthread_once_t once = PTHREAD_ONCE_INIT; - -// These hooks are installed in MallocHook as the only initial hooks. The first -// hook that is called will run RemoveInitialHooksAndCallInitializers (see the -// definition below) and then redispatch to any malloc hooks installed by -// RemoveInitialHooksAndCallInitializers. -// -// Note(llib): there is a possibility of a race in the event that there are -// multiple threads running before the first allocation. This is pretty -// difficult to achieve, but if it is then multiple threads may concurrently do -// allocations. The first caller will call -// RemoveInitialHooksAndCallInitializers via one of the initial hooks. A -// concurrent allocation may, depending on timing either: -// * still have its initial malloc hook installed, run that and block on waiting -// for the first caller to finish its call to -// RemoveInitialHooksAndCallInitializers, and proceed normally. -// * occur some time during the RemoveInitialHooksAndCallInitializers call, at -// which point there could be no initial hooks and the subsequent hooks that -// are about to be set up by RemoveInitialHooksAndCallInitializers haven't -// been installed yet. I think the worst we can get is that some allocations -// will not get reported to some hooks set by the initializers called from -// RemoveInitialHooksAndCallInitializers. - -void InitialNewHook(const void* ptr, size_t size) { - perftools_pthread_once(&once, &RemoveInitialHooksAndCallInitializers); - MallocHook::InvokeNewHook(ptr, size); -} - -void InitialPreMMapHook(const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset) { - perftools_pthread_once(&once, &RemoveInitialHooksAndCallInitializers); - MallocHook::InvokePreMmapHook(start, size, protection, flags, fd, offset); -} - -void InitialPreSbrkHook(ptrdiff_t increment) { - perftools_pthread_once(&once, &RemoveInitialHooksAndCallInitializers); - MallocHook::InvokePreSbrkHook(increment); -} - -// This function is called at most once by one of the above initial malloc -// hooks. It removes all initial hooks and initializes all other clients that -// want to get control at the very first memory allocation. The initializers -// may assume that the initial malloc hooks have been removed. The initializers -// may set up malloc hooks and allocate memory. -void RemoveInitialHooksAndCallInitializers() { - RAW_CHECK(MallocHook::RemoveNewHook(&InitialNewHook), ""); - RAW_CHECK(MallocHook::RemovePreMmapHook(&InitialPreMMapHook), ""); - RAW_CHECK(MallocHook::RemovePreSbrkHook(&InitialPreSbrkHook), ""); - - // HeapLeakChecker is currently the only module that needs to get control on - // the first memory allocation, but one can add other modules by following the - // same weak/strong function pattern. - MallocHook_InitAtFirstAllocation_HeapLeakChecker(); -} - -} // namespace - -// Weak default initialization function that must go after its use. -extern "C" void MallocHook_InitAtFirstAllocation_HeapLeakChecker() { - // Do nothing. -} - -namespace base { namespace internal { - -// This lock is shared between all implementations of HookList::Add & Remove. -// The potential for contention is very small. This needs to be a SpinLock and -// not a Mutex since it's possible for Mutex locking to allocate memory (e.g., -// per-thread allocation in debug builds), which could cause infinite recursion. -static SpinLock hooklist_spinlock(base::LINKER_INITIALIZED); - -template -bool HookList::Add(T value_as_t) { - AtomicWord value = bit_cast(value_as_t); - if (value == 0) { - return false; - } - SpinLockHolder l(&hooklist_spinlock); - // Find the first slot in data that is 0. - int index = 0; - while ((index < kHookListMaxValues) && - (base::subtle::NoBarrier_Load(&priv_data[index]) != 0)) { - ++index; - } - if (index == kHookListMaxValues) { - return false; - } - AtomicWord prev_num_hooks = base::subtle::Acquire_Load(&priv_end); - base::subtle::NoBarrier_Store(&priv_data[index], value); - if (prev_num_hooks <= index) { - base::subtle::NoBarrier_Store(&priv_end, index + 1); - } - return true; -} - -template -void HookList::FixupPrivEndLocked() { - AtomicWord hooks_end = base::subtle::NoBarrier_Load(&priv_end); - while ((hooks_end > 0) && - (base::subtle::NoBarrier_Load(&priv_data[hooks_end - 1]) == 0)) { - --hooks_end; - } - base::subtle::NoBarrier_Store(&priv_end, hooks_end); -} - -template -bool HookList::Remove(T value_as_t) { - if (value_as_t == 0) { - return false; - } - SpinLockHolder l(&hooklist_spinlock); - AtomicWord hooks_end = base::subtle::NoBarrier_Load(&priv_end); - int index = 0; - while (index < hooks_end && value_as_t != bit_cast( - base::subtle::NoBarrier_Load(&priv_data[index]))) { - ++index; - } - if (index == hooks_end) { - return false; - } - base::subtle::NoBarrier_Store(&priv_data[index], 0); - FixupPrivEndLocked(); - return true; -} - -template -int HookList::Traverse(T* output_array, int n) const { - AtomicWord hooks_end = base::subtle::Acquire_Load(&priv_end); - int actual_hooks_end = 0; - for (int i = 0; i < hooks_end && n > 0; ++i) { - AtomicWord data = base::subtle::Acquire_Load(&priv_data[i]); - if (data != 0) { - *output_array++ = bit_cast(data); - ++actual_hooks_end; - --n; - } - } - return actual_hooks_end; -} - -template -T HookList::ExchangeSingular(T value_as_t) { - AtomicWord value = bit_cast(value_as_t); - AtomicWord old_value; - SpinLockHolder l(&hooklist_spinlock); - old_value = base::subtle::NoBarrier_Load(&priv_data[kHookListSingularIdx]); - base::subtle::NoBarrier_Store(&priv_data[kHookListSingularIdx], value); - if (value != 0) { - base::subtle::NoBarrier_Store(&priv_end, kHookListSingularIdx + 1); - } else { - FixupPrivEndLocked(); - } - return bit_cast(old_value); -} - -// Initialize a HookList (optionally with the given initial_value in index 0). -#define INIT_HOOK_LIST { 0 } -#define INIT_HOOK_LIST_WITH_VALUE(initial_value) \ - { 1, { reinterpret_cast(initial_value) } } - -// Explicit instantiation for malloc_hook_test.cc. This ensures all the methods -// are instantiated. -template struct HookList; - -HookList new_hooks_ = - INIT_HOOK_LIST_WITH_VALUE(&InitialNewHook); -HookList delete_hooks_ = INIT_HOOK_LIST; -HookList premmap_hooks_ = - INIT_HOOK_LIST_WITH_VALUE(&InitialPreMMapHook); -HookList mmap_hooks_ = INIT_HOOK_LIST; -HookList munmap_hooks_ = INIT_HOOK_LIST; -HookList mremap_hooks_ = INIT_HOOK_LIST; -HookList presbrk_hooks_ = - INIT_HOOK_LIST_WITH_VALUE(InitialPreSbrkHook); -HookList sbrk_hooks_ = INIT_HOOK_LIST; - -// These lists contain either 0 or 1 hooks. -HookList mmap_replacement_ = { 0 }; -HookList munmap_replacement_ = { 0 }; - -#undef INIT_HOOK_LIST_WITH_VALUE -#undef INIT_HOOK_LIST - -} } // namespace base::internal - -using base::internal::kHookListMaxValues; -using base::internal::new_hooks_; -using base::internal::delete_hooks_; -using base::internal::premmap_hooks_; -using base::internal::mmap_hooks_; -using base::internal::mmap_replacement_; -using base::internal::munmap_hooks_; -using base::internal::munmap_replacement_; -using base::internal::mremap_hooks_; -using base::internal::presbrk_hooks_; -using base::internal::sbrk_hooks_; - -// These are available as C bindings as well as C++, hence their -// definition outside the MallocHook class. -extern "C" -int MallocHook_AddNewHook(MallocHook_NewHook hook) { - RAW_VLOG(10, "AddNewHook(%p)", hook); - return new_hooks_.Add(hook); -} - -extern "C" -int MallocHook_RemoveNewHook(MallocHook_NewHook hook) { - RAW_VLOG(10, "RemoveNewHook(%p)", hook); - return new_hooks_.Remove(hook); -} - -extern "C" -int MallocHook_AddDeleteHook(MallocHook_DeleteHook hook) { - RAW_VLOG(10, "AddDeleteHook(%p)", hook); - return delete_hooks_.Add(hook); -} - -extern "C" -int MallocHook_RemoveDeleteHook(MallocHook_DeleteHook hook) { - RAW_VLOG(10, "RemoveDeleteHook(%p)", hook); - return delete_hooks_.Remove(hook); -} - -extern "C" -int MallocHook_AddPreMmapHook(MallocHook_PreMmapHook hook) { - RAW_VLOG(10, "AddPreMmapHook(%p)", hook); - return premmap_hooks_.Add(hook); -} - -extern "C" -int MallocHook_RemovePreMmapHook(MallocHook_PreMmapHook hook) { - RAW_VLOG(10, "RemovePreMmapHook(%p)", hook); - return premmap_hooks_.Remove(hook); -} - -extern "C" -int MallocHook_SetMmapReplacement(MallocHook_MmapReplacement hook) { - RAW_VLOG(10, "SetMmapReplacement(%p)", hook); - // NOTE this is a best effort CHECK. Concurrent sets could succeed since - // this test is outside of the Add spin lock. - RAW_CHECK(mmap_replacement_.empty(), "Only one MMapReplacement is allowed."); - return mmap_replacement_.Add(hook); -} - -extern "C" -int MallocHook_RemoveMmapReplacement(MallocHook_MmapReplacement hook) { - RAW_VLOG(10, "RemoveMmapReplacement(%p)", hook); - return mmap_replacement_.Remove(hook); -} - -extern "C" -int MallocHook_AddMmapHook(MallocHook_MmapHook hook) { - RAW_VLOG(10, "AddMmapHook(%p)", hook); - return mmap_hooks_.Add(hook); -} - -extern "C" -int MallocHook_RemoveMmapHook(MallocHook_MmapHook hook) { - RAW_VLOG(10, "RemoveMmapHook(%p)", hook); - return mmap_hooks_.Remove(hook); -} - -extern "C" -int MallocHook_AddMunmapHook(MallocHook_MunmapHook hook) { - RAW_VLOG(10, "AddMunmapHook(%p)", hook); - return munmap_hooks_.Add(hook); -} - -extern "C" -int MallocHook_RemoveMunmapHook(MallocHook_MunmapHook hook) { - RAW_VLOG(10, "RemoveMunmapHook(%p)", hook); - return munmap_hooks_.Remove(hook); -} - -extern "C" -int MallocHook_SetMunmapReplacement(MallocHook_MunmapReplacement hook) { - RAW_VLOG(10, "SetMunmapReplacement(%p)", hook); - // NOTE this is a best effort CHECK. Concurrent sets could succeed since - // this test is outside of the Add spin lock. - RAW_CHECK(munmap_replacement_.empty(), - "Only one MunmapReplacement is allowed."); - return munmap_replacement_.Add(hook); -} - -extern "C" -int MallocHook_RemoveMunmapReplacement(MallocHook_MunmapReplacement hook) { - RAW_VLOG(10, "RemoveMunmapReplacement(%p)", hook); - return munmap_replacement_.Remove(hook); -} - -extern "C" -int MallocHook_AddMremapHook(MallocHook_MremapHook hook) { - RAW_VLOG(10, "AddMremapHook(%p)", hook); - return mremap_hooks_.Add(hook); -} - -extern "C" -int MallocHook_RemoveMremapHook(MallocHook_MremapHook hook) { - RAW_VLOG(10, "RemoveMremapHook(%p)", hook); - return mremap_hooks_.Remove(hook); -} - -extern "C" -int MallocHook_AddPreSbrkHook(MallocHook_PreSbrkHook hook) { - RAW_VLOG(10, "AddPreSbrkHook(%p)", hook); - return presbrk_hooks_.Add(hook); -} - -extern "C" -int MallocHook_RemovePreSbrkHook(MallocHook_PreSbrkHook hook) { - RAW_VLOG(10, "RemovePreSbrkHook(%p)", hook); - return presbrk_hooks_.Remove(hook); -} - -extern "C" -int MallocHook_AddSbrkHook(MallocHook_SbrkHook hook) { - RAW_VLOG(10, "AddSbrkHook(%p)", hook); - return sbrk_hooks_.Add(hook); -} - -extern "C" -int MallocHook_RemoveSbrkHook(MallocHook_SbrkHook hook) { - RAW_VLOG(10, "RemoveSbrkHook(%p)", hook); - return sbrk_hooks_.Remove(hook); -} - -// The code below is DEPRECATED. -extern "C" -MallocHook_NewHook MallocHook_SetNewHook(MallocHook_NewHook hook) { - RAW_VLOG(10, "SetNewHook(%p)", hook); - return new_hooks_.ExchangeSingular(hook); -} - -extern "C" -MallocHook_DeleteHook MallocHook_SetDeleteHook(MallocHook_DeleteHook hook) { - RAW_VLOG(10, "SetDeleteHook(%p)", hook); - return delete_hooks_.ExchangeSingular(hook); -} - -extern "C" -MallocHook_PreMmapHook MallocHook_SetPreMmapHook(MallocHook_PreMmapHook hook) { - RAW_VLOG(10, "SetPreMmapHook(%p)", hook); - return premmap_hooks_.ExchangeSingular(hook); -} - -extern "C" -MallocHook_MmapHook MallocHook_SetMmapHook(MallocHook_MmapHook hook) { - RAW_VLOG(10, "SetMmapHook(%p)", hook); - return mmap_hooks_.ExchangeSingular(hook); -} - -extern "C" -MallocHook_MunmapHook MallocHook_SetMunmapHook(MallocHook_MunmapHook hook) { - RAW_VLOG(10, "SetMunmapHook(%p)", hook); - return munmap_hooks_.ExchangeSingular(hook); -} - -extern "C" -MallocHook_MremapHook MallocHook_SetMremapHook(MallocHook_MremapHook hook) { - RAW_VLOG(10, "SetMremapHook(%p)", hook); - return mremap_hooks_.ExchangeSingular(hook); -} - -extern "C" -MallocHook_PreSbrkHook MallocHook_SetPreSbrkHook(MallocHook_PreSbrkHook hook) { - RAW_VLOG(10, "SetPreSbrkHook(%p)", hook); - return presbrk_hooks_.ExchangeSingular(hook); -} - -extern "C" -MallocHook_SbrkHook MallocHook_SetSbrkHook(MallocHook_SbrkHook hook) { - RAW_VLOG(10, "SetSbrkHook(%p)", hook); - return sbrk_hooks_.ExchangeSingular(hook); -} -// End of DEPRECATED code section. - -// Note: embedding the function calls inside the traversal of HookList would be -// very confusing, as it is legal for a hook to remove itself and add other -// hooks. Doing traversal first, and then calling the hooks ensures we only -// call the hooks registered at the start. -#define INVOKE_HOOKS(HookType, hook_list, args) do { \ - HookType hooks[kHookListMaxValues]; \ - int num_hooks = hook_list.Traverse(hooks, kHookListMaxValues); \ - for (int i = 0; i < num_hooks; ++i) { \ - (*hooks[i])args; \ - } \ - } while (0) - -// There should only be one replacement. Return the result of the first -// one, or false if there is none. -#define INVOKE_REPLACEMENT(HookType, hook_list, args) do { \ - HookType hooks[kHookListMaxValues]; \ - int num_hooks = hook_list.Traverse(hooks, kHookListMaxValues); \ - return (num_hooks > 0 && (*hooks[0])args); \ - } while (0) - - -void MallocHook::InvokeNewHookSlow(const void* p, size_t s) { - if (tcmalloc::IsEmergencyPtr(p)) { - return; - } - INVOKE_HOOKS(NewHook, new_hooks_, (p, s)); -} - -void MallocHook::InvokeDeleteHookSlow(const void* p) { - if (tcmalloc::IsEmergencyPtr(p)) { - return; - } - INVOKE_HOOKS(DeleteHook, delete_hooks_, (p)); -} - -void MallocHook::InvokePreMmapHookSlow(const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset) { - INVOKE_HOOKS(PreMmapHook, premmap_hooks_, (start, size, protection, flags, fd, - offset)); -} - -void MallocHook::InvokeMmapHookSlow(const void* result, - const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset) { - INVOKE_HOOKS(MmapHook, mmap_hooks_, (result, start, size, protection, flags, - fd, offset)); -} - -bool MallocHook::InvokeMmapReplacementSlow(const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset, - void** result) { - INVOKE_REPLACEMENT(MmapReplacement, mmap_replacement_, - (start, size, protection, flags, fd, offset, result)); -} - -void MallocHook::InvokeMunmapHookSlow(const void* p, size_t s) { - INVOKE_HOOKS(MunmapHook, munmap_hooks_, (p, s)); -} - -bool MallocHook::InvokeMunmapReplacementSlow(const void* p, - size_t s, - int* result) { - INVOKE_REPLACEMENT(MunmapReplacement, munmap_replacement_, (p, s, result)); -} - -void MallocHook::InvokeMremapHookSlow(const void* result, - const void* old_addr, - size_t old_size, - size_t new_size, - int flags, - const void* new_addr) { - INVOKE_HOOKS(MremapHook, mremap_hooks_, (result, old_addr, old_size, new_size, - flags, new_addr)); -} - -void MallocHook::InvokePreSbrkHookSlow(ptrdiff_t increment) { - INVOKE_HOOKS(PreSbrkHook, presbrk_hooks_, (increment)); -} - -void MallocHook::InvokeSbrkHookSlow(const void* result, ptrdiff_t increment) { - INVOKE_HOOKS(SbrkHook, sbrk_hooks_, (result, increment)); -} - -#undef INVOKE_HOOKS - -#ifndef NO_TCMALLOC_SAMPLES - -DEFINE_ATTRIBUTE_SECTION_VARS(google_malloc); -DECLARE_ATTRIBUTE_SECTION_VARS(google_malloc); - // actual functions are in debugallocation.cc or tcmalloc.cc -DEFINE_ATTRIBUTE_SECTION_VARS(malloc_hook); -DECLARE_ATTRIBUTE_SECTION_VARS(malloc_hook); - // actual functions are in this file, malloc_hook.cc, and low_level_alloc.cc - -#define ADDR_IN_ATTRIBUTE_SECTION(addr, name) \ - (reinterpret_cast(ATTRIBUTE_SECTION_START(name)) <= \ - reinterpret_cast(addr) && \ - reinterpret_cast(addr) < \ - reinterpret_cast(ATTRIBUTE_SECTION_STOP(name))) - -// Return true iff 'caller' is a return address within a function -// that calls one of our hooks via MallocHook:Invoke*. -// A helper for GetCallerStackTrace. -static inline bool InHookCaller(const void* caller) { - return ADDR_IN_ATTRIBUTE_SECTION(caller, google_malloc) || - ADDR_IN_ATTRIBUTE_SECTION(caller, malloc_hook); - // We can use one section for everything except tcmalloc_or_debug - // due to its special linkage mode, which prevents merging of the sections. -} - -#undef ADDR_IN_ATTRIBUTE_SECTION - -static bool checked_sections = false; - -static inline void CheckInHookCaller() { - if (!checked_sections) { - INIT_ATTRIBUTE_SECTION_VARS(google_malloc); - if (ATTRIBUTE_SECTION_START(google_malloc) == - ATTRIBUTE_SECTION_STOP(google_malloc)) { - RAW_LOG(ERROR, "google_malloc section is missing, " - "thus InHookCaller is broken!"); - } - INIT_ATTRIBUTE_SECTION_VARS(malloc_hook); - if (ATTRIBUTE_SECTION_START(malloc_hook) == - ATTRIBUTE_SECTION_STOP(malloc_hook)) { - RAW_LOG(ERROR, "malloc_hook section is missing, " - "thus InHookCaller is broken!"); - } - checked_sections = true; - } -} - -#endif // !NO_TCMALLOC_SAMPLES - -// We can improve behavior/compactness of this function -// if we pass a generic test function (with a generic arg) -// into the implementations for GetStackTrace instead of the skip_count. -extern "C" int MallocHook_GetCallerStackTrace(void** result, int max_depth, - int skip_count) { -#if defined(NO_TCMALLOC_SAMPLES) - return 0; -#elif !defined(HAVE_ATTRIBUTE_SECTION_START) - // Fall back to GetStackTrace and good old but fragile frame skip counts. - // Note: this path is inaccurate when a hook is not called directly by an - // allocation function but is daisy-chained through another hook, - // search for MallocHook::(Get|Set|Invoke)* to find such cases. - return GetStackTrace(result, max_depth, skip_count + int(DEBUG_MODE)); - // due to -foptimize-sibling-calls in opt mode - // there's no need for extra frame skip here then -#else - CheckInHookCaller(); - // MallocHook caller determination via InHookCaller works, use it: - static const int kMaxSkip = 32 + 6 + 3; - // Constant tuned to do just one GetStackTrace call below in practice - // and not get many frames that we don't actually need: - // currently max passsed max_depth is 32, - // max passed/needed skip_count is 6 - // and 3 is to account for some hook daisy chaining. - static const int kStackSize = kMaxSkip + 1; - void* stack[kStackSize]; - int depth = GetStackTrace(stack, kStackSize, 1); // skip this function frame - if (depth == 0) // silenty propagate cases when GetStackTrace does not work - return 0; - for (int i = 0; i < depth; ++i) { // stack[0] is our immediate caller - if (InHookCaller(stack[i])) { - RAW_VLOG(10, "Found hooked allocator at %d: %p <- %p", - i, stack[i], stack[i+1]); - i += 1; // skip hook caller frame - depth -= i; // correct depth - if (depth > max_depth) depth = max_depth; - copy(stack + i, stack + i + depth, result); - if (depth < max_depth && depth + i == kStackSize) { - // get frames for the missing depth - depth += - GetStackTrace(result + depth, max_depth - depth, 1 + kStackSize); - } - return depth; - } - } - RAW_LOG(WARNING, "Hooked allocator frame not found, returning empty trace"); - // If this happens try increasing kMaxSkip - // or else something must be wrong with InHookCaller, - // e.g. for every section used in InHookCaller - // all functions in that section must be inside the same library. - return 0; -#endif -} - -// On systems where we know how, we override mmap/munmap/mremap/sbrk -// to provide support for calling the related hooks (in addition, -// of course, to doing what these functions normally do). - -#if defined(__linux) -# include "malloc_hook_mmap_linux.h" - -#elif defined(__FreeBSD__) -# include "malloc_hook_mmap_freebsd.h" - -#else - -/*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot, - int flags, int fd, off_t offset) { - void* result; - if (!MallocHook::InvokeMmapReplacement( - start, length, prot, flags, fd, offset, &result)) { - result = mmap(start, length, prot, flags, fd, offset); - } - return result; -} - -/*static*/int MallocHook::UnhookedMUnmap(void *start, size_t length) { - int result; - if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { - result = munmap(start, length); - } - return result; -} - -#endif diff --git a/contrib/libtcmalloc/src/malloc_hook_mmap_freebsd.h b/contrib/libtcmalloc/src/malloc_hook_mmap_freebsd.h deleted file mode 100644 index 8575dcc7c08..00000000000 --- a/contrib/libtcmalloc/src/malloc_hook_mmap_freebsd.h +++ /dev/null @@ -1,135 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2011, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Override mmap/munmap/mremap/sbrk to provide support for calling the -// related hooks (in addition, of course, to doing what these -// functions normally do). - -#ifndef __FreeBSD__ -# error Should only be including malloc_hook_mmap_freebsd.h on FreeBSD systems. -#endif - -#include -#include -#include -#include -#include - -// Make sure mmap doesn't get #define'd away by -#undef mmap - -// According to the FreeBSD documentation, use syscall if you do not -// need 64-bit alignment otherwise use __syscall. Indeed, syscall -// doesn't work correctly in most situations on 64-bit. It's return -// type is 'int' so for things like SYS_mmap, it actually truncates -// the returned address to 32-bits. -#if defined(__amd64__) || defined(__x86_64__) -# define MALLOC_HOOK_SYSCALL __syscall -#else -# define MALLOC_HOOK_SYSCALL syscall -#endif - - -extern "C" { - void* mmap(void *start, size_t length,int prot, int flags, - int fd, off_t offset) __THROW - ATTRIBUTE_SECTION(malloc_hook); - int munmap(void* start, size_t length) __THROW - ATTRIBUTE_SECTION(malloc_hook); - void* sbrk(intptr_t increment) __THROW - ATTRIBUTE_SECTION(malloc_hook); -} - -static inline void* do_mmap(void *start, size_t length, - int prot, int flags, - int fd, off_t offset) __THROW { - return (void *)MALLOC_HOOK_SYSCALL(SYS_mmap, - start, length, prot, flags, fd, offset); -} - -static inline void* do_sbrk(intptr_t increment) { - static void *(*libc_sbrk)(intptr_t); - if (libc_sbrk == NULL) - libc_sbrk = (void *(*)(intptr_t))dlsym(RTLD_NEXT, "sbrk"); - - return libc_sbrk(increment); -} - - -extern "C" void* mmap(void *start, size_t length, int prot, int flags, - int fd, off_t offset) __THROW { - MallocHook::InvokePreMmapHook(start, length, prot, flags, fd, offset); - void *result; - if (!MallocHook::InvokeMmapReplacement( - start, length, prot, flags, fd, offset, &result)) { - result = do_mmap(start, length, prot, flags, fd, - static_cast(offset)); // avoid sign extension - } - MallocHook::InvokeMmapHook(result, start, length, prot, flags, fd, offset); - return result; -} - -extern "C" int munmap(void* start, size_t length) __THROW { - MallocHook::InvokeMunmapHook(start, length); - int result; - if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { - result = MALLOC_HOOK_SYSCALL(SYS_munmap, start, length); - } - - return result; -} - -extern "C" void* sbrk(intptr_t increment) __THROW { - MallocHook::InvokePreSbrkHook(increment); - void *result = do_sbrk(increment); - MallocHook::InvokeSbrkHook(result, increment); - return result; -} - -/*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot, - int flags, int fd, off_t offset) { - void* result; - if (!MallocHook::InvokeMmapReplacement( - start, length, prot, flags, fd, offset, &result)) { - result = do_mmap(start, length, prot, flags, fd, offset); - } - - return result; -} - -/*static*/int MallocHook::UnhookedMUnmap(void *start, size_t length) { - int result; - if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { - result = MALLOC_HOOK_SYSCALL(SYS_munmap, start, length); - } - return result; -} - -#undef MALLOC_HOOK_SYSCALL diff --git a/contrib/libtcmalloc/src/malloc_hook_mmap_linux.h b/contrib/libtcmalloc/src/malloc_hook_mmap_linux.h deleted file mode 100644 index 4b1386185bc..00000000000 --- a/contrib/libtcmalloc/src/malloc_hook_mmap_linux.h +++ /dev/null @@ -1,238 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat - -// We define mmap() and mmap64(), which somewhat reimplements libc's mmap -// syscall stubs. Unfortunately libc only exports the stubs via weak symbols -// (which we're overriding with our mmap64() and mmap() wrappers) so we can't -// just call through to them. - -#ifndef __linux -# error Should only be including malloc_hook_mmap_linux.h on linux systems. -#endif - -#include -#include -#include -#include -#include "base/linux_syscall_support.h" - -// The x86-32 case and the x86-64 case differ: -// 32b has a mmap2() syscall, 64b does not. -// 64b and 32b have different calling conventions for mmap(). - -// I test for 64-bit first so I don't have to do things like -// '#if (defined(__mips__) && !defined(__MIPS64__))' as a mips32 check. -#if defined(__x86_64__) || defined(__PPC64__) || defined(__aarch64__) || (defined(_MIPS_SIM) && _MIPS_SIM == _ABI64) || defined(__s390__) - -static inline void* do_mmap64(void *start, size_t length, - int prot, int flags, - int fd, __off64_t offset) __THROW { - return sys_mmap(start, length, prot, flags, fd, offset); -} - -#define MALLOC_HOOK_HAVE_DO_MMAP64 1 - -#elif defined(__i386__) || defined(__PPC__) || defined(__mips__) || \ - defined(__arm__) - -static inline void* do_mmap64(void *start, size_t length, - int prot, int flags, - int fd, __off64_t offset) __THROW { - void *result; - - // Try mmap2() unless it's not supported - static bool have_mmap2 = true; - if (have_mmap2) { - static int pagesize = 0; - if (!pagesize) pagesize = getpagesize(); - - // Check that the offset is page aligned - if (offset & (pagesize - 1)) { - result = MAP_FAILED; - errno = EINVAL; - goto out; - } - - result = (void *)syscall(SYS_mmap2, - start, length, prot, flags, fd, - (off_t) (offset / pagesize)); - if (result != MAP_FAILED || errno != ENOSYS) goto out; - - // We don't have mmap2() after all - don't bother trying it in future - have_mmap2 = false; - } - - if (((off_t)offset) != offset) { - // If we're trying to map a 64-bit offset, fail now since we don't - // have 64-bit mmap() support. - result = MAP_FAILED; - errno = EINVAL; - goto out; - } - -#ifdef __NR_mmap - { - // Fall back to old 32-bit offset mmap() call - // Old syscall interface cannot handle six args, so pass in an array - int32 args[6] = { (int32) start, (int32) length, prot, flags, fd, - (int32)(off_t) offset }; - result = (void *)syscall(SYS_mmap, args); - } -#else - // Some Linux ports like ARM EABI Linux has no mmap, just mmap2. - result = MAP_FAILED; -#endif - - out: - return result; -} - -#define MALLOC_HOOK_HAVE_DO_MMAP64 1 - -#endif // #if defined(__x86_64__) - - -#ifdef MALLOC_HOOK_HAVE_DO_MMAP64 - -// We use do_mmap64 abstraction to put MallocHook::InvokeMmapHook -// calls right into mmap and mmap64, so that the stack frames in the caller's -// stack are at the same offsets for all the calls of memory allocating -// functions. - -// Put all callers of MallocHook::Invoke* in this module into -// malloc_hook section, -// so that MallocHook::GetCallerStackTrace can function accurately: - -// Make sure mmap doesn't get #define'd away by -# undef mmap - -extern "C" { - void* mmap64(void *start, size_t length, int prot, int flags, - int fd, __off64_t offset ) __THROW - ATTRIBUTE_SECTION(malloc_hook); - void* mmap(void *start, size_t length,int prot, int flags, - int fd, off_t offset) __THROW - ATTRIBUTE_SECTION(malloc_hook); - int munmap(void* start, size_t length) __THROW - ATTRIBUTE_SECTION(malloc_hook); - void* mremap(void* old_addr, size_t old_size, size_t new_size, - int flags, ...) __THROW - ATTRIBUTE_SECTION(malloc_hook); - void* sbrk(ptrdiff_t increment) __THROW - ATTRIBUTE_SECTION(malloc_hook); -} - -extern "C" void* mmap64(void *start, size_t length, int prot, int flags, - int fd, __off64_t offset) __THROW { - MallocHook::InvokePreMmapHook(start, length, prot, flags, fd, offset); - void *result; - if (!MallocHook::InvokeMmapReplacement( - start, length, prot, flags, fd, offset, &result)) { - result = do_mmap64(start, length, prot, flags, fd, offset); - } - MallocHook::InvokeMmapHook(result, start, length, prot, flags, fd, offset); - return result; -} - -# if !defined(__USE_FILE_OFFSET64) || !defined(__REDIRECT_NTH) - -extern "C" void* mmap(void *start, size_t length, int prot, int flags, - int fd, off_t offset) __THROW { - MallocHook::InvokePreMmapHook(start, length, prot, flags, fd, offset); - void *result; - if (!MallocHook::InvokeMmapReplacement( - start, length, prot, flags, fd, offset, &result)) { - result = do_mmap64(start, length, prot, flags, fd, - static_cast(offset)); // avoid sign extension - } - MallocHook::InvokeMmapHook(result, start, length, prot, flags, fd, offset); - return result; -} - -# endif // !defined(__USE_FILE_OFFSET64) || !defined(__REDIRECT_NTH) - -extern "C" int munmap(void* start, size_t length) __THROW { - MallocHook::InvokeMunmapHook(start, length); - int result; - if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { - result = sys_munmap(start, length); - } - return result; -} - -extern "C" void* mremap(void* old_addr, size_t old_size, size_t new_size, - int flags, ...) __THROW { - va_list ap; - va_start(ap, flags); - void *new_address = va_arg(ap, void *); - va_end(ap); - void* result = sys_mremap(old_addr, old_size, new_size, flags, new_address); - MallocHook::InvokeMremapHook(result, old_addr, old_size, new_size, flags, - new_address); - return result; -} - -#ifndef __UCLIBC__ -// libc's version: -extern "C" void* __sbrk(ptrdiff_t increment); - -extern "C" void* sbrk(ptrdiff_t increment) __THROW { - MallocHook::InvokePreSbrkHook(increment); - void *result = __sbrk(increment); - MallocHook::InvokeSbrkHook(result, increment); - return result; -} - -#endif - -/*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot, - int flags, int fd, off_t offset) { - void* result; - if (!MallocHook::InvokeMmapReplacement( - start, length, prot, flags, fd, offset, &result)) { - result = do_mmap64(start, length, prot, flags, fd, offset); - } - return result; -} - -/*static*/int MallocHook::UnhookedMUnmap(void *start, size_t length) { - int result; - if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { - result = syscall(SYS_munmap, start, length); - } - return result; -} - -#undef MALLOC_HOOK_HAVE_DO_MMAP64 - -#endif // #ifdef MALLOC_HOOK_HAVE_DO_MMAP64 diff --git a/contrib/libtcmalloc/src/maybe_emergency_malloc.h b/contrib/libtcmalloc/src/maybe_emergency_malloc.h deleted file mode 100644 index 250ecf01a3f..00000000000 --- a/contrib/libtcmalloc/src/maybe_emergency_malloc.h +++ /dev/null @@ -1,55 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2014, gperftools Contributors -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#ifndef MAYBE_EMERGENCY_MALLOC_H -#define MAYBE_EMERGENCY_MALLOC_H - -#include "config.h" - -#ifdef ENABLE_EMERGENCY_MALLOC - -#include "emergency_malloc.h" - -#else - -namespace tcmalloc { - static inline void *EmergencyMalloc(size_t size) {return NULL;} - static inline void EmergencyFree(void *p) {} - static inline void *EmergencyCalloc(size_t n, size_t elem_size) {return NULL;} - static inline void *EmergencyRealloc(void *old_ptr, size_t new_size) {return NULL;} - - static inline bool IsEmergencyPtr(const void *_ptr) { - return false; - } -} - -#endif // ENABLE_EMERGENCY_MALLOC - -#endif diff --git a/contrib/libtcmalloc/src/maybe_threads.cc b/contrib/libtcmalloc/src/maybe_threads.cc deleted file mode 100644 index acfc99a5ae5..00000000000 --- a/contrib/libtcmalloc/src/maybe_threads.cc +++ /dev/null @@ -1,171 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Paul Menage -// -// Some wrappers for pthread functions so that we can be LD_PRELOADed -// against non-pthreads apps. -// -// This module will behave very strangely if some pthreads functions -// exist and others don't. - -#include "config.h" -#include -#include // for memcmp -#include // for __isthreaded on FreeBSD -// We don't actually need strings. But including this header seems to -// stop the compiler trying to short-circuit our pthreads existence -// tests and claiming that the address of a function is always -// non-zero. I have no idea why ... -#include -#include "maybe_threads.h" -#include "base/basictypes.h" -#include "base/logging.h" - -// __THROW is defined in glibc systems. It means, counter-intuitively, -// "This function will never throw an exception." It's an optional -// optimization tool, but we may need to use it to match glibc prototypes. -#ifndef __THROW // I guess we're not on a glibc system -# define __THROW // __THROW is just an optimization, so ok to make it "" -#endif - -// These are the methods we're going to conditionally include. -extern "C" { - int pthread_key_create (pthread_key_t*, void (*)(void*)) - __THROW ATTRIBUTE_WEAK; - int pthread_key_delete (pthread_key_t) - __THROW ATTRIBUTE_WEAK; - void *pthread_getspecific(pthread_key_t) - __THROW ATTRIBUTE_WEAK; - int pthread_setspecific(pthread_key_t, const void*) - __THROW ATTRIBUTE_WEAK; - int pthread_once(pthread_once_t *, void (*)(void)) - ATTRIBUTE_WEAK; - int pthread_atfork(void (*__prepare) (void), - void (*__parent) (void), - void (*__child) (void)) - __THROW ATTRIBUTE_WEAK; -} - -#define MAX_PERTHREAD_VALS 16 -static void *perftools_pthread_specific_vals[MAX_PERTHREAD_VALS]; -static int next_key; - -// NOTE: it's similar to bitcast defined in basic_types.h with -// exception of ignoring sizes mismatch -template -static T2 memcpy_cast(const T1 &input) { - T2 output; - size_t s = sizeof(input); - if (sizeof(output) < s) { - s = sizeof(output); - } - memcpy(&output, &input, s); - return output; -} - -int perftools_pthread_key_create(pthread_key_t *key, - void (*destr_function) (void *)) { - if (pthread_key_create) { - return pthread_key_create(key, destr_function); - } else { - assert(next_key < MAX_PERTHREAD_VALS); - *key = memcpy_cast(next_key++); - return 0; - } -} - -int perftools_pthread_key_delete(pthread_key_t key) { - if (pthread_key_delete) { - return pthread_key_delete(key); - } else { - return 0; - } -} - -void *perftools_pthread_getspecific(pthread_key_t key) { - if (pthread_getspecific) { - return pthread_getspecific(key); - } else { - return perftools_pthread_specific_vals[memcpy_cast(key)]; - } -} - -int perftools_pthread_setspecific(pthread_key_t key, void *val) { - if (pthread_setspecific) { - return pthread_setspecific(key, val); - } else { - perftools_pthread_specific_vals[memcpy_cast(key)] = val; - return 0; - } -} - - -static pthread_once_t pthread_once_init = PTHREAD_ONCE_INIT; -int perftools_pthread_once(pthread_once_t *ctl, - void (*init_routine) (void)) { -#ifdef __FreeBSD__ - // On __FreeBSD__, calling pthread_once on a system that is not - // linked with -pthread is silently a noop. :-( Luckily, we have a - // workaround: FreeBSD exposes __isthreaded in , which is - // set to 1 when the first thread is spawned. So on those systems, - // we can use our own separate pthreads-once mechanism, which is - // used until __isthreaded is 1 (which will never be true if the app - // is not linked with -pthread). - static bool pthread_once_ran_before_threads = false; - if (pthread_once_ran_before_threads) { - return 0; - } - if (!__isthreaded) { - init_routine(); - pthread_once_ran_before_threads = true; - return 0; - } -#endif - if (pthread_once) { - return pthread_once(ctl, init_routine); - } else { - if (memcmp(ctl, &pthread_once_init, sizeof(*ctl)) == 0) { - init_routine(); - ++*(char*)(ctl); // make it so it's no longer equal to init - } - return 0; - } -} - -void perftools_pthread_atfork(void (*before)(), - void (*parent_after)(), - void (*child_after)()) { - if (pthread_atfork) { - int rv = pthread_atfork(before, parent_after, child_after); - CHECK(rv == 0); - } -} diff --git a/contrib/libtcmalloc/src/maybe_threads.h b/contrib/libtcmalloc/src/maybe_threads.h deleted file mode 100644 index c6cfdf7d158..00000000000 --- a/contrib/libtcmalloc/src/maybe_threads.h +++ /dev/null @@ -1,61 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Paul Menage - -//------------------------------------------------------------------- -// Some wrappers for pthread functions so that we can be LD_PRELOADed -// against non-pthreads apps. -//------------------------------------------------------------------- - -#ifndef GOOGLE_MAYBE_THREADS_H_ -#define GOOGLE_MAYBE_THREADS_H_ - -#ifdef HAVE_PTHREAD -#include -#endif - -int perftools_pthread_key_create(pthread_key_t *key, - void (*destr_function) (void *)); -int perftools_pthread_key_delete(pthread_key_t key); -void *perftools_pthread_getspecific(pthread_key_t key); -int perftools_pthread_setspecific(pthread_key_t key, void *val); -int perftools_pthread_once(pthread_once_t *ctl, - void (*init_routine) (void)); - -// Our wrapper for pthread_atfork. Does _nothing_ when there are no -// threads. See static_vars.cc:SetupAtForkLocksHandler for only user -// of this. -void perftools_pthread_atfork(void (*before)(), - void (*parent_after)(), - void (*child_after)()); - -#endif /* GOOGLE_MAYBE_THREADS_H_ */ diff --git a/contrib/libtcmalloc/src/memfs_malloc.cc b/contrib/libtcmalloc/src/memfs_malloc.cc deleted file mode 100644 index 419ef24e43b..00000000000 --- a/contrib/libtcmalloc/src/memfs_malloc.cc +++ /dev/null @@ -1,272 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2007, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Arun Sharma -// -// A tcmalloc system allocator that uses a memory based filesystem such as -// tmpfs or hugetlbfs -// -// Since these only exist on linux, we only register this allocator there. - -#ifdef __linux - -#include "config.h" -#include // for errno, EINVAL -#include // for PRId64 -#include // for PATH_MAX -#include // for size_t, NULL -#ifdef HAVE_STDINT_H -#include // for int64_t, uintptr_t -#endif -#include // for snprintf -#include // for mkstemp -#include // for strerror -#include // for mmap, MAP_FAILED, etc -#include // for fstatfs, statfs -#include // for ftruncate, off_t, unlink -#include // for operator new -#include - -#include -#include "base/basictypes.h" -#include "base/googleinit.h" -#include "base/sysinfo.h" -#include "internal_logging.h" - -// TODO(sanjay): Move the code below into the tcmalloc namespace -using tcmalloc::kLog; -using tcmalloc::kCrash; -using tcmalloc::Log; -using std::string; - -DEFINE_string(memfs_malloc_path, EnvToString("TCMALLOC_MEMFS_MALLOC_PATH", ""), - "Path where hugetlbfs or tmpfs is mounted. The caller is " - "responsible for ensuring that the path is unique and does " - "not conflict with another process"); -DEFINE_int64(memfs_malloc_limit_mb, - EnvToInt("TCMALLOC_MEMFS_LIMIT_MB", 0), - "Limit total allocation size to the " - "specified number of MiB. 0 == no limit."); -DEFINE_bool(memfs_malloc_abort_on_fail, - EnvToBool("TCMALLOC_MEMFS_ABORT_ON_FAIL", false), - "abort() whenever memfs_malloc fails to satisfy an allocation " - "for any reason."); -DEFINE_bool(memfs_malloc_ignore_mmap_fail, - EnvToBool("TCMALLOC_MEMFS_IGNORE_MMAP_FAIL", false), - "Ignore failures from mmap"); -DEFINE_bool(memfs_malloc_map_private, - EnvToBool("TCMALLOC_MEMFS_MAP_PRIVATE", false), - "Use MAP_PRIVATE with mmap"); - -// Hugetlbfs based allocator for tcmalloc -class HugetlbSysAllocator: public SysAllocator { -public: - explicit HugetlbSysAllocator(SysAllocator* fallback) - : failed_(true), // To disable allocator until Initialize() is called. - big_page_size_(0), - hugetlb_fd_(-1), - hugetlb_base_(0), - fallback_(fallback) { - } - - void* Alloc(size_t size, size_t *actual_size, size_t alignment); - bool Initialize(); - - bool failed_; // Whether failed to allocate memory. - -private: - void* AllocInternal(size_t size, size_t *actual_size, size_t alignment); - - int64 big_page_size_; - int hugetlb_fd_; // file descriptor for hugetlb - off_t hugetlb_base_; - - SysAllocator* fallback_; // Default system allocator to fall back to. -}; -static union { - char buf[sizeof(HugetlbSysAllocator)]; - void *ptr; -} hugetlb_space; - -// No locking needed here since we assume that tcmalloc calls -// us with an internal lock held (see tcmalloc/system-alloc.cc). -void* HugetlbSysAllocator::Alloc(size_t size, size_t *actual_size, - size_t alignment) { - if (failed_) { - return fallback_->Alloc(size, actual_size, alignment); - } - - // We don't respond to allocation requests smaller than big_page_size_ unless - // the caller is ok to take more than they asked for. Used by MetaDataAlloc. - if (actual_size == NULL && size < big_page_size_) { - return fallback_->Alloc(size, actual_size, alignment); - } - - // Enforce huge page alignment. Be careful to deal with overflow. - size_t new_alignment = alignment; - if (new_alignment < big_page_size_) new_alignment = big_page_size_; - size_t aligned_size = ((size + new_alignment - 1) / - new_alignment) * new_alignment; - if (aligned_size < size) { - return fallback_->Alloc(size, actual_size, alignment); - } - - void* result = AllocInternal(aligned_size, actual_size, new_alignment); - if (result != NULL) { - return result; - } - Log(kLog, __FILE__, __LINE__, - "HugetlbSysAllocator: (failed, allocated)", failed_, hugetlb_base_); - if (FLAGS_memfs_malloc_abort_on_fail) { - Log(kCrash, __FILE__, __LINE__, - "memfs_malloc_abort_on_fail is set"); - } - return fallback_->Alloc(size, actual_size, alignment); -} - -void* HugetlbSysAllocator::AllocInternal(size_t size, size_t* actual_size, - size_t alignment) { - // Ask for extra memory if alignment > pagesize - size_t extra = 0; - if (alignment > big_page_size_) { - extra = alignment - big_page_size_; - } - - // Test if this allocation would put us over the limit. - off_t limit = FLAGS_memfs_malloc_limit_mb*1024*1024; - if (limit > 0 && hugetlb_base_ + size + extra > limit) { - // Disable the allocator when there's less than one page left. - if (limit - hugetlb_base_ < big_page_size_) { - Log(kLog, __FILE__, __LINE__, "reached memfs_malloc_limit_mb"); - failed_ = true; - } - else { - Log(kLog, __FILE__, __LINE__, - "alloc too large (size, bytes left)", size, limit-hugetlb_base_); - } - return NULL; - } - - // This is not needed for hugetlbfs, but needed for tmpfs. Annoyingly - // hugetlbfs returns EINVAL for ftruncate. - int ret = ftruncate(hugetlb_fd_, hugetlb_base_ + size + extra); - if (ret != 0 && errno != EINVAL) { - Log(kLog, __FILE__, __LINE__, - "ftruncate failed", strerror(errno)); - failed_ = true; - return NULL; - } - - // Note: size + extra does not overflow since: - // size + alignment < (1<(MAP_FAILED)) { - if (!FLAGS_memfs_malloc_ignore_mmap_fail) { - Log(kLog, __FILE__, __LINE__, - "mmap failed (size, error)", size + extra, strerror(errno)); - failed_ = true; - } - return NULL; - } - uintptr_t ptr = reinterpret_cast(result); - - // Adjust the return memory so it is aligned - size_t adjust = 0; - if ((ptr & (alignment - 1)) != 0) { - adjust = alignment - (ptr & (alignment - 1)); - } - ptr += adjust; - hugetlb_base_ += (size + extra); - - if (actual_size) { - *actual_size = size + extra - adjust; - } - - return reinterpret_cast(ptr); -} - -bool HugetlbSysAllocator::Initialize() { - char path[PATH_MAX]; - const int pathlen = FLAGS_memfs_malloc_path.size(); - if (pathlen + 8 > sizeof(path)) { - Log(kCrash, __FILE__, __LINE__, "XX fatal: memfs_malloc_path too long"); - return false; - } - memcpy(path, FLAGS_memfs_malloc_path.data(), pathlen); - memcpy(path + pathlen, ".XXXXXX", 8); // Also copies terminating \0 - - int hugetlb_fd = mkstemp(path); - if (hugetlb_fd == -1) { - Log(kLog, __FILE__, __LINE__, - "warning: unable to create memfs_malloc_path", - path, strerror(errno)); - return false; - } - - // Cleanup memory on process exit - if (unlink(path) == -1) { - Log(kCrash, __FILE__, __LINE__, - "fatal: error unlinking memfs_malloc_path", path, strerror(errno)); - return false; - } - - // Use fstatfs to figure out the default page size for memfs - struct statfs sfs; - if (fstatfs(hugetlb_fd, &sfs) == -1) { - Log(kCrash, __FILE__, __LINE__, - "fatal: error fstatfs of memfs_malloc_path", strerror(errno)); - return false; - } - int64 page_size = sfs.f_bsize; - - hugetlb_fd_ = hugetlb_fd; - big_page_size_ = page_size; - failed_ = false; - return true; -} - -REGISTER_MODULE_INITIALIZER(memfs_malloc, { - if (FLAGS_memfs_malloc_path.length()) { - SysAllocator* alloc = MallocExtension::instance()->GetSystemAllocator(); - HugetlbSysAllocator* hp = - new (hugetlb_space.buf) HugetlbSysAllocator(alloc); - if (hp->Initialize()) { - MallocExtension::instance()->SetSystemAllocator(hp); - } - } -}); - -#endif /* ifdef __linux */ diff --git a/contrib/libtcmalloc/src/memory_region_map.cc b/contrib/libtcmalloc/src/memory_region_map.cc deleted file mode 100644 index 841d6f3cf85..00000000000 --- a/contrib/libtcmalloc/src/memory_region_map.cc +++ /dev/null @@ -1,831 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2006, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Maxim Lifantsev - */ - -// -// Background and key design points of MemoryRegionMap. -// -// MemoryRegionMap is a low-level module with quite atypical requirements that -// result in some degree of non-triviality of the implementation and design. -// -// MemoryRegionMap collects info about *all* memory regions created with -// mmap, munmap, mremap, sbrk. -// They key word above is 'all': all that are happening in a process -// during its lifetime frequently starting even before global object -// constructor execution. -// -// This is needed by the primary client of MemoryRegionMap: -// HeapLeakChecker uses the regions and the associated stack traces -// to figure out what part of the memory is the heap: -// if MemoryRegionMap were to miss some (early) regions, leak checking would -// stop working correctly. -// -// To accomplish the goal of functioning before/during global object -// constructor execution MemoryRegionMap is done as a singleton service -// that relies on own on-demand initialized static constructor-less data, -// and only relies on other low-level modules that can also function properly -// even before global object constructors run. -// -// Accomplishing the goal of collecting data about all mmap, munmap, mremap, -// sbrk occurrences is a more involved: conceptually to do this one needs to -// record some bits of data in particular about any mmap or sbrk call, -// but to do that one needs to allocate memory for that data at some point, -// but all memory allocations in the end themselves come from an mmap -// or sbrk call (that's how the address space of the process grows). -// -// Also note that we need to do all the above recording from -// within an mmap/sbrk hook which is sometimes/frequently is made by a memory -// allocator, including the allocator MemoryRegionMap itself must rely on. -// In the case of heap-checker usage this includes even the very first -// mmap/sbrk call happening in the program: heap-checker gets activated due to -// a link-time installed mmap/sbrk hook and it initializes MemoryRegionMap -// and asks it to record info about this very first call right from that -// very first hook invocation. -// -// MemoryRegionMap is doing its memory allocations via LowLevelAlloc: -// unlike more complex standard memory allocator, LowLevelAlloc cooperates with -// MemoryRegionMap by not holding any of its own locks while it calls mmap -// to get memory, thus we are able to call LowLevelAlloc from -// our mmap/sbrk hooks without causing a deadlock in it. -// For the same reason of deadlock prevention the locking in MemoryRegionMap -// itself is write-recursive which is an exception to Google's mutex usage. -// -// We still need to break the infinite cycle of mmap calling our hook, -// which asks LowLevelAlloc for memory to record this mmap, -// which (sometimes) causes mmap, which calls our hook, and so on. -// We do this as follows: on a recursive call of MemoryRegionMap's -// mmap/sbrk/mremap hook we record the data about the allocation in a -// static fixed-sized stack (saved_regions and saved_buckets), when the -// recursion unwinds but before returning from the outer hook call we unwind -// this stack and move the data from saved_regions and saved_buckets to its -// permanent place in the RegionSet and "bucket_table" respectively, -// which can cause more allocations and mmap-s and recursion and unwinding, -// but the whole process ends eventually due to the fact that for the small -// allocations we are doing LowLevelAlloc reuses one mmap call and parcels out -// the memory it created to satisfy several of our allocation requests. -// - -// ========================================================================= // - -#include - -#ifdef HAVE_UNISTD_H -#include -#endif -#ifdef HAVE_INTTYPES_H -#include -#endif -#ifdef HAVE_MMAP -#include -#elif !defined(MAP_FAILED) -#define MAP_FAILED -1 // the only thing we need from mman.h -#endif -#ifdef HAVE_PTHREAD -#include // for pthread_t, pthread_self() -#endif -#include - -#include -#include - -#include "memory_region_map.h" - -#include "base/googleinit.h" -#include "base/logging.h" -#include "base/low_level_alloc.h" -#include "malloc_hook-inl.h" - -#include -#include - -// MREMAP_FIXED is a linux extension. How it's used in this file, -// setting it to 0 is equivalent to saying, "This feature isn't -// supported", which is right. -#ifndef MREMAP_FIXED -# define MREMAP_FIXED 0 -#endif - -using std::max; - -// ========================================================================= // - -int MemoryRegionMap::client_count_ = 0; -int MemoryRegionMap::max_stack_depth_ = 0; -MemoryRegionMap::RegionSet* MemoryRegionMap::regions_ = NULL; -LowLevelAlloc::Arena* MemoryRegionMap::arena_ = NULL; -SpinLock MemoryRegionMap::lock_(SpinLock::LINKER_INITIALIZED); -SpinLock MemoryRegionMap::owner_lock_( // ACQUIRED_AFTER(lock_) - SpinLock::LINKER_INITIALIZED); -int MemoryRegionMap::recursion_count_ = 0; // GUARDED_BY(owner_lock_) -pthread_t MemoryRegionMap::lock_owner_tid_; // GUARDED_BY(owner_lock_) -int64 MemoryRegionMap::map_size_ = 0; -int64 MemoryRegionMap::unmap_size_ = 0; -HeapProfileBucket** MemoryRegionMap::bucket_table_ = NULL; // GUARDED_BY(lock_) -int MemoryRegionMap::num_buckets_ = 0; // GUARDED_BY(lock_) -int MemoryRegionMap::saved_buckets_count_ = 0; // GUARDED_BY(lock_) -HeapProfileBucket MemoryRegionMap::saved_buckets_[20]; // GUARDED_BY(lock_) - -// GUARDED_BY(lock_) -const void* MemoryRegionMap::saved_buckets_keys_[20][kMaxStackDepth]; - -// ========================================================================= // - -// Simple hook into execution of global object constructors, -// so that we do not call pthread_self() when it does not yet work. -static bool libpthread_initialized = false; -REGISTER_MODULE_INITIALIZER(libpthread_initialized_setter, - libpthread_initialized = true); - -static inline bool current_thread_is(pthread_t should_be) { - // Before main() runs, there's only one thread, so we're always that thread - if (!libpthread_initialized) return true; - // this starts working only sometime well into global constructor execution: - return pthread_equal(pthread_self(), should_be); -} - -// ========================================================================= // - -// Constructor-less place-holder to store a RegionSet in. -union MemoryRegionMap::RegionSetRep { - char rep[sizeof(RegionSet)]; - void* align_it; // do not need a better alignment for 'rep' than this - RegionSet* region_set() { return reinterpret_cast(rep); } -}; - -// The bytes where MemoryRegionMap::regions_ will point to. -// We use RegionSetRep with noop c-tor so that global construction -// does not interfere. -static MemoryRegionMap::RegionSetRep regions_rep; - -// ========================================================================= // - -// Has InsertRegionLocked been called recursively -// (or rather should we *not* use regions_ to record a hooked mmap). -static bool recursive_insert = false; - -void MemoryRegionMap::Init(int max_stack_depth, bool use_buckets) { - RAW_VLOG(10, "MemoryRegionMap Init"); - RAW_CHECK(max_stack_depth >= 0, ""); - // Make sure we don't overflow the memory in region stacks: - RAW_CHECK(max_stack_depth <= kMaxStackDepth, - "need to increase kMaxStackDepth?"); - Lock(); - client_count_ += 1; - max_stack_depth_ = max(max_stack_depth_, max_stack_depth); - if (client_count_ > 1) { - // not first client: already did initialization-proper - Unlock(); - RAW_VLOG(10, "MemoryRegionMap Init increment done"); - return; - } - // Set our hooks and make sure they were installed: - RAW_CHECK(MallocHook::AddMmapHook(&MmapHook), ""); - RAW_CHECK(MallocHook::AddMremapHook(&MremapHook), ""); - RAW_CHECK(MallocHook::AddSbrkHook(&SbrkHook), ""); - RAW_CHECK(MallocHook::AddMunmapHook(&MunmapHook), ""); - // We need to set recursive_insert since the NewArena call itself - // will already do some allocations with mmap which our hooks will catch - // recursive_insert allows us to buffer info about these mmap calls. - // Note that Init() can be (and is) sometimes called - // already from within an mmap/sbrk hook. - recursive_insert = true; - arena_ = LowLevelAlloc::NewArena(0, LowLevelAlloc::DefaultArena()); - recursive_insert = false; - HandleSavedRegionsLocked(&InsertRegionLocked); // flush the buffered ones - // Can't instead use HandleSavedRegionsLocked(&DoInsertRegionLocked) before - // recursive_insert = false; as InsertRegionLocked will also construct - // regions_ on demand for us. - if (use_buckets) { - const int table_bytes = kHashTableSize * sizeof(*bucket_table_); - recursive_insert = true; - bucket_table_ = static_cast( - MyAllocator::Allocate(table_bytes)); - recursive_insert = false; - memset(bucket_table_, 0, table_bytes); - num_buckets_ = 0; - } - Unlock(); - RAW_VLOG(10, "MemoryRegionMap Init done"); -} - -bool MemoryRegionMap::Shutdown() { - RAW_VLOG(10, "MemoryRegionMap Shutdown"); - Lock(); - RAW_CHECK(client_count_ > 0, ""); - client_count_ -= 1; - if (client_count_ != 0) { // not last client; need not really shutdown - Unlock(); - RAW_VLOG(10, "MemoryRegionMap Shutdown decrement done"); - return true; - } - if (bucket_table_ != NULL) { - for (int i = 0; i < kHashTableSize; i++) { - for (HeapProfileBucket* curr = bucket_table_[i]; curr != 0; /**/) { - HeapProfileBucket* bucket = curr; - curr = curr->next; - MyAllocator::Free(bucket->stack, 0); - MyAllocator::Free(bucket, 0); - } - } - MyAllocator::Free(bucket_table_, 0); - num_buckets_ = 0; - bucket_table_ = NULL; - } - RAW_CHECK(MallocHook::RemoveMmapHook(&MmapHook), ""); - RAW_CHECK(MallocHook::RemoveMremapHook(&MremapHook), ""); - RAW_CHECK(MallocHook::RemoveSbrkHook(&SbrkHook), ""); - RAW_CHECK(MallocHook::RemoveMunmapHook(&MunmapHook), ""); - if (regions_) regions_->~RegionSet(); - regions_ = NULL; - bool deleted_arena = LowLevelAlloc::DeleteArena(arena_); - if (deleted_arena) { - arena_ = 0; - } else { - RAW_LOG(WARNING, "Can't delete LowLevelAlloc arena: it's being used"); - } - Unlock(); - RAW_VLOG(10, "MemoryRegionMap Shutdown done"); - return deleted_arena; -} - -bool MemoryRegionMap::IsRecordingLocked() { - RAW_CHECK(LockIsHeld(), "should be held (by this thread)"); - return client_count_ > 0; -} - -// Invariants (once libpthread_initialized is true): -// * While lock_ is not held, recursion_count_ is 0 (and -// lock_owner_tid_ is the previous owner, but we don't rely on -// that). -// * recursion_count_ and lock_owner_tid_ are only written while -// both lock_ and owner_lock_ are held. They may be read under -// just owner_lock_. -// * At entry and exit of Lock() and Unlock(), the current thread -// owns lock_ iff pthread_equal(lock_owner_tid_, pthread_self()) -// && recursion_count_ > 0. -void MemoryRegionMap::Lock() { - { - SpinLockHolder l(&owner_lock_); - if (recursion_count_ > 0 && current_thread_is(lock_owner_tid_)) { - RAW_CHECK(lock_.IsHeld(), "Invariants violated"); - recursion_count_++; - RAW_CHECK(recursion_count_ <= 5, - "recursive lock nesting unexpectedly deep"); - return; - } - } - lock_.Lock(); - { - SpinLockHolder l(&owner_lock_); - RAW_CHECK(recursion_count_ == 0, - "Last Unlock didn't reset recursion_count_"); - if (libpthread_initialized) - lock_owner_tid_ = pthread_self(); - recursion_count_ = 1; - } -} - -void MemoryRegionMap::Unlock() { - SpinLockHolder l(&owner_lock_); - RAW_CHECK(recursion_count_ > 0, "unlock when not held"); - RAW_CHECK(lock_.IsHeld(), - "unlock when not held, and recursion_count_ is wrong"); - RAW_CHECK(current_thread_is(lock_owner_tid_), "unlock by non-holder"); - recursion_count_--; - if (recursion_count_ == 0) { - lock_.Unlock(); - } -} - -bool MemoryRegionMap::LockIsHeld() { - SpinLockHolder l(&owner_lock_); - return lock_.IsHeld() && current_thread_is(lock_owner_tid_); -} - -const MemoryRegionMap::Region* -MemoryRegionMap::DoFindRegionLocked(uintptr_t addr) { - RAW_CHECK(LockIsHeld(), "should be held (by this thread)"); - if (regions_ != NULL) { - Region sample; - sample.SetRegionSetKey(addr); - RegionSet::iterator region = regions_->lower_bound(sample); - if (region != regions_->end()) { - RAW_CHECK(addr <= region->end_addr, ""); - if (region->start_addr <= addr && addr < region->end_addr) { - return &(*region); - } - } - } - return NULL; -} - -bool MemoryRegionMap::FindRegion(uintptr_t addr, Region* result) { - Lock(); - const Region* region = DoFindRegionLocked(addr); - if (region != NULL) *result = *region; // create it as an independent copy - Unlock(); - return region != NULL; -} - -bool MemoryRegionMap::FindAndMarkStackRegion(uintptr_t stack_top, - Region* result) { - Lock(); - const Region* region = DoFindRegionLocked(stack_top); - if (region != NULL) { - RAW_VLOG(10, "Stack at %p is inside region %p..%p", - reinterpret_cast(stack_top), - reinterpret_cast(region->start_addr), - reinterpret_cast(region->end_addr)); - const_cast(region)->set_is_stack(); // now we know - // cast is safe (set_is_stack does not change the set ordering key) - *result = *region; // create *result as an independent copy - } - Unlock(); - return region != NULL; -} - -HeapProfileBucket* MemoryRegionMap::GetBucket(int depth, - const void* const key[]) { - RAW_CHECK(LockIsHeld(), "should be held (by this thread)"); - // Make hash-value - uintptr_t hash = 0; - for (int i = 0; i < depth; i++) { - hash += reinterpret_cast(key[i]); - hash += hash << 10; - hash ^= hash >> 6; - } - hash += hash << 3; - hash ^= hash >> 11; - - // Lookup stack trace in table - unsigned int hash_index = (static_cast(hash)) % kHashTableSize; - for (HeapProfileBucket* bucket = bucket_table_[hash_index]; - bucket != 0; - bucket = bucket->next) { - if ((bucket->hash == hash) && (bucket->depth == depth) && - std::equal(key, key + depth, bucket->stack)) { - return bucket; - } - } - - // Create new bucket - const size_t key_size = sizeof(key[0]) * depth; - HeapProfileBucket* bucket; - if (recursive_insert) { // recursion: save in saved_buckets_ - const void** key_copy = saved_buckets_keys_[saved_buckets_count_]; - std::copy(key, key + depth, key_copy); - bucket = &saved_buckets_[saved_buckets_count_]; - memset(bucket, 0, sizeof(*bucket)); - ++saved_buckets_count_; - bucket->stack = key_copy; - bucket->next = NULL; - } else { - recursive_insert = true; - const void** key_copy = static_cast( - MyAllocator::Allocate(key_size)); - recursive_insert = false; - std::copy(key, key + depth, key_copy); - recursive_insert = true; - bucket = static_cast( - MyAllocator::Allocate(sizeof(HeapProfileBucket))); - recursive_insert = false; - memset(bucket, 0, sizeof(*bucket)); - bucket->stack = key_copy; - bucket->next = bucket_table_[hash_index]; - } - bucket->hash = hash; - bucket->depth = depth; - bucket_table_[hash_index] = bucket; - ++num_buckets_; - return bucket; -} - -MemoryRegionMap::RegionIterator MemoryRegionMap::BeginRegionLocked() { - RAW_CHECK(LockIsHeld(), "should be held (by this thread)"); - RAW_CHECK(regions_ != NULL, ""); - return regions_->begin(); -} - -MemoryRegionMap::RegionIterator MemoryRegionMap::EndRegionLocked() { - RAW_CHECK(LockIsHeld(), "should be held (by this thread)"); - RAW_CHECK(regions_ != NULL, ""); - return regions_->end(); -} - -inline void MemoryRegionMap::DoInsertRegionLocked(const Region& region) { - RAW_VLOG(12, "Inserting region %p..%p from %p", - reinterpret_cast(region.start_addr), - reinterpret_cast(region.end_addr), - reinterpret_cast(region.caller())); - RegionSet::const_iterator i = regions_->lower_bound(region); - if (i != regions_->end() && i->start_addr <= region.start_addr) { - RAW_DCHECK(region.end_addr <= i->end_addr, ""); // lower_bound ensures this - return; // 'region' is a subset of an already recorded region; do nothing - // We can be stricter and allow this only when *i has been created via - // an mmap with MAP_NORESERVE flag set. - } - if (DEBUG_MODE) { - RAW_CHECK(i == regions_->end() || !region.Overlaps(*i), - "Wow, overlapping memory regions"); - Region sample; - sample.SetRegionSetKey(region.start_addr); - i = regions_->lower_bound(sample); - RAW_CHECK(i == regions_->end() || !region.Overlaps(*i), - "Wow, overlapping memory regions"); - } - region.AssertIsConsistent(); // just making sure - // This inserts and allocates permanent storage for region - // and its call stack data: it's safe to do it now: - regions_->insert(region); - RAW_VLOG(12, "Inserted region %p..%p :", - reinterpret_cast(region.start_addr), - reinterpret_cast(region.end_addr)); - if (VLOG_IS_ON(12)) LogAllLocked(); -} - -// These variables are local to MemoryRegionMap::InsertRegionLocked() -// and MemoryRegionMap::HandleSavedRegionsLocked() -// and are file-level to ensure that they are initialized at load time. - -// Number of unprocessed region inserts. -static int saved_regions_count = 0; - -// Unprocessed inserts (must be big enough to hold all allocations that can -// be caused by a InsertRegionLocked call). -// Region has no constructor, so that c-tor execution does not interfere -// with the any-time use of the static memory behind saved_regions. -static MemoryRegionMap::Region saved_regions[20]; - -inline void MemoryRegionMap::HandleSavedRegionsLocked( - void (*insert_func)(const Region& region)) { - while (saved_regions_count > 0) { - // Making a local-var copy of the region argument to insert_func - // including its stack (w/o doing any memory allocations) is important: - // in many cases the memory in saved_regions - // will get written-to during the (*insert_func)(r) call below. - Region r = saved_regions[--saved_regions_count]; - (*insert_func)(r); - } -} - -void MemoryRegionMap::RestoreSavedBucketsLocked() { - RAW_CHECK(LockIsHeld(), "should be held (by this thread)"); - while (saved_buckets_count_ > 0) { - HeapProfileBucket bucket = saved_buckets_[--saved_buckets_count_]; - unsigned int hash_index = - static_cast(bucket.hash) % kHashTableSize; - bool is_found = false; - for (HeapProfileBucket* curr = bucket_table_[hash_index]; - curr != 0; - curr = curr->next) { - if ((curr->hash == bucket.hash) && (curr->depth == bucket.depth) && - std::equal(bucket.stack, bucket.stack + bucket.depth, curr->stack)) { - curr->allocs += bucket.allocs; - curr->alloc_size += bucket.alloc_size; - curr->frees += bucket.frees; - curr->free_size += bucket.free_size; - is_found = true; - break; - } - } - if (is_found) continue; - - const size_t key_size = sizeof(bucket.stack[0]) * bucket.depth; - const void** key_copy = static_cast( - MyAllocator::Allocate(key_size)); - std::copy(bucket.stack, bucket.stack + bucket.depth, key_copy); - HeapProfileBucket* new_bucket = static_cast( - MyAllocator::Allocate(sizeof(HeapProfileBucket))); - memset(new_bucket, 0, sizeof(*new_bucket)); - new_bucket->hash = bucket.hash; - new_bucket->depth = bucket.depth; - new_bucket->stack = key_copy; - new_bucket->next = bucket_table_[hash_index]; - bucket_table_[hash_index] = new_bucket; - ++num_buckets_; - } -} - -inline void MemoryRegionMap::InsertRegionLocked(const Region& region) { - RAW_CHECK(LockIsHeld(), "should be held (by this thread)"); - // We can be called recursively, because RegionSet constructor - // and DoInsertRegionLocked() (called below) can call the allocator. - // recursive_insert tells us if that's the case. When this happens, - // region insertion information is recorded in saved_regions[], - // and taken into account when the recursion unwinds. - // Do the insert: - if (recursive_insert) { // recursion: save in saved_regions - RAW_VLOG(12, "Saving recursive insert of region %p..%p from %p", - reinterpret_cast(region.start_addr), - reinterpret_cast(region.end_addr), - reinterpret_cast(region.caller())); - RAW_CHECK(saved_regions_count < arraysize(saved_regions), ""); - // Copy 'region' to saved_regions[saved_regions_count] - // together with the contents of its call_stack, - // then increment saved_regions_count. - saved_regions[saved_regions_count++] = region; - } else { // not a recusrive call - if (regions_ == NULL) { // init regions_ - RAW_VLOG(12, "Initializing region set"); - regions_ = regions_rep.region_set(); - recursive_insert = true; - new(regions_) RegionSet(); - HandleSavedRegionsLocked(&DoInsertRegionLocked); - recursive_insert = false; - } - recursive_insert = true; - // Do the actual insertion work to put new regions into regions_: - DoInsertRegionLocked(region); - HandleSavedRegionsLocked(&DoInsertRegionLocked); - recursive_insert = false; - } -} - -// We strip out different number of stack frames in debug mode -// because less inlining happens in that case -#ifdef NDEBUG -static const int kStripFrames = 1; -#else -static const int kStripFrames = 3; -#endif - -void MemoryRegionMap::RecordRegionAddition(const void* start, size_t size) { - // Record start/end info about this memory acquisition call in a new region: - Region region; - region.Create(start, size); - // First get the call stack info into the local varible 'region': - int depth = 0; - // NOTE: libunwind also does mmap and very much likely while holding - // it's own lock(s). So some threads may first take libunwind lock, - // and then take region map lock (necessary to record mmap done from - // inside libunwind). On the other hand other thread(s) may do - // normal mmap. Which would call this method to record it. Which - // would then proceed with installing that record to region map - // while holding region map lock. That may cause mmap from our own - // internal allocators, so attempt to unwind in this case may cause - // reverse order of taking libuwind and region map locks. Which is - // obvious deadlock. - // - // Thankfully, we can easily detect if we're holding region map lock - // and avoid recording backtrace in this (rare and largely - // irrelevant) case. By doing this we "declare" that thread needing - // both locks must take region map lock last. In other words we do - // not allow taking libuwind lock when we already have region map - // lock. Note, this is generally impossible when somebody tries to - // mix cpu profiling and heap checking/profiling, because cpu - // profiler grabs backtraces at arbitrary places. But at least such - // combination is rarer and less relevant. - if (max_stack_depth_ > 0 && !LockIsHeld()) { - depth = MallocHook::GetCallerStackTrace(const_cast(region.call_stack), - max_stack_depth_, kStripFrames + 1); - } - region.set_call_stack_depth(depth); // record stack info fully - RAW_VLOG(10, "New global region %p..%p from %p", - reinterpret_cast(region.start_addr), - reinterpret_cast(region.end_addr), - reinterpret_cast(region.caller())); - // Note: none of the above allocates memory. - Lock(); // recursively lock - map_size_ += size; - InsertRegionLocked(region); - // This will (eventually) allocate storage for and copy over the stack data - // from region.call_stack_data_ that is pointed by region.call_stack(). - if (bucket_table_ != NULL) { - HeapProfileBucket* b = GetBucket(depth, region.call_stack); - ++b->allocs; - b->alloc_size += size; - if (!recursive_insert) { - recursive_insert = true; - RestoreSavedBucketsLocked(); - recursive_insert = false; - } - } - Unlock(); -} - -void MemoryRegionMap::RecordRegionRemoval(const void* start, size_t size) { - Lock(); - if (recursive_insert) { - // First remove the removed region from saved_regions, if it's - // there, to prevent overrunning saved_regions in recursive - // map/unmap call sequences, and also from later inserting regions - // which have already been unmapped. - uintptr_t start_addr = reinterpret_cast(start); - uintptr_t end_addr = start_addr + size; - int put_pos = 0; - int old_count = saved_regions_count; - for (int i = 0; i < old_count; ++i, ++put_pos) { - Region& r = saved_regions[i]; - if (r.start_addr == start_addr && r.end_addr == end_addr) { - // An exact match, so it's safe to remove. - RecordRegionRemovalInBucket(r.call_stack_depth, r.call_stack, size); - --saved_regions_count; - --put_pos; - RAW_VLOG(10, ("Insta-Removing saved region %p..%p; " - "now have %d saved regions"), - reinterpret_cast(start_addr), - reinterpret_cast(end_addr), - saved_regions_count); - } else { - if (put_pos < i) { - saved_regions[put_pos] = saved_regions[i]; - } - } - } - } - if (regions_ == NULL) { // We must have just unset the hooks, - // but this thread was already inside the hook. - Unlock(); - return; - } - if (!recursive_insert) { - HandleSavedRegionsLocked(&InsertRegionLocked); - } - // first handle adding saved regions if any - uintptr_t start_addr = reinterpret_cast(start); - uintptr_t end_addr = start_addr + size; - // subtract start_addr, end_addr from all the regions - RAW_VLOG(10, "Removing global region %p..%p; have %" PRIuS " regions", - reinterpret_cast(start_addr), - reinterpret_cast(end_addr), - regions_->size()); - Region sample; - sample.SetRegionSetKey(start_addr); - // Only iterate over the regions that might overlap start_addr..end_addr: - for (RegionSet::iterator region = regions_->lower_bound(sample); - region != regions_->end() && region->start_addr < end_addr; - /*noop*/) { - RAW_VLOG(13, "Looking at region %p..%p", - reinterpret_cast(region->start_addr), - reinterpret_cast(region->end_addr)); - if (start_addr <= region->start_addr && - region->end_addr <= end_addr) { // full deletion - RAW_VLOG(12, "Deleting region %p..%p", - reinterpret_cast(region->start_addr), - reinterpret_cast(region->end_addr)); - RecordRegionRemovalInBucket(region->call_stack_depth, region->call_stack, - region->end_addr - region->start_addr); - RegionSet::iterator d = region; - ++region; - regions_->erase(d); - continue; - } else if (region->start_addr < start_addr && - end_addr < region->end_addr) { // cutting-out split - RAW_VLOG(12, "Splitting region %p..%p in two", - reinterpret_cast(region->start_addr), - reinterpret_cast(region->end_addr)); - RecordRegionRemovalInBucket(region->call_stack_depth, region->call_stack, - end_addr - start_addr); - // Make another region for the start portion: - // The new region has to be the start portion because we can't - // just modify region->end_addr as it's the sorting key. - Region r = *region; - r.set_end_addr(start_addr); - InsertRegionLocked(r); - // cut *region from start: - const_cast(*region).set_start_addr(end_addr); - } else if (end_addr > region->start_addr && - start_addr <= region->start_addr) { // cut from start - RAW_VLOG(12, "Start-chopping region %p..%p", - reinterpret_cast(region->start_addr), - reinterpret_cast(region->end_addr)); - RecordRegionRemovalInBucket(region->call_stack_depth, region->call_stack, - end_addr - region->start_addr); - const_cast(*region).set_start_addr(end_addr); - } else if (start_addr > region->start_addr && - start_addr < region->end_addr) { // cut from end - RAW_VLOG(12, "End-chopping region %p..%p", - reinterpret_cast(region->start_addr), - reinterpret_cast(region->end_addr)); - RecordRegionRemovalInBucket(region->call_stack_depth, region->call_stack, - region->end_addr - start_addr); - // Can't just modify region->end_addr (it's the sorting key): - Region r = *region; - r.set_end_addr(start_addr); - RegionSet::iterator d = region; - ++region; - // It's safe to erase before inserting since r is independent of *d: - // r contains an own copy of the call stack: - regions_->erase(d); - InsertRegionLocked(r); - continue; - } - ++region; - } - RAW_VLOG(12, "Removed region %p..%p; have %" PRIuS " regions", - reinterpret_cast(start_addr), - reinterpret_cast(end_addr), - regions_->size()); - if (VLOG_IS_ON(12)) LogAllLocked(); - unmap_size_ += size; - Unlock(); -} - -void MemoryRegionMap::RecordRegionRemovalInBucket(int depth, - const void* const stack[], - size_t size) { - RAW_CHECK(LockIsHeld(), "should be held (by this thread)"); - if (bucket_table_ == NULL) return; - HeapProfileBucket* b = GetBucket(depth, stack); - ++b->frees; - b->free_size += size; -} - -void MemoryRegionMap::MmapHook(const void* result, - const void* start, size_t size, - int prot, int flags, - int fd, off_t offset) { - // TODO(maxim): replace all 0x%" PRIxS " by %p when RAW_VLOG uses a safe - // snprintf reimplementation that does not malloc to pretty-print NULL - RAW_VLOG(10, "MMap = 0x%" PRIxPTR " of %" PRIuS " at %" PRIu64 " " - "prot %d flags %d fd %d offs %" PRId64, - reinterpret_cast(result), size, - reinterpret_cast(start), prot, flags, fd, - static_cast(offset)); - if (result != reinterpret_cast(MAP_FAILED) && size != 0) { - RecordRegionAddition(result, size); - } -} - -void MemoryRegionMap::MunmapHook(const void* ptr, size_t size) { - RAW_VLOG(10, "MUnmap of %p %" PRIuS "", ptr, size); - if (size != 0) { - RecordRegionRemoval(ptr, size); - } -} - -void MemoryRegionMap::MremapHook(const void* result, - const void* old_addr, size_t old_size, - size_t new_size, int flags, - const void* new_addr) { - RAW_VLOG(10, "MRemap = 0x%" PRIxPTR " of 0x%" PRIxPTR " %" PRIuS " " - "to %" PRIuS " flags %d new_addr=0x%" PRIxPTR, - (uintptr_t)result, (uintptr_t)old_addr, - old_size, new_size, flags, - flags & MREMAP_FIXED ? (uintptr_t)new_addr : 0); - if (result != reinterpret_cast(-1)) { - RecordRegionRemoval(old_addr, old_size); - RecordRegionAddition(result, new_size); - } -} - -void MemoryRegionMap::SbrkHook(const void* result, ptrdiff_t increment) { - RAW_VLOG(10, "Sbrk = 0x%" PRIxPTR " of %" PRIdS "", (uintptr_t)result, increment); - if (result != reinterpret_cast(-1)) { - if (increment > 0) { - void* new_end = sbrk(0); - RecordRegionAddition(result, reinterpret_cast(new_end) - - reinterpret_cast(result)); - } else if (increment < 0) { - void* new_end = sbrk(0); - RecordRegionRemoval(new_end, reinterpret_cast(result) - - reinterpret_cast(new_end)); - } - } -} - -void MemoryRegionMap::LogAllLocked() { - RAW_CHECK(LockIsHeld(), "should be held (by this thread)"); - RAW_LOG(INFO, "List of regions:"); - uintptr_t previous = 0; - for (RegionSet::const_iterator r = regions_->begin(); - r != regions_->end(); ++r) { - RAW_LOG(INFO, "Memory region 0x%" PRIxPTR "..0x%" PRIxPTR " " - "from 0x%" PRIxPTR " stack=%d", - r->start_addr, r->end_addr, r->caller(), r->is_stack); - RAW_CHECK(previous < r->end_addr, "wow, we messed up the set order"); - // this must be caused by uncontrolled recursive operations on regions_ - previous = r->end_addr; - } - RAW_LOG(INFO, "End of regions list"); -} diff --git a/contrib/libtcmalloc/src/memory_region_map.h b/contrib/libtcmalloc/src/memory_region_map.h deleted file mode 100644 index ec388e1cc54..00000000000 --- a/contrib/libtcmalloc/src/memory_region_map.h +++ /dev/null @@ -1,413 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -/* Copyright (c) 2006, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Maxim Lifantsev - */ - -#ifndef BASE_MEMORY_REGION_MAP_H_ -#define BASE_MEMORY_REGION_MAP_H_ - -#include - -#ifdef HAVE_PTHREAD -#include -#endif -#include -#include -#include "base/stl_allocator.h" -#include "base/spinlock.h" -#include "base/thread_annotations.h" -#include "base/low_level_alloc.h" -#include "heap-profile-stats.h" - -// TODO(maxim): add a unittest: -// execute a bunch of mmaps and compare memory map what strace logs -// execute a bunch of mmap/munmup and compare memory map with -// own accounting of what those mmaps generated - -// Thread-safe class to collect and query the map of all memory regions -// in a process that have been created with mmap, munmap, mremap, sbrk. -// For each memory region, we keep track of (and provide to users) -// the stack trace that allocated that memory region. -// The recorded stack trace depth is bounded by -// a user-supplied max_stack_depth parameter of Init(). -// After initialization with Init() -// (which can happened even before global object constructor execution) -// we collect the map by installing and monitoring MallocHook-s -// to mmap, munmap, mremap, sbrk. -// At any time one can query this map via provided interface. -// For more details on the design of MemoryRegionMap -// see the comment at the top of our .cc file. -class MemoryRegionMap { - private: - // Max call stack recording depth supported by Init(). Set it to be - // high enough for all our clients. Note: we do not define storage - // for this (doing that requires special handling in windows), so - // don't take the address of it! - static const int kMaxStackDepth = 32; - - // Size of the hash table of buckets. A structure of the bucket table is - // described in heap-profile-stats.h. - static const int kHashTableSize = 179999; - - public: - // interface ================================================================ - - // Every client of MemoryRegionMap must call Init() before first use, - // and Shutdown() after last use. This allows us to reference count - // this (singleton) class properly. MemoryRegionMap assumes it's the - // only client of MallocHooks, so a client can only register other - // MallocHooks after calling Init() and must unregister them before - // calling Shutdown(). - - // Initialize this module to record memory allocation stack traces. - // Stack traces that have more than "max_stack_depth" frames - // are automatically shrunk to "max_stack_depth" when they are recorded. - // Init() can be called more than once w/o harm, largest max_stack_depth - // will be the effective one. - // When "use_buckets" is true, then counts of mmap and munmap sizes will be - // recorded with each stack trace. If Init() is called more than once, then - // counting will be effective after any call contained "use_buckets" of true. - // It will install mmap, munmap, mremap, sbrk hooks - // and initialize arena_ and our hook and locks, hence one can use - // MemoryRegionMap::Lock()/Unlock() to manage the locks. - // Uses Lock/Unlock inside. - static void Init(int max_stack_depth, bool use_buckets); - - // Try to shutdown this module undoing what Init() did. - // Returns true iff could do full shutdown (or it was not attempted). - // Full shutdown is attempted when the number of Shutdown() calls equals - // the number of Init() calls. - static bool Shutdown(); - - // Return true if MemoryRegionMap is initialized and recording, i.e. when - // then number of Init() calls are more than the number of Shutdown() calls. - static bool IsRecordingLocked(); - - // Locks to protect our internal data structures. - // These also protect use of arena_ if our Init() has been done. - // The lock is recursive. - static void Lock() EXCLUSIVE_LOCK_FUNCTION(lock_); - static void Unlock() UNLOCK_FUNCTION(lock_); - - // Returns true when the lock is held by this thread (for use in RAW_CHECK-s). - static bool LockIsHeld(); - - // Locker object that acquires the MemoryRegionMap::Lock - // for the duration of its lifetime (a C++ scope). - class LockHolder { - public: - LockHolder() { Lock(); } - ~LockHolder() { Unlock(); } - private: - DISALLOW_COPY_AND_ASSIGN(LockHolder); - }; - - // A memory region that we know about through malloc_hook-s. - // This is essentially an interface through which MemoryRegionMap - // exports the collected data to its clients. Thread-compatible. - struct Region { - uintptr_t start_addr; // region start address - uintptr_t end_addr; // region end address - int call_stack_depth; // number of caller stack frames that we saved - const void* call_stack[kMaxStackDepth]; // caller address stack array - // filled to call_stack_depth size - bool is_stack; // does this region contain a thread's stack: - // a user of MemoryRegionMap supplies this info - - // Convenience accessor for call_stack[0], - // i.e. (the program counter of) the immediate caller - // of this region's allocation function, - // but it also returns NULL when call_stack_depth is 0, - // i.e whe we weren't able to get the call stack. - // This usually happens in recursive calls, when the stack-unwinder - // calls mmap() which in turn calls the stack-unwinder. - uintptr_t caller() const { - return reinterpret_cast(call_stack_depth >= 1 - ? call_stack[0] : NULL); - } - - // Return true iff this region overlaps region x. - bool Overlaps(const Region& x) const { - return start_addr < x.end_addr && end_addr > x.start_addr; - } - - private: // helpers for MemoryRegionMap - friend class MemoryRegionMap; - - // The ways we create Region-s: - void Create(const void* start, size_t size) { - start_addr = reinterpret_cast(start); - end_addr = start_addr + size; - is_stack = false; // not a stack till marked such - call_stack_depth = 0; - AssertIsConsistent(); - } - void set_call_stack_depth(int depth) { - RAW_DCHECK(call_stack_depth == 0, ""); // only one such set is allowed - call_stack_depth = depth; - AssertIsConsistent(); - } - - // The ways we modify Region-s: - void set_is_stack() { is_stack = true; } - void set_start_addr(uintptr_t addr) { - start_addr = addr; - AssertIsConsistent(); - } - void set_end_addr(uintptr_t addr) { - end_addr = addr; - AssertIsConsistent(); - } - - // Verifies that *this contains consistent data, crashes if not the case. - void AssertIsConsistent() const { - RAW_DCHECK(start_addr < end_addr, ""); - RAW_DCHECK(call_stack_depth >= 0 && - call_stack_depth <= kMaxStackDepth, ""); - } - - // Post-default construction helper to make a Region suitable - // for searching in RegionSet regions_. - void SetRegionSetKey(uintptr_t addr) { - // make sure *this has no usable data: - if (DEBUG_MODE) memset(this, 0xFF, sizeof(*this)); - end_addr = addr; - } - - // Note: call_stack[kMaxStackDepth] as a member lets us make Region - // a simple self-contained struct with correctly behaving bit-vise copying. - // This simplifies the code of this module but wastes some memory: - // in most-often use case of this module (leak checking) - // only one call_stack element out of kMaxStackDepth is actually needed. - // Making the storage for call_stack variable-sized, - // substantially complicates memory management for the Region-s: - // as they need to be created and manipulated for some time - // w/o any memory allocations, yet are also given out to the users. - }; - - // Find the region that covers addr and write its data into *result if found, - // in which case *result gets filled so that it stays fully functional - // even when the underlying region gets removed from MemoryRegionMap. - // Returns success. Uses Lock/Unlock inside. - static bool FindRegion(uintptr_t addr, Region* result); - - // Find the region that contains stack_top, mark that region as - // a stack region, and write its data into *result if found, - // in which case *result gets filled so that it stays fully functional - // even when the underlying region gets removed from MemoryRegionMap. - // Returns success. Uses Lock/Unlock inside. - static bool FindAndMarkStackRegion(uintptr_t stack_top, Region* result); - - // Iterate over the buckets which store mmap and munmap counts per stack - // trace. It calls "callback" for each bucket, and passes "arg" to it. - template - static void IterateBuckets(void (*callback)(const HeapProfileBucket*, Type), - Type arg); - - // Get the bucket whose caller stack trace is "key". The stack trace is - // used to a depth of "depth" at most. The requested bucket is created if - // needed. - // The bucket table is described in heap-profile-stats.h. - static HeapProfileBucket* GetBucket(int depth, const void* const key[]); - - private: // our internal types ============================================== - - // Region comparator for sorting with STL - struct RegionCmp { - bool operator()(const Region& x, const Region& y) const { - return x.end_addr < y.end_addr; - } - }; - - // We allocate STL objects in our own arena. - struct MyAllocator { - static void *Allocate(size_t n) { - return LowLevelAlloc::AllocWithArena(n, arena_); - } - static void Free(const void *p, size_t /* n */) { - LowLevelAlloc::Free(const_cast(p)); - } - }; - - // Set of the memory regions - typedef std::set > RegionSet; - - public: // more in-depth interface ========================================== - - // STL iterator with values of Region - typedef RegionSet::const_iterator RegionIterator; - - // Return the begin/end iterators to all the regions. - // These need Lock/Unlock protection around their whole usage (loop). - // Even when the same thread causes modifications during such a loop - // (which are permitted due to recursive locking) - // the loop iterator will still be valid as long as its region - // has not been deleted, but EndRegionLocked should be - // re-evaluated whenever the set of regions has changed. - static RegionIterator BeginRegionLocked(); - static RegionIterator EndRegionLocked(); - - // Return the accumulated sizes of mapped and unmapped regions. - static int64 MapSize() { return map_size_; } - static int64 UnmapSize() { return unmap_size_; } - - // Effectively private type from our .cc ================================= - // public to let us declare global objects: - union RegionSetRep; - - private: - // representation =========================================================== - - // Counter of clients of this module that have called Init(). - static int client_count_; - - // Maximal number of caller stack frames to save (>= 0). - static int max_stack_depth_; - - // Arena used for our allocations in regions_. - static LowLevelAlloc::Arena* arena_; - - // Set of the mmap/sbrk/mremap-ed memory regions - // To be accessed *only* when Lock() is held. - // Hence we protect the non-recursive lock used inside of arena_ - // with our recursive Lock(). This lets a user prevent deadlocks - // when threads are stopped by TCMalloc_ListAllProcessThreads at random spots - // simply by acquiring our recursive Lock() before that. - static RegionSet* regions_; - - // Lock to protect regions_ and buckets_ variables and the data behind. - static SpinLock lock_; - // Lock to protect the recursive lock itself. - static SpinLock owner_lock_; - - // Recursion count for the recursive lock. - static int recursion_count_; - // The thread id of the thread that's inside the recursive lock. - static pthread_t lock_owner_tid_; - - // Total size of all mapped pages so far - static int64 map_size_; - // Total size of all unmapped pages so far - static int64 unmap_size_; - - // Bucket hash table which is described in heap-profile-stats.h. - static HeapProfileBucket** bucket_table_ GUARDED_BY(lock_); - static int num_buckets_ GUARDED_BY(lock_); - - // The following members are local to MemoryRegionMap::GetBucket() - // and MemoryRegionMap::HandleSavedBucketsLocked() - // and are file-level to ensure that they are initialized at load time. - // - // These are used as temporary storage to break the infinite cycle of mmap - // calling our hook which (sometimes) causes mmap. It must be a static - // fixed-size array. The size 20 is just an expected value for safety. - // The details are described in memory_region_map.cc. - - // Number of unprocessed bucket inserts. - static int saved_buckets_count_ GUARDED_BY(lock_); - - // Unprocessed inserts (must be big enough to hold all mmaps that can be - // caused by a GetBucket call). - // Bucket has no constructor, so that c-tor execution does not interfere - // with the any-time use of the static memory behind saved_buckets. - static HeapProfileBucket saved_buckets_[20] GUARDED_BY(lock_); - - static const void* saved_buckets_keys_[20][kMaxStackDepth] GUARDED_BY(lock_); - - // helpers ================================================================== - - // Helper for FindRegion and FindAndMarkStackRegion: - // returns the region covering 'addr' or NULL; assumes our lock_ is held. - static const Region* DoFindRegionLocked(uintptr_t addr); - - // Verifying wrapper around regions_->insert(region) - // To be called to do InsertRegionLocked's work only! - inline static void DoInsertRegionLocked(const Region& region); - // Handle regions saved by InsertRegionLocked into a tmp static array - // by calling insert_func on them. - inline static void HandleSavedRegionsLocked( - void (*insert_func)(const Region& region)); - - // Restore buckets saved in a tmp static array by GetBucket to the bucket - // table where all buckets eventually should be. - static void RestoreSavedBucketsLocked(); - - // Wrapper around DoInsertRegionLocked - // that handles the case of recursive allocator calls. - inline static void InsertRegionLocked(const Region& region); - - // Record addition of a memory region at address "start" of size "size" - // (called from our mmap/mremap/sbrk hooks). - static void RecordRegionAddition(const void* start, size_t size); - // Record deletion of a memory region at address "start" of size "size" - // (called from our munmap/mremap/sbrk hooks). - static void RecordRegionRemoval(const void* start, size_t size); - - // Record deletion of a memory region of size "size" in a bucket whose - // caller stack trace is "key". The stack trace is used to a depth of - // "depth" at most. - static void RecordRegionRemovalInBucket(int depth, - const void* const key[], - size_t size); - - // Hooks for MallocHook - static void MmapHook(const void* result, - const void* start, size_t size, - int prot, int flags, - int fd, off_t offset); - static void MunmapHook(const void* ptr, size_t size); - static void MremapHook(const void* result, const void* old_addr, - size_t old_size, size_t new_size, int flags, - const void* new_addr); - static void SbrkHook(const void* result, ptrdiff_t increment); - - // Log all memory regions; Useful for debugging only. - // Assumes Lock() is held - static void LogAllLocked(); - - DISALLOW_COPY_AND_ASSIGN(MemoryRegionMap); -}; - -template -void MemoryRegionMap::IterateBuckets( - void (*callback)(const HeapProfileBucket*, Type), Type callback_arg) { - for (int index = 0; index < kHashTableSize; index++) { - for (HeapProfileBucket* bucket = bucket_table_[index]; - bucket != NULL; - bucket = bucket->next) { - callback(bucket, callback_arg); - } - } -} - -#endif // BASE_MEMORY_REGION_MAP_H_ diff --git a/contrib/libtcmalloc/src/packed-cache-inl.h b/contrib/libtcmalloc/src/packed-cache-inl.h deleted file mode 100644 index 09462608ece..00000000000 --- a/contrib/libtcmalloc/src/packed-cache-inl.h +++ /dev/null @@ -1,239 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2007, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Geoff Pike -// -// This file provides a minimal cache that can hold a pair -// with little if any wasted space. The types of the key and value -// must be unsigned integral types or at least have unsigned semantics -// for >>, casting, and similar operations. -// -// Synchronization is not provided. However, the cache is implemented -// as an array of cache entries whose type is chosen at compile time. -// If a[i] is atomic on your hardware for the chosen array type then -// raciness will not necessarily lead to bugginess. The cache entries -// must be large enough to hold a partial key and a value packed -// together. The partial keys are bit strings of length -// kKeybits - kHashbits, and the values are bit strings of length kValuebits. -// -// In an effort to use minimal space, every cache entry represents -// some pair; the class provides no way to mark a cache -// entry as empty or uninitialized. In practice, you may want to have -// reserved keys or values to get around this limitation. For example, in -// tcmalloc's PageID-to-sizeclass cache, a value of 0 is used as -// "unknown sizeclass." -// -// Usage Considerations -// -------------------- -// -// kHashbits controls the size of the cache. The best value for -// kHashbits will of course depend on the application. Perhaps try -// tuning the value of kHashbits by measuring different values on your -// favorite benchmark. Also remember not to be a pig; other -// programs that need resources may suffer if you are. -// -// The main uses for this class will be when performance is -// critical and there's a convenient type to hold the cache's -// entries. As described above, the number of bits required -// for a cache entry is (kKeybits - kHashbits) + kValuebits. Suppose -// kKeybits + kValuebits is 43. Then it probably makes sense to -// chose kHashbits >= 11 so that cache entries fit in a uint32. -// -// On the other hand, suppose kKeybits = kValuebits = 64. Then -// using this class may be less worthwhile. You'll probably -// be using 128 bits for each entry anyway, so maybe just pick -// a hash function, H, and use an array indexed by H(key): -// void Put(K key, V value) { a_[H(key)] = pair(key, value); } -// V GetOrDefault(K key, V default) { const pair &p = a_[H(key)]; ... } -// etc. -// -// Further Details -// --------------- -// -// For caches used only by one thread, the following is true: -// 1. For a cache c, -// (c.Put(key, value), c.GetOrDefault(key, 0)) == value -// and -// (c.Put(key, value), <...>, c.GetOrDefault(key, 0)) == value -// if the elided code contains no c.Put calls. -// -// 2. Has(key) will return false if no pair with that key -// has ever been Put. However, a newly initialized cache will have -// some pairs already present. When you create a new -// cache, you must specify an "initial value." The initialization -// procedure is equivalent to Clear(initial_value), which is -// equivalent to Put(k, initial_value) for all keys k from 0 to -// 2^kHashbits - 1. -// -// 3. If key and key' differ then the only way Put(key, value) may -// cause Has(key') to change is that Has(key') may change from true to -// false. Furthermore, a Put() call that doesn't change Has(key') -// doesn't change GetOrDefault(key', ...) either. -// -// Implementation details: -// -// This is a direct-mapped cache with 2^kHashbits entries; the hash -// function simply takes the low bits of the key. We store whole keys -// if a whole key plus a whole value fits in an entry. Otherwise, an -// entry is the high bits of a key and a value, packed together. -// E.g., a 20 bit key and a 7 bit value only require a uint16 for each -// entry if kHashbits >= 11. -// -// Alternatives to this scheme will be added as needed. - -#ifndef TCMALLOC_PACKED_CACHE_INL_H_ -#define TCMALLOC_PACKED_CACHE_INL_H_ - -#include "config.h" -#include // for size_t -#ifdef HAVE_STDINT_H -#include // for uintptr_t -#endif -#include "base/basictypes.h" -#include "internal_logging.h" - -// A safe way of doing "(1 << n) - 1" -- without worrying about overflow -// Note this will all be resolved to a constant expression at compile-time -#define N_ONES_(IntType, N) \ - ( (N) == 0 ? 0 : ((static_cast(1) << ((N)-1))-1 + \ - (static_cast(1) << ((N)-1))) ) - -// The types K and V provide upper bounds on the number of valid keys -// and values, but we explicitly require the keys to be less than -// 2^kKeybits and the values to be less than 2^kValuebits. The size of -// the table is controlled by kHashbits, and the type of each entry in -// the cache is T. See also the big comment at the top of the file. -template -class PackedCache { - public: - typedef uintptr_t K; - typedef size_t V; -#ifdef TCMALLOC_SMALL_BUT_SLOW - // Decrease the size map cache if running in the small memory mode. - static const int kHashbits = 12; -#else - static const int kHashbits = 16; -#endif - static const int kValuebits = 7; - static const bool kUseWholeKeys = kKeybits + kValuebits <= 8 * sizeof(T); - - explicit PackedCache(V initial_value) { - COMPILE_ASSERT(kKeybits <= sizeof(K) * 8, key_size); - COMPILE_ASSERT(kValuebits <= sizeof(V) * 8, value_size); - COMPILE_ASSERT(kHashbits <= kKeybits, hash_function); - COMPILE_ASSERT(kKeybits - kHashbits + kValuebits <= kTbits, - entry_size_must_be_big_enough); - Clear(initial_value); - } - - void Put(K key, V value) { - ASSERT(key == (key & kKeyMask)); - ASSERT(value == (value & kValueMask)); - array_[Hash(key)] = KeyToUpper(key) | value; - } - - bool Has(K key) const { - ASSERT(key == (key & kKeyMask)); - return KeyMatch(array_[Hash(key)], key); - } - - V GetOrDefault(K key, V default_value) const { - // As with other code in this class, we touch array_ as few times - // as we can. Assuming entries are read atomically (e.g., their - // type is uintptr_t on most hardware) then certain races are - // harmless. - ASSERT(key == (key & kKeyMask)); - T entry = array_[Hash(key)]; - return KeyMatch(entry, key) ? EntryToValue(entry) : default_value; - } - - void Clear(V value) { - ASSERT(value == (value & kValueMask)); - for (int i = 0; i < 1 << kHashbits; i++) { - ASSERT(kUseWholeKeys || KeyToUpper(i) == 0); - array_[i] = kUseWholeKeys ? (value | KeyToUpper(i)) : value; - } - } - - private: - // We are going to pack a value and the upper part of a key (or a - // whole key) into an entry of type T. The UPPER type is for the - // upper part of a key, after the key has been masked and shifted - // for inclusion in an entry. - typedef T UPPER; - - static V EntryToValue(T t) { return t & kValueMask; } - - // If we have space for a whole key, we just shift it left. - // Otherwise kHashbits determines where in a K to find the upper - // part of the key, and kValuebits determines where in the entry to - // put it. - static UPPER KeyToUpper(K k) { - if (kUseWholeKeys) { - return static_cast(k) << kValuebits; - } else { - const int shift = kHashbits - kValuebits; - // Assume kHashbits >= kValuebits. It'd be easy to lift this assumption. - return static_cast(k >> shift) & kUpperMask; - } - } - - static size_t Hash(K key) { - return static_cast(key) & N_ONES_(size_t, kHashbits); - } - - // Does the entry match the relevant part of the given key? - static bool KeyMatch(T entry, K key) { - return kUseWholeKeys ? - (entry >> kValuebits == key) : - ((KeyToUpper(key) ^ entry) & kUpperMask) == 0; - } - - static const int kTbits = 8 * sizeof(T); - static const int kUpperbits = kUseWholeKeys ? kKeybits : kKeybits - kHashbits; - - // For masking a K. - static const K kKeyMask = N_ONES_(K, kKeybits); - - // For masking a T. - static const T kUpperMask = N_ONES_(T, kUpperbits) << kValuebits; - - // For masking a V or a T. - static const V kValueMask = N_ONES_(V, kValuebits); - - // array_ is the cache. Its elements are volatile because any - // thread can write any array element at any time. - volatile T array_[1 << kHashbits]; -}; - -#undef N_ONES_ - -#endif // TCMALLOC_PACKED_CACHE_INL_H_ diff --git a/contrib/libtcmalloc/src/page_heap.cc b/contrib/libtcmalloc/src/page_heap.cc deleted file mode 100644 index f1915623308..00000000000 --- a/contrib/libtcmalloc/src/page_heap.cc +++ /dev/null @@ -1,682 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat - -#include "config.h" -#ifdef HAVE_INTTYPES_H -#include // for PRIuPTR -#endif -#include // for ENOMEM, errno -#include // for MallocRange, etc -#include "base/basictypes.h" -#include "base/commandlineflags.h" -#include "internal_logging.h" // for ASSERT, TCMalloc_Printer, etc -#include "page_heap_allocator.h" // for PageHeapAllocator -#include "static_vars.h" // for Static -#include "system-alloc.h" // for TCMalloc_SystemAlloc, etc - -DEFINE_double(tcmalloc_release_rate, - EnvToDouble("TCMALLOC_RELEASE_RATE", 1.0), - "Rate at which we release unused memory to the system. " - "Zero means we never release memory back to the system. " - "Increase this flag to return memory faster; decrease it " - "to return memory slower. Reasonable rates are in the " - "range [0,10]"); - -DEFINE_int64(tcmalloc_heap_limit_mb, - EnvToInt("TCMALLOC_HEAP_LIMIT_MB", 0), - "Limit total size of the process heap to the " - "specified number of MiB. " - "When we approach the limit the memory is released " - "to the system more aggressively (more minor page faults). " - "Zero means to allocate as long as system allows."); - -namespace tcmalloc { - -PageHeap::PageHeap() - : pagemap_(MetaDataAlloc), - pagemap_cache_(0), - scavenge_counter_(0), - // Start scavenging at kMaxPages list - release_index_(kMaxPages), - aggressive_decommit_(false) { - COMPILE_ASSERT(kNumClasses <= (1 << PageMapCache::kValuebits), valuebits); - DLL_Init(&large_.normal); - DLL_Init(&large_.returned); - for (int i = 0; i < kMaxPages; i++) { - DLL_Init(&free_[i].normal); - DLL_Init(&free_[i].returned); - } -} - -Span* PageHeap::SearchFreeAndLargeLists(Length n) { - ASSERT(Check()); - ASSERT(n > 0); - - // Find first size >= n that has a non-empty list - for (Length s = n; s < kMaxPages; s++) { - Span* ll = &free_[s].normal; - // If we're lucky, ll is non-empty, meaning it has a suitable span. - if (!DLL_IsEmpty(ll)) { - ASSERT(ll->next->location == Span::ON_NORMAL_FREELIST); - return Carve(ll->next, n); - } - // Alternatively, maybe there's a usable returned span. - ll = &free_[s].returned; - if (!DLL_IsEmpty(ll)) { - // We did not call EnsureLimit before, to avoid releasing the span - // that will be taken immediately back. - // Calling EnsureLimit here is not very expensive, as it fails only if - // there is no more normal spans (and it fails efficiently) - // or SystemRelease does not work (there is probably no returned spans). - if (EnsureLimit(n)) { - // ll may have became empty due to coalescing - if (!DLL_IsEmpty(ll)) { - ASSERT(ll->next->location == Span::ON_RETURNED_FREELIST); - return Carve(ll->next, n); - } - } - } - } - // No luck in free lists, our last chance is in a larger class. - return AllocLarge(n); // May be NULL -} - -static const size_t kForcedCoalesceInterval = 128*1024*1024; - -Span* PageHeap::New(Length n) { - ASSERT(Check()); - ASSERT(n > 0); - - Span* result = SearchFreeAndLargeLists(n); - if (result != NULL) - return result; - - if (stats_.free_bytes != 0 && stats_.unmapped_bytes != 0 - && stats_.free_bytes + stats_.unmapped_bytes >= stats_.system_bytes / 4 - && (stats_.system_bytes / kForcedCoalesceInterval - != (stats_.system_bytes + (n << kPageShift)) / kForcedCoalesceInterval)) { - // We're about to grow heap, but there are lots of free pages. - // tcmalloc's design decision to keep unmapped and free spans - // separately and never coalesce them means that sometimes there - // can be free pages span of sufficient size, but it consists of - // "segments" of different type so page heap search cannot find - // it. In order to prevent growing heap and wasting memory in such - // case we're going to unmap all free pages. So that all free - // spans are maximally coalesced. - // - // We're also limiting 'rate' of going into this path to be at - // most once per 128 megs of heap growth. Otherwise programs that - // grow heap frequently (and that means by small amount) could be - // penalized with higher count of minor page faults. - // - // See also large_heap_fragmentation_unittest.cc and - // https://code.google.com/p/gperftools/issues/detail?id=368 - ReleaseAtLeastNPages(static_cast(0x7fffffff)); - - // then try again. If we are forced to grow heap because of large - // spans fragmentation and not because of problem described above, - // then at the very least we've just unmapped free but - // insufficiently big large spans back to OS. So in case of really - // unlucky memory fragmentation we'll be consuming virtual address - // space, but not real memory - result = SearchFreeAndLargeLists(n); - if (result != NULL) return result; - } - - // Grow the heap and try again. - if (!GrowHeap(n)) { - ASSERT(stats_.unmapped_bytes+ stats_.committed_bytes==stats_.system_bytes); - ASSERT(Check()); - // underlying SysAllocator likely set ENOMEM but we can get here - // due to EnsureLimit so we set it here too. - // - // Setting errno to ENOMEM here allows us to avoid dealing with it - // in fast-path. - errno = ENOMEM; - return NULL; - } - return SearchFreeAndLargeLists(n); -} - -Span* PageHeap::AllocLarge(Length n) { - // find the best span (closest to n in size). - // The following loops implements address-ordered best-fit. - Span *best = NULL; - - // Search through normal list - for (Span* span = large_.normal.next; - span != &large_.normal; - span = span->next) { - if (span->length >= n) { - if ((best == NULL) - || (span->length < best->length) - || ((span->length == best->length) && (span->start < best->start))) { - best = span; - ASSERT(best->location == Span::ON_NORMAL_FREELIST); - } - } - } - - Span *bestNormal = best; - - // Search through released list in case it has a better fit - for (Span* span = large_.returned.next; - span != &large_.returned; - span = span->next) { - if (span->length >= n) { - if ((best == NULL) - || (span->length < best->length) - || ((span->length == best->length) && (span->start < best->start))) { - best = span; - ASSERT(best->location == Span::ON_RETURNED_FREELIST); - } - } - } - - if (best == bestNormal) { - return best == NULL ? NULL : Carve(best, n); - } - - // best comes from returned list. - - if (EnsureLimit(n, false)) { - return Carve(best, n); - } - - if (EnsureLimit(n, true)) { - // best could have been destroyed by coalescing. - // bestNormal is not a best-fit, and it could be destroyed as well. - // We retry, the limit is already ensured: - return AllocLarge(n); - } - - // If bestNormal existed, EnsureLimit would succeeded: - ASSERT(bestNormal == NULL); - // We are not allowed to take best from returned list. - return NULL; -} - -Span* PageHeap::Split(Span* span, Length n) { - ASSERT(0 < n); - ASSERT(n < span->length); - ASSERT(span->location == Span::IN_USE); - ASSERT(span->sizeclass == 0); - Event(span, 'T', n); - - const int extra = span->length - n; - Span* leftover = NewSpan(span->start + n, extra); - ASSERT(leftover->location == Span::IN_USE); - Event(leftover, 'U', extra); - RecordSpan(leftover); - pagemap_.set(span->start + n - 1, span); // Update map from pageid to span - span->length = n; - - return leftover; -} - -void PageHeap::CommitSpan(Span* span) { - TCMalloc_SystemCommit(reinterpret_cast(span->start << kPageShift), - static_cast(span->length << kPageShift)); - stats_.committed_bytes += span->length << kPageShift; -} - -bool PageHeap::DecommitSpan(Span* span) { - bool rv = TCMalloc_SystemRelease(reinterpret_cast(span->start << kPageShift), - static_cast(span->length << kPageShift)); - if (rv) { - stats_.committed_bytes -= span->length << kPageShift; - } - - return rv; -} - -Span* PageHeap::Carve(Span* span, Length n) { - ASSERT(n > 0); - ASSERT(span->location != Span::IN_USE); - const int old_location = span->location; - RemoveFromFreeList(span); - span->location = Span::IN_USE; - Event(span, 'A', n); - - const int extra = span->length - n; - ASSERT(extra >= 0); - if (extra > 0) { - Span* leftover = NewSpan(span->start + n, extra); - leftover->location = old_location; - Event(leftover, 'S', extra); - RecordSpan(leftover); - - // The previous span of |leftover| was just splitted -- no need to - // coalesce them. The next span of |leftover| was not previously coalesced - // with |span|, i.e. is NULL or has got location other than |old_location|. -#ifndef NDEBUG - const PageID p = leftover->start; - const Length len = leftover->length; - Span* next = GetDescriptor(p+len); - ASSERT (next == NULL || - next->location == Span::IN_USE || - next->location != leftover->location); -#endif - - PrependToFreeList(leftover); // Skip coalescing - no candidates possible - span->length = n; - pagemap_.set(span->start + n - 1, span); - } - ASSERT(Check()); - if (old_location == Span::ON_RETURNED_FREELIST) { - // We need to recommit this address space. - CommitSpan(span); - } - ASSERT(span->location == Span::IN_USE); - ASSERT(span->length == n); - ASSERT(stats_.unmapped_bytes+ stats_.committed_bytes==stats_.system_bytes); - return span; -} - -void PageHeap::Delete(Span* span) { - ASSERT(Check()); - ASSERT(span->location == Span::IN_USE); - ASSERT(span->length > 0); - ASSERT(GetDescriptor(span->start) == span); - ASSERT(GetDescriptor(span->start + span->length - 1) == span); - const Length n = span->length; - span->sizeclass = 0; - span->sample = 0; - span->location = Span::ON_NORMAL_FREELIST; - Event(span, 'D', span->length); - MergeIntoFreeList(span); // Coalesces if possible - IncrementalScavenge(n); - ASSERT(stats_.unmapped_bytes+ stats_.committed_bytes==stats_.system_bytes); - ASSERT(Check()); -} - -bool PageHeap::MayMergeSpans(Span *span, Span *other) { - if (aggressive_decommit_) { - return other->location != Span::IN_USE; - } - return span->location == other->location; -} - -void PageHeap::MergeIntoFreeList(Span* span) { - ASSERT(span->location != Span::IN_USE); - - // Coalesce -- we guarantee that "p" != 0, so no bounds checking - // necessary. We do not bother resetting the stale pagemap - // entries for the pieces we are merging together because we only - // care about the pagemap entries for the boundaries. - // - // Note: depending on aggressive_decommit_ mode we allow only - // similar spans to be coalesced. - // - // The following applies if aggressive_decommit_ is enabled: - // - // Note that the adjacent spans we merge into "span" may come out of a - // "normal" (committed) list, and cleanly merge with our IN_USE span, which - // is implicitly committed. If the adjacents spans are on the "returned" - // (decommitted) list, then we must get both spans into the same state before - // or after we coalesce them. The current code always decomits. This is - // achieved by blindly decommitting the entire coalesced region, which may - // include any combination of committed and decommitted spans, at the end of - // the method. - - // TODO(jar): "Always decommit" causes some extra calls to commit when we are - // called in GrowHeap() during an allocation :-/. We need to eval the cost of - // that oscillation, and possibly do something to reduce it. - - // TODO(jar): We need a better strategy for deciding to commit, or decommit, - // based on memory usage and free heap sizes. - - uint64_t temp_committed = 0; - - const PageID p = span->start; - const Length n = span->length; - Span* prev = GetDescriptor(p-1); - if (prev != NULL && MayMergeSpans(span, prev)) { - // Merge preceding span into this span - ASSERT(prev->start + prev->length == p); - const Length len = prev->length; - if (aggressive_decommit_ && prev->location == Span::ON_RETURNED_FREELIST) { - // We're about to put the merge span into the returned freelist and call - // DecommitSpan() on it, which will mark the entire span including this - // one as released and decrease stats_.committed_bytes by the size of the - // merged span. To make the math work out we temporarily increase the - // stats_.committed_bytes amount. - temp_committed = prev->length << kPageShift; - } - RemoveFromFreeList(prev); - DeleteSpan(prev); - span->start -= len; - span->length += len; - pagemap_.set(span->start, span); - Event(span, 'L', len); - } - Span* next = GetDescriptor(p+n); - if (next != NULL && MayMergeSpans(span, next)) { - // Merge next span into this span - ASSERT(next->start == p+n); - const Length len = next->length; - if (aggressive_decommit_ && next->location == Span::ON_RETURNED_FREELIST) { - // See the comment below 'if (prev->location ...' for explanation. - temp_committed += next->length << kPageShift; - } - RemoveFromFreeList(next); - DeleteSpan(next); - span->length += len; - pagemap_.set(span->start + span->length - 1, span); - Event(span, 'R', len); - } - - if (aggressive_decommit_) { - if (DecommitSpan(span)) { - span->location = Span::ON_RETURNED_FREELIST; - stats_.committed_bytes += temp_committed; - } else { - ASSERT(temp_committed == 0); - } - } - PrependToFreeList(span); -} - -void PageHeap::PrependToFreeList(Span* span) { - ASSERT(span->location != Span::IN_USE); - SpanList* list = (span->length < kMaxPages) ? &free_[span->length] : &large_; - if (span->location == Span::ON_NORMAL_FREELIST) { - stats_.free_bytes += (span->length << kPageShift); - DLL_Prepend(&list->normal, span); - } else { - stats_.unmapped_bytes += (span->length << kPageShift); - DLL_Prepend(&list->returned, span); - } -} - -void PageHeap::RemoveFromFreeList(Span* span) { - ASSERT(span->location != Span::IN_USE); - if (span->location == Span::ON_NORMAL_FREELIST) { - stats_.free_bytes -= (span->length << kPageShift); - } else { - stats_.unmapped_bytes -= (span->length << kPageShift); - } - DLL_Remove(span); -} - -void PageHeap::IncrementalScavenge(Length n) { - // Fast path; not yet time to release memory - scavenge_counter_ -= n; - if (scavenge_counter_ >= 0) return; // Not yet time to scavenge - - const double rate = FLAGS_tcmalloc_release_rate; - if (rate <= 1e-6) { - // Tiny release rate means that releasing is disabled. - scavenge_counter_ = kDefaultReleaseDelay; - return; - } - - Length released_pages = ReleaseAtLeastNPages(1); - - if (released_pages == 0) { - // Nothing to scavenge, delay for a while. - scavenge_counter_ = kDefaultReleaseDelay; - } else { - // Compute how long to wait until we return memory. - // FLAGS_tcmalloc_release_rate==1 means wait for 1000 pages - // after releasing one page. - const double mult = 1000.0 / rate; - double wait = mult * static_cast(released_pages); - if (wait > kMaxReleaseDelay) { - // Avoid overflow and bound to reasonable range. - wait = kMaxReleaseDelay; - } - scavenge_counter_ = static_cast(wait); - } -} - -Length PageHeap::ReleaseLastNormalSpan(SpanList* slist) { - Span* s = slist->normal.prev; - ASSERT(s->location == Span::ON_NORMAL_FREELIST); - - if (DecommitSpan(s)) { - RemoveFromFreeList(s); - const Length n = s->length; - s->location = Span::ON_RETURNED_FREELIST; - MergeIntoFreeList(s); // Coalesces if possible. - return n; - } - - return 0; -} - -Length PageHeap::ReleaseAtLeastNPages(Length num_pages) { - Length released_pages = 0; - - // Round robin through the lists of free spans, releasing the last - // span in each list. Stop after releasing at least num_pages - // or when there is nothing more to release. - while (released_pages < num_pages && stats_.free_bytes > 0) { - for (int i = 0; i < kMaxPages+1 && released_pages < num_pages; - i++, release_index_++) { - if (release_index_ > kMaxPages) release_index_ = 0; - SpanList* slist = (release_index_ == kMaxPages) ? - &large_ : &free_[release_index_]; - if (!DLL_IsEmpty(&slist->normal)) { - Length released_len = ReleaseLastNormalSpan(slist); - // Some systems do not support release - if (released_len == 0) return released_pages; - released_pages += released_len; - } - } - } - return released_pages; -} - -bool PageHeap::EnsureLimit(Length n, bool withRelease) -{ - Length limit = (FLAGS_tcmalloc_heap_limit_mb*1024*1024) >> kPageShift; - if (limit == 0) return true; //there is no limit - - // We do not use stats_.system_bytes because it does not take - // MetaDataAllocs into account. - Length takenPages = TCMalloc_SystemTaken >> kPageShift; - //XXX takenPages may be slightly bigger than limit for two reasons: - //* MetaDataAllocs ignore the limit (it is not easy to handle - // out of memory there) - //* sys_alloc may round allocation up to huge page size, - // although smaller limit was ensured - - ASSERT(takenPages >= stats_.unmapped_bytes >> kPageShift); - takenPages -= stats_.unmapped_bytes >> kPageShift; - - if (takenPages + n > limit && withRelease) { - takenPages -= ReleaseAtLeastNPages(takenPages + n - limit); - } - - return takenPages + n <= limit; -} - -void PageHeap::RegisterSizeClass(Span* span, size_t sc) { - // Associate span object with all interior pages as well - ASSERT(span->location == Span::IN_USE); - ASSERT(GetDescriptor(span->start) == span); - ASSERT(GetDescriptor(span->start+span->length-1) == span); - Event(span, 'C', sc); - span->sizeclass = sc; - for (Length i = 1; i < span->length-1; i++) { - pagemap_.set(span->start+i, span); - } -} - -void PageHeap::GetSmallSpanStats(SmallSpanStats* result) { - for (int s = 0; s < kMaxPages; s++) { - result->normal_length[s] = DLL_Length(&free_[s].normal); - result->returned_length[s] = DLL_Length(&free_[s].returned); - } -} - -void PageHeap::GetLargeSpanStats(LargeSpanStats* result) { - result->spans = 0; - result->normal_pages = 0; - result->returned_pages = 0; - for (Span* s = large_.normal.next; s != &large_.normal; s = s->next) { - result->normal_pages += s->length;; - result->spans++; - } - for (Span* s = large_.returned.next; s != &large_.returned; s = s->next) { - result->returned_pages += s->length; - result->spans++; - } -} - -bool PageHeap::GetNextRange(PageID start, base::MallocRange* r) { - Span* span = reinterpret_cast(pagemap_.Next(start)); - if (span == NULL) { - return false; - } - r->address = span->start << kPageShift; - r->length = span->length << kPageShift; - r->fraction = 0; - switch (span->location) { - case Span::IN_USE: - r->type = base::MallocRange::INUSE; - r->fraction = 1; - if (span->sizeclass > 0) { - // Only some of the objects in this span may be in use. - const size_t osize = Static::sizemap()->class_to_size(span->sizeclass); - r->fraction = (1.0 * osize * span->refcount) / r->length; - } - break; - case Span::ON_NORMAL_FREELIST: - r->type = base::MallocRange::FREE; - break; - case Span::ON_RETURNED_FREELIST: - r->type = base::MallocRange::UNMAPPED; - break; - default: - r->type = base::MallocRange::UNKNOWN; - break; - } - return true; -} - -static void RecordGrowth(size_t growth) { - StackTrace* t = Static::stacktrace_allocator()->New(); - t->depth = GetStackTrace(t->stack, kMaxStackDepth-1, 3); - t->size = growth; - t->stack[kMaxStackDepth-1] = reinterpret_cast(Static::growth_stacks()); - Static::set_growth_stacks(t); -} - -bool PageHeap::GrowHeap(Length n) { - ASSERT(kMaxPages >= kMinSystemAlloc); - if (n > kMaxValidPages) return false; - Length ask = (n>kMinSystemAlloc) ? n : static_cast(kMinSystemAlloc); - size_t actual_size; - void* ptr = NULL; - if (EnsureLimit(ask)) { - ptr = TCMalloc_SystemAlloc(ask << kPageShift, &actual_size, kPageSize); - } - if (ptr == NULL) { - if (n < ask) { - // Try growing just "n" pages - ask = n; - if (EnsureLimit(ask)) { - ptr = TCMalloc_SystemAlloc(ask << kPageShift, &actual_size, kPageSize); - } - } - if (ptr == NULL) return false; - } - ask = actual_size >> kPageShift; - RecordGrowth(ask << kPageShift); - - uint64_t old_system_bytes = stats_.system_bytes; - stats_.system_bytes += (ask << kPageShift); - stats_.committed_bytes += (ask << kPageShift); - const PageID p = reinterpret_cast(ptr) >> kPageShift; - ASSERT(p > 0); - - // If we have already a lot of pages allocated, just pre allocate a bunch of - // memory for the page map. This prevents fragmentation by pagemap metadata - // when a program keeps allocating and freeing large blocks. - - if (old_system_bytes < kPageMapBigAllocationThreshold - && stats_.system_bytes >= kPageMapBigAllocationThreshold) { - pagemap_.PreallocateMoreMemory(); - } - - // Make sure pagemap_ has entries for all of the new pages. - // Plus ensure one before and one after so coalescing code - // does not need bounds-checking. - if (pagemap_.Ensure(p-1, ask+2)) { - // Pretend the new area is allocated and then Delete() it to cause - // any necessary coalescing to occur. - Span* span = NewSpan(p, ask); - RecordSpan(span); - Delete(span); - ASSERT(stats_.unmapped_bytes+ stats_.committed_bytes==stats_.system_bytes); - ASSERT(Check()); - return true; - } else { - // We could not allocate memory within "pagemap_" - // TODO: Once we can return memory to the system, return the new span - return false; - } -} - -bool PageHeap::Check() { - ASSERT(free_[0].normal.next == &free_[0].normal); - ASSERT(free_[0].returned.next == &free_[0].returned); - return true; -} - -bool PageHeap::CheckExpensive() { - bool result = Check(); - CheckList(&large_.normal, kMaxPages, 1000000000, Span::ON_NORMAL_FREELIST); - CheckList(&large_.returned, kMaxPages, 1000000000, Span::ON_RETURNED_FREELIST); - for (Length s = 1; s < kMaxPages; s++) { - CheckList(&free_[s].normal, s, s, Span::ON_NORMAL_FREELIST); - CheckList(&free_[s].returned, s, s, Span::ON_RETURNED_FREELIST); - } - return result; -} - -bool PageHeap::CheckList(Span* list, Length min_pages, Length max_pages, - int freelist) { - for (Span* s = list->next; s != list; s = s->next) { - CHECK_CONDITION(s->location == freelist); // NORMAL or RETURNED - CHECK_CONDITION(s->length >= min_pages); - CHECK_CONDITION(s->length <= max_pages); - CHECK_CONDITION(GetDescriptor(s->start) == s); - CHECK_CONDITION(GetDescriptor(s->start+s->length-1) == s); - } - return true; -} - -} // namespace tcmalloc diff --git a/contrib/libtcmalloc/src/page_heap.h b/contrib/libtcmalloc/src/page_heap.h deleted file mode 100644 index 89fab81da69..00000000000 --- a/contrib/libtcmalloc/src/page_heap.h +++ /dev/null @@ -1,316 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat - -#ifndef TCMALLOC_PAGE_HEAP_H_ -#define TCMALLOC_PAGE_HEAP_H_ - -#include "config.h" -#include // for size_t -#ifdef HAVE_STDINT_H -#include // for uint64_t, int64_t, uint16_t -#endif -#include -#include "base/basictypes.h" -#include "common.h" -#include "packed-cache-inl.h" -#include "pagemap.h" -#include "span.h" - -// We need to dllexport PageHeap just for the unittest. MSVC complains -// that we don't dllexport the PageHeap members, but we don't need to -// test those, so I just suppress this warning. -#ifdef _MSC_VER -#pragma warning(push) -#pragma warning(disable:4251) -#endif - -// This #ifdef should almost never be set. Set NO_TCMALLOC_SAMPLES if -// you're porting to a system where you really can't get a stacktrace. -// Because we control the definition of GetStackTrace, all clients of -// GetStackTrace should #include us rather than stacktrace.h. -#ifdef NO_TCMALLOC_SAMPLES - // We use #define so code compiles even if you #include stacktrace.h somehow. -# define GetStackTrace(stack, depth, skip) (0) -#else -# include -#endif - -namespace base { -struct MallocRange; -} - -namespace tcmalloc { - -// ------------------------------------------------------------------------- -// Map from page-id to per-page data -// ------------------------------------------------------------------------- - -// We use PageMap2<> for 32-bit and PageMap3<> for 64-bit machines. -// We also use a simple one-level cache for hot PageID-to-sizeclass mappings, -// because sometimes the sizeclass is all the information we need. - -// Selector class -- general selector uses 3-level map -template class MapSelector { - public: - typedef TCMalloc_PageMap3 Type; - typedef PackedCache CacheType; -}; - -// A two-level map for 32-bit machines -template <> class MapSelector<32> { - public: - typedef TCMalloc_PageMap2<32-kPageShift> Type; - typedef PackedCache<32-kPageShift, uint16_t> CacheType; -}; - -// ------------------------------------------------------------------------- -// Page-level allocator -// * Eager coalescing -// -// Heap for page-level allocation. We allow allocating and freeing a -// contiguous runs of pages (called a "span"). -// ------------------------------------------------------------------------- - -class PERFTOOLS_DLL_DECL PageHeap { - public: - PageHeap(); - - // Allocate a run of "n" pages. Returns zero if out of memory. - // Caller should not pass "n == 0" -- instead, n should have - // been rounded up already. - Span* New(Length n); - - // Delete the span "[p, p+n-1]". - // REQUIRES: span was returned by earlier call to New() and - // has not yet been deleted. - void Delete(Span* span); - - // Mark an allocated span as being used for small objects of the - // specified size-class. - // REQUIRES: span was returned by an earlier call to New() - // and has not yet been deleted. - void RegisterSizeClass(Span* span, size_t sc); - - // Split an allocated span into two spans: one of length "n" pages - // followed by another span of length "span->length - n" pages. - // Modifies "*span" to point to the first span of length "n" pages. - // Returns a pointer to the second span. - // - // REQUIRES: "0 < n < span->length" - // REQUIRES: span->location == IN_USE - // REQUIRES: span->sizeclass == 0 - Span* Split(Span* span, Length n); - - // Return the descriptor for the specified page. Returns NULL if - // this PageID was not allocated previously. - inline Span* GetDescriptor(PageID p) const { - return reinterpret_cast(pagemap_.get(p)); - } - - // If this page heap is managing a range with starting page # >= start, - // store info about the range in *r and return true. Else return false. - bool GetNextRange(PageID start, base::MallocRange* r); - - // Page heap statistics - struct Stats { - Stats() : system_bytes(0), free_bytes(0), unmapped_bytes(0), committed_bytes(0) {} - uint64_t system_bytes; // Total bytes allocated from system - uint64_t free_bytes; // Total bytes on normal freelists - uint64_t unmapped_bytes; // Total bytes on returned freelists - uint64_t committed_bytes; // Bytes committed, always <= system_bytes_. - - }; - inline Stats stats() const { return stats_; } - - struct SmallSpanStats { - // For each free list of small spans, the length (in spans) of the - // normal and returned free lists for that size. - int64 normal_length[kMaxPages]; - int64 returned_length[kMaxPages]; - }; - void GetSmallSpanStats(SmallSpanStats* result); - - // Stats for free large spans (i.e., spans with more than kMaxPages pages). - struct LargeSpanStats { - int64 spans; // Number of such spans - int64 normal_pages; // Combined page length of normal large spans - int64 returned_pages; // Combined page length of unmapped spans - }; - void GetLargeSpanStats(LargeSpanStats* result); - - bool Check(); - // Like Check() but does some more comprehensive checking. - bool CheckExpensive(); - bool CheckList(Span* list, Length min_pages, Length max_pages, - int freelist); // ON_NORMAL_FREELIST or ON_RETURNED_FREELIST - - // Try to release at least num_pages for reuse by the OS. Returns - // the actual number of pages released, which may be less than - // num_pages if there weren't enough pages to release. The result - // may also be larger than num_pages since page_heap might decide to - // release one large range instead of fragmenting it into two - // smaller released and unreleased ranges. - Length ReleaseAtLeastNPages(Length num_pages); - - // Return 0 if we have no information, or else the correct sizeclass for p. - // Reads and writes to pagemap_cache_ do not require locking. - // The entries are 64 bits on 64-bit hardware and 16 bits on - // 32-bit hardware, and we don't mind raciness as long as each read of - // an entry yields a valid entry, not a partially updated entry. - size_t GetSizeClassIfCached(PageID p) const { - return pagemap_cache_.GetOrDefault(p, 0); - } - void CacheSizeClass(PageID p, size_t cl) const { pagemap_cache_.Put(p, cl); } - - bool GetAggressiveDecommit(void) {return aggressive_decommit_;} - void SetAggressiveDecommit(bool aggressive_decommit) { - aggressive_decommit_ = aggressive_decommit; - } - - private: - // Allocates a big block of memory for the pagemap once we reach more than - // 128MB - static const size_t kPageMapBigAllocationThreshold = 128 << 20; - - // Minimum number of pages to fetch from system at a time. Must be - // significantly bigger than kBlockSize to amortize system-call - // overhead, and also to reduce external fragementation. Also, we - // should keep this value big because various incarnations of Linux - // have small limits on the number of mmap() regions per - // address-space. - // REQUIRED: kMinSystemAlloc <= kMaxPages; - static const int kMinSystemAlloc = kMaxPages; - - // Never delay scavenging for more than the following number of - // deallocated pages. With 4K pages, this comes to 4GB of - // deallocation. - static const int kMaxReleaseDelay = 1 << 20; - - // If there is nothing to release, wait for so many pages before - // scavenging again. With 4K pages, this comes to 1GB of memory. - static const int kDefaultReleaseDelay = 1 << 18; - - // Pick the appropriate map and cache types based on pointer size - typedef MapSelector::Type PageMap; - typedef MapSelector::CacheType PageMapCache; - PageMap pagemap_; - mutable PageMapCache pagemap_cache_; - - // We segregate spans of a given size into two circular linked - // lists: one for normal spans, and one for spans whose memory - // has been returned to the system. - struct SpanList { - Span normal; - Span returned; - }; - - // List of free spans of length >= kMaxPages - SpanList large_; - - // Array mapping from span length to a doubly linked list of free spans - SpanList free_[kMaxPages]; - - // Statistics on system, free, and unmapped bytes - Stats stats_; - - Span* SearchFreeAndLargeLists(Length n); - - bool GrowHeap(Length n); - - // REQUIRES: span->length >= n - // REQUIRES: span->location != IN_USE - // Remove span from its free list, and move any leftover part of - // span into appropriate free lists. Also update "span" to have - // length exactly "n" and mark it as non-free so it can be returned - // to the client. After all that, decrease free_pages_ by n and - // return span. - Span* Carve(Span* span, Length n); - - void RecordSpan(Span* span) { - pagemap_.set(span->start, span); - if (span->length > 1) { - pagemap_.set(span->start + span->length - 1, span); - } - } - - // Allocate a large span of length == n. If successful, returns a - // span of exactly the specified length. Else, returns NULL. - Span* AllocLarge(Length n); - - // Coalesce span with neighboring spans if possible, prepend to - // appropriate free list, and adjust stats. - void MergeIntoFreeList(Span* span); - - // Commit the span. - void CommitSpan(Span* span); - - // Decommit the span. - bool DecommitSpan(Span* span); - - // Prepends span to appropriate free list, and adjusts stats. - void PrependToFreeList(Span* span); - - // Removes span from its free list, and adjust stats. - void RemoveFromFreeList(Span* span); - - // Incrementally release some memory to the system. - // IncrementalScavenge(n) is called whenever n pages are freed. - void IncrementalScavenge(Length n); - - // Release the last span on the normal portion of this list. - // Return the length of that span or zero if release failed. - Length ReleaseLastNormalSpan(SpanList* slist); - - // Checks if we are allowed to take more memory from the system. - // If limit is reached and allowRelease is true, tries to release - // some unused spans. - bool EnsureLimit(Length n, bool allowRelease = true); - - bool MayMergeSpans(Span *span, Span *other); - - // Number of pages to deallocate before doing more scavenging - int64_t scavenge_counter_; - - // Index of last free list where we released memory to the OS. - int release_index_; - - bool aggressive_decommit_; -}; - -} // namespace tcmalloc - -#ifdef _MSC_VER -#pragma warning(pop) -#endif - -#endif // TCMALLOC_PAGE_HEAP_H_ diff --git a/contrib/libtcmalloc/src/page_heap_allocator.h b/contrib/libtcmalloc/src/page_heap_allocator.h deleted file mode 100644 index 892d1c1abe3..00000000000 --- a/contrib/libtcmalloc/src/page_heap_allocator.h +++ /dev/null @@ -1,114 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat - -#ifndef TCMALLOC_PAGE_HEAP_ALLOCATOR_H_ -#define TCMALLOC_PAGE_HEAP_ALLOCATOR_H_ - -#include // for NULL, size_t - -#include "common.h" // for MetaDataAlloc -#include "internal_logging.h" // for ASSERT - -namespace tcmalloc { - -// Simple allocator for objects of a specified type. External locking -// is required before accessing one of these objects. -template -class PageHeapAllocator { - public: - // We use an explicit Init function because these variables are statically - // allocated and their constructors might not have run by the time some - // other static variable tries to allocate memory. - void Init() { - ASSERT(sizeof(T) <= kAllocIncrement); - inuse_ = 0; - free_area_ = NULL; - free_avail_ = 0; - free_list_ = NULL; - // Reserve some space at the beginning to avoid fragmentation. - Delete(New()); - } - - T* New() { - // Consult free list - void* result; - if (free_list_ != NULL) { - result = free_list_; - free_list_ = *(reinterpret_cast(result)); - } else { - if (free_avail_ < sizeof(T)) { - // Need more room. We assume that MetaDataAlloc returns - // suitably aligned memory. - free_area_ = reinterpret_cast(MetaDataAlloc(kAllocIncrement)); - if (free_area_ == NULL) { - Log(kCrash, __FILE__, __LINE__, - "FATAL ERROR: Out of memory trying to allocate internal " - "tcmalloc data (bytes, object-size)", - kAllocIncrement, sizeof(T)); - } - free_avail_ = kAllocIncrement; - } - result = free_area_; - free_area_ += sizeof(T); - free_avail_ -= sizeof(T); - } - inuse_++; - return reinterpret_cast(result); - } - - void Delete(T* p) { - *(reinterpret_cast(p)) = free_list_; - free_list_ = p; - inuse_--; - } - - int inuse() const { return inuse_; } - - private: - // How much to allocate from system at a time - static const int kAllocIncrement = 128 << 10; - - // Free area from which to carve new objects - char* free_area_; - size_t free_avail_; - - // Free list of already carved objects - void* free_list_; - - // Number of allocated but unfreed objects - int inuse_; -}; - -} // namespace tcmalloc - -#endif // TCMALLOC_PAGE_HEAP_ALLOCATOR_H_ diff --git a/contrib/libtcmalloc/src/pagemap.h b/contrib/libtcmalloc/src/pagemap.h deleted file mode 100644 index dd9442313af..00000000000 --- a/contrib/libtcmalloc/src/pagemap.h +++ /dev/null @@ -1,324 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat -// -// A data structure used by the caching malloc. It maps from page# to -// a pointer that contains info about that page. We use two -// representations: one for 32-bit addresses, and another for 64 bit -// addresses. Both representations provide the same interface. The -// first representation is implemented as a flat array, the seconds as -// a three-level radix tree that strips away approximately 1/3rd of -// the bits every time. -// -// The BITS parameter should be the number of bits required to hold -// a page number. E.g., with 32 bit pointers and 4K pages (i.e., -// page offset fits in lower 12 bits), BITS == 20. - -#ifndef TCMALLOC_PAGEMAP_H_ -#define TCMALLOC_PAGEMAP_H_ - -#include "config.h" - -#include // for NULL, size_t -#include // for memset -#if defined HAVE_STDINT_H -#include -#elif defined HAVE_INTTYPES_H -#include -#else -#include -#endif -#include "internal_logging.h" // for ASSERT - -// Single-level array -template -class TCMalloc_PageMap1 { - private: - static const int LENGTH = 1 << BITS; - - void** array_; - - public: - typedef uintptr_t Number; - - explicit TCMalloc_PageMap1(void* (*allocator)(size_t)) { - array_ = reinterpret_cast((*allocator)(sizeof(void*) << BITS)); - memset(array_, 0, sizeof(void*) << BITS); - } - - // Ensure that the map contains initialized entries "x .. x+n-1". - // Returns true if successful, false if we could not allocate memory. - bool Ensure(Number x, size_t n) { - // Nothing to do since flat array was allocated at start. All - // that's left is to check for overflow (that is, we don't want to - // ensure a number y where array_[y] would be an out-of-bounds - // access). - return n <= LENGTH - x; // an overflow-free way to do "x + n <= LENGTH" - } - - void PreallocateMoreMemory() {} - - // Return the current value for KEY. Returns NULL if not yet set, - // or if k is out of range. - void* get(Number k) const { - if ((k >> BITS) > 0) { - return NULL; - } - return array_[k]; - } - - // REQUIRES "k" is in range "[0,2^BITS-1]". - // REQUIRES "k" has been ensured before. - // - // Sets the value 'v' for key 'k'. - void set(Number k, void* v) { - array_[k] = v; - } - - // Return the first non-NULL pointer found in this map for - // a page number >= k. Returns NULL if no such number is found. - void* Next(Number k) const { - while (k < (1 << BITS)) { - if (array_[k] != NULL) return array_[k]; - k++; - } - return NULL; - } -}; - -// Two-level radix tree -template -class TCMalloc_PageMap2 { - private: - // Put 32 entries in the root and (2^BITS)/32 entries in each leaf. - static const int ROOT_BITS = 5; - static const int ROOT_LENGTH = 1 << ROOT_BITS; - - static const int LEAF_BITS = BITS - ROOT_BITS; - static const int LEAF_LENGTH = 1 << LEAF_BITS; - - // Leaf node - struct Leaf { - void* values[LEAF_LENGTH]; - }; - - Leaf* root_[ROOT_LENGTH]; // Pointers to 32 child nodes - void* (*allocator_)(size_t); // Memory allocator - - public: - typedef uintptr_t Number; - - explicit TCMalloc_PageMap2(void* (*allocator)(size_t)) { - allocator_ = allocator; - memset(root_, 0, sizeof(root_)); - } - - void* get(Number k) const { - const Number i1 = k >> LEAF_BITS; - const Number i2 = k & (LEAF_LENGTH-1); - if ((k >> BITS) > 0 || root_[i1] == NULL) { - return NULL; - } - return root_[i1]->values[i2]; - } - - void set(Number k, void* v) { - const Number i1 = k >> LEAF_BITS; - const Number i2 = k & (LEAF_LENGTH-1); - ASSERT(i1 < ROOT_LENGTH); - root_[i1]->values[i2] = v; - } - - bool Ensure(Number start, size_t n) { - for (Number key = start; key <= start + n - 1; ) { - const Number i1 = key >> LEAF_BITS; - - // Check for overflow - if (i1 >= ROOT_LENGTH) - return false; - - // Make 2nd level node if necessary - if (root_[i1] == NULL) { - Leaf* leaf = reinterpret_cast((*allocator_)(sizeof(Leaf))); - if (leaf == NULL) return false; - memset(leaf, 0, sizeof(*leaf)); - root_[i1] = leaf; - } - - // Advance key past whatever is covered by this leaf node - key = ((key >> LEAF_BITS) + 1) << LEAF_BITS; - } - return true; - } - - void PreallocateMoreMemory() { - // Allocate enough to keep track of all possible pages - Ensure(0, 1 << BITS); - } - - void* Next(Number k) const { - while (k < (1 << BITS)) { - const Number i1 = k >> LEAF_BITS; - Leaf* leaf = root_[i1]; - if (leaf != NULL) { - // Scan forward in leaf - for (Number i2 = k & (LEAF_LENGTH - 1); i2 < LEAF_LENGTH; i2++) { - if (leaf->values[i2] != NULL) { - return leaf->values[i2]; - } - } - } - // Skip to next top-level entry - k = (i1 + 1) << LEAF_BITS; - } - return NULL; - } -}; - -// Three-level radix tree -template -class TCMalloc_PageMap3 { - private: - // How many bits should we consume at each interior level - static const int INTERIOR_BITS = (BITS + 2) / 3; // Round-up - static const int INTERIOR_LENGTH = 1 << INTERIOR_BITS; - - // How many bits should we consume at leaf level - static const int LEAF_BITS = BITS - 2*INTERIOR_BITS; - static const int LEAF_LENGTH = 1 << LEAF_BITS; - - // Interior node - struct Node { - Node* ptrs[INTERIOR_LENGTH]; - }; - - // Leaf node - struct Leaf { - void* values[LEAF_LENGTH]; - }; - - Node* root_; // Root of radix tree - void* (*allocator_)(size_t); // Memory allocator - - Node* NewNode() { - Node* result = reinterpret_cast((*allocator_)(sizeof(Node))); - if (result != NULL) { - memset(result, 0, sizeof(*result)); - } - return result; - } - - public: - typedef uintptr_t Number; - - explicit TCMalloc_PageMap3(void* (*allocator)(size_t)) { - allocator_ = allocator; - root_ = NewNode(); - } - - void* get(Number k) const { - const Number i1 = k >> (LEAF_BITS + INTERIOR_BITS); - const Number i2 = (k >> LEAF_BITS) & (INTERIOR_LENGTH-1); - const Number i3 = k & (LEAF_LENGTH-1); - if ((k >> BITS) > 0 || - root_->ptrs[i1] == NULL || root_->ptrs[i1]->ptrs[i2] == NULL) { - return NULL; - } - return reinterpret_cast(root_->ptrs[i1]->ptrs[i2])->values[i3]; - } - - void set(Number k, void* v) { - ASSERT(k >> BITS == 0); - const Number i1 = k >> (LEAF_BITS + INTERIOR_BITS); - const Number i2 = (k >> LEAF_BITS) & (INTERIOR_LENGTH-1); - const Number i3 = k & (LEAF_LENGTH-1); - reinterpret_cast(root_->ptrs[i1]->ptrs[i2])->values[i3] = v; - } - - bool Ensure(Number start, size_t n) { - for (Number key = start; key <= start + n - 1; ) { - const Number i1 = key >> (LEAF_BITS + INTERIOR_BITS); - const Number i2 = (key >> LEAF_BITS) & (INTERIOR_LENGTH-1); - - // Check for overflow - if (i1 >= INTERIOR_LENGTH || i2 >= INTERIOR_LENGTH) - return false; - - // Make 2nd level node if necessary - if (root_->ptrs[i1] == NULL) { - Node* n = NewNode(); - if (n == NULL) return false; - root_->ptrs[i1] = n; - } - - // Make leaf node if necessary - if (root_->ptrs[i1]->ptrs[i2] == NULL) { - Leaf* leaf = reinterpret_cast((*allocator_)(sizeof(Leaf))); - if (leaf == NULL) return false; - memset(leaf, 0, sizeof(*leaf)); - root_->ptrs[i1]->ptrs[i2] = reinterpret_cast(leaf); - } - - // Advance key past whatever is covered by this leaf node - key = ((key >> LEAF_BITS) + 1) << LEAF_BITS; - } - return true; - } - - void PreallocateMoreMemory() { - } - - void* Next(Number k) const { - while (k < (Number(1) << BITS)) { - const Number i1 = k >> (LEAF_BITS + INTERIOR_BITS); - const Number i2 = (k >> LEAF_BITS) & (INTERIOR_LENGTH-1); - if (root_->ptrs[i1] == NULL) { - // Advance to next top-level entry - k = (i1 + 1) << (LEAF_BITS + INTERIOR_BITS); - } else { - Leaf* leaf = reinterpret_cast(root_->ptrs[i1]->ptrs[i2]); - if (leaf != NULL) { - for (Number i3 = (k & (LEAF_LENGTH-1)); i3 < LEAF_LENGTH; i3++) { - if (leaf->values[i3] != NULL) { - return leaf->values[i3]; - } - } - } - // Advance to next interior entry - k = ((k >> LEAF_BITS) + 1) << LEAF_BITS; - } - } - return NULL; - } -}; - -#endif // TCMALLOC_PAGEMAP_H_ diff --git a/contrib/libtcmalloc/src/raw_printer.cc b/contrib/libtcmalloc/src/raw_printer.cc deleted file mode 100644 index 3cf028eeae0..00000000000 --- a/contrib/libtcmalloc/src/raw_printer.cc +++ /dev/null @@ -1,72 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: sanjay@google.com (Sanjay Ghemawat) - -#include -#include -#include -#include "raw_printer.h" -#include "base/logging.h" - -namespace base { - -RawPrinter::RawPrinter(char* buf, int length) - : base_(buf), - ptr_(buf), - limit_(buf + length - 1) { - RAW_DCHECK(length > 0, ""); - *ptr_ = '\0'; - *limit_ = '\0'; -} - -void RawPrinter::Printf(const char* format, ...) { - if (limit_ > ptr_) { - va_list ap; - va_start(ap, format); - int avail = limit_ - ptr_; - // We pass avail+1 to vsnprintf() since that routine needs room - // to store the trailing \0. - const int r = perftools_vsnprintf(ptr_, avail+1, format, ap); - va_end(ap); - if (r < 0) { - // Perhaps an old glibc that returns -1 on truncation? - ptr_ = limit_; - } else if (r > avail) { - // Truncation - ptr_ = limit_; - } else { - ptr_ += r; - } - } -} - -} diff --git a/contrib/libtcmalloc/src/raw_printer.h b/contrib/libtcmalloc/src/raw_printer.h deleted file mode 100644 index 9288bb5eeaa..00000000000 --- a/contrib/libtcmalloc/src/raw_printer.h +++ /dev/null @@ -1,90 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat -// -// A printf() wrapper that writes into a fixed length buffer. -// Useful in low-level code that does not want to use allocating -// routines like StringPrintf(). -// -// The implementation currently uses vsnprintf(). This seems to -// be fine for use in many low-level contexts, but we may need to -// rethink this decision if we hit a problem with it calling -// down into malloc() etc. - -#ifndef BASE_RAW_PRINTER_H_ -#define BASE_RAW_PRINTER_H_ - -#include -#include "base/basictypes.h" - -namespace base { - -class RawPrinter { - public: - // REQUIRES: "length > 0" - // Will printf any data added to this into "buf[0,length-1]" and - // will arrange to always keep buf[] null-terminated. - RawPrinter(char* buf, int length); - - // Return the number of bytes that have been appended to the string - // so far. Does not count any bytes that were dropped due to overflow. - int length() const { return (ptr_ - base_); } - - // Return the number of bytes that can be added to this. - int space_left() const { return (limit_ - ptr_); } - - // Format the supplied arguments according to the "format" string - // and append to this. Will silently truncate the output if it does - // not fit. - void Printf(const char* format, ...) -#ifdef HAVE___ATTRIBUTE__ - __attribute__ ((__format__ (__printf__, 2, 3))) -#endif -; - - private: - // We can write into [ptr_ .. limit_-1]. - // *limit_ is also writable, but reserved for a terminating \0 - // in case we overflow. - // - // Invariants: *ptr_ == \0 - // Invariants: *limit_ == \0 - char* base_; // Initial pointer - char* ptr_; // Where should we write next - char* limit_; // One past last non-\0 char we can write - - DISALLOW_COPY_AND_ASSIGN(RawPrinter); -}; - -} - -#endif // BASE_RAW_PRINTER_H_ diff --git a/contrib/libtcmalloc/src/sampler.cc b/contrib/libtcmalloc/src/sampler.cc deleted file mode 100644 index cc711123340..00000000000 --- a/contrib/libtcmalloc/src/sampler.cc +++ /dev/null @@ -1,131 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// All Rights Reserved. -// -// Author: Daniel Ford - -#include "sampler.h" - -#include // For min() -#include -#include "base/commandlineflags.h" - -using std::min; - -// The approximate gap in bytes between sampling actions. -// I.e., we take one sample approximately once every -// tcmalloc_sample_parameter bytes of allocation -// i.e. about once every 512KB if value is 1<<19. -#ifdef NO_TCMALLOC_SAMPLES -DEFINE_int64(tcmalloc_sample_parameter, 0, - "Unused: code is compiled with NO_TCMALLOC_SAMPLES"); -#else -DEFINE_int64(tcmalloc_sample_parameter, - EnvToInt64("TCMALLOC_SAMPLE_PARAMETER", 0), - "The approximate gap in bytes between sampling actions. " - "This must be between 1 and 2^58."); -#endif - -namespace tcmalloc { - -// Statics for Sampler -double Sampler::log_table_[1<(i+0.5)/(1<(reinterpret_cast(this)); - if (rnd_ == 0) { - rnd_ = 1; - } - } - // Step it forward 20 times for good measure - for (int i = 0; i < 20; i++) { - rnd_ = NextRandom(rnd_); - } - // Initialize counter - bytes_until_sample_ = PickNextSamplingPoint(); -} - -// Initialize the Statics for the Sampler class -void Sampler::InitStatics() { - PopulateFastLog2Table(); -} - -// Generates a geometric variable with the specified mean (512K by default). -// This is done by generating a random number between 0 and 1 and applying -// the inverse cumulative distribution function for an exponential. -// Specifically: Let m be the inverse of the sample period, then -// the probability distribution function is m*exp(-mx) so the CDF is -// p = 1 - exp(-mx), so -// q = 1 - p = exp(-mx) -// log_e(q) = -mx -// -log_e(q)/m = x -// log_2(q) * (-log_e(2) * 1/m) = x -// In the code, q is actually in the range 1 to 2**26, hence the -26 below -size_t Sampler::PickNextSamplingPoint() { - rnd_ = NextRandom(rnd_); - // Take the top 26 bits as the random number - // (This plus the 1<<58 sampling bound give a max possible step of - // 5194297183973780480 bytes.) - const uint64_t prng_mod_power = 48; // Number of bits in prng - // The uint32_t cast is to prevent a (hard-to-reproduce) NAN - // under piii debug for some binaries. - double q = static_cast(rnd_ >> (prng_mod_power - 26)) + 1.0; - // Put the computed p-value through the CDF of a geometric. - // For faster performance (save ~1/20th exec time), replace - // min(0.0, FastLog2(q) - 26) by (Fastlog2(q) - 26.000705) - // The value 26.000705 is used rather than 26 to compensate - // for inaccuracies in FastLog2 which otherwise result in a - // negative answer. - return static_cast(min(0.0, (FastLog2(q) - 26)) * (-log(2.0) - * FLAGS_tcmalloc_sample_parameter) + 1); -} - -} // namespace tcmalloc diff --git a/contrib/libtcmalloc/src/sampler.h b/contrib/libtcmalloc/src/sampler.h deleted file mode 100644 index eb316d7493d..00000000000 --- a/contrib/libtcmalloc/src/sampler.h +++ /dev/null @@ -1,180 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// All Rights Reserved. -// -// Author: Daniel Ford - -#ifndef TCMALLOC_SAMPLER_H_ -#define TCMALLOC_SAMPLER_H_ - -#include "config.h" -#include // for size_t -#ifdef HAVE_STDINT_H -#include // for uint64_t, uint32_t, int32_t -#endif -#include // for memcpy -#include "base/basictypes.h" // for ASSERT -#include "internal_logging.h" // for ASSERT - -namespace tcmalloc { - -//------------------------------------------------------------------- -// Sampler to decide when to create a sample trace for an allocation -// Not thread safe: Each thread should have it's own sampler object. -// Caller must use external synchronization if used -// from multiple threads. -// -// With 512K average sample step (the default): -// the probability of sampling a 4K allocation is about 0.00778 -// the probability of sampling a 1MB allocation is about 0.865 -// the probability of sampling a 1GB allocation is about 1.00000 -// In general, the probablity of sampling is an allocation of size X -// given a flag value of Y (default 1M) is: -// 1 - e^(-X/Y) -// -// With 128K average sample step: -// the probability of sampling a 1MB allocation is about 0.99966 -// the probability of sampling a 1GB allocation is about 1.0 -// (about 1 - 2**(-26)) -// With 1M average sample step: -// the probability of sampling a 4K allocation is about 0.00390 -// the probability of sampling a 1MB allocation is about 0.632 -// the probability of sampling a 1GB allocation is about 1.0 -// -// The sampler works by representing memory as a long stream from -// which allocations are taken. Some of the bytes in this stream are -// marked and if an allocation includes a marked byte then it is -// sampled. Bytes are marked according to a Poisson point process -// with each byte being marked independently with probability -// p = 1/tcmalloc_sample_parameter. This makes the probability -// of sampling an allocation of X bytes equal to the CDF of -// a geometric with mean tcmalloc_sample_parameter. (ie. the -// probability that at least one byte in the range is marked). This -// is accurately given by the CDF of the corresponding exponential -// distribution : 1 - e^(X/tcmalloc_sample_parameter_) -// Independence of the byte marking ensures independence of -// the sampling of each allocation. -// -// This scheme is implemented by noting that, starting from any -// fixed place, the number of bytes until the next marked byte -// is geometrically distributed. This number is recorded as -// bytes_until_sample_. Every allocation subtracts from this -// number until it is less than 0. When this happens the current -// allocation is sampled. -// -// When an allocation occurs, bytes_until_sample_ is reset to -// a new independtly sampled geometric number of bytes. The -// memoryless property of the point process means that this may -// be taken as the number of bytes after the end of the current -// allocation until the next marked byte. This ensures that -// very large allocations which would intersect many marked bytes -// only result in a single call to PickNextSamplingPoint. -//------------------------------------------------------------------- - -class PERFTOOLS_DLL_DECL Sampler { - public: - // Initialize this sampler. - // Passing a seed of 0 gives a non-deterministic - // seed value given by casting the object ("this") - void Init(uint32_t seed); - void Cleanup(); - - // Record allocation of "k" bytes. Return true iff allocation - // should be sampled - bool SampleAllocation(size_t k); - - // Generate a geometric with mean 512K (or FLAG_tcmalloc_sample_parameter) - size_t PickNextSamplingPoint(); - - // Initialize the statics for the Sampler class - static void InitStatics(); - - // Returns the current sample period - int GetSamplePeriod(); - - // The following are public for the purposes of testing - static uint64_t NextRandom(uint64_t rnd_); // Returns the next prng value - static double FastLog2(const double & d); // Computes Log2(x) quickly - static void PopulateFastLog2Table(); // Populate the lookup table - - private: - size_t bytes_until_sample_; // Bytes until we sample next - uint64_t rnd_; // Cheap random number generator - - // Statics for the fast log - // Note that this code may not depend on anything in //util - // hence the duplication of functionality here - static const int kFastlogNumBits = 10; - static const int kFastlogMask = (1 << kFastlogNumBits) - 1; - static double log_table_[1<(0)) << prng_mod_power); - return (prng_mult * rnd + prng_add) & prng_mod_mask; -} - -// Adapted from //util/math/fastmath.[h|cc] by Noam Shazeer -// This mimics the VeryFastLog2 code in those files -inline double Sampler::FastLog2(const double & d) { - ASSERT(d>0); - COMPILE_ASSERT(sizeof(d) == sizeof(uint64_t), DoubleMustBe64Bits); - uint64_t x; - memcpy(&x, &d, sizeof(x)); // we depend on the compiler inlining this - const uint32_t x_high = x >> 32; - const uint32_t y = x_high >> (20 - kFastlogNumBits) & kFastlogMask; - const int32_t exponent = ((x_high >> 20) & 0x7FF) - 1023; - return exponent + log_table_[y]; -} - -} // namespace tcmalloc - -#endif // TCMALLOC_SAMPLER_H_ diff --git a/contrib/libtcmalloc/src/span.cc b/contrib/libtcmalloc/src/span.cc deleted file mode 100644 index 5f7ae436086..00000000000 --- a/contrib/libtcmalloc/src/span.cc +++ /dev/null @@ -1,102 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat - -#include "config.h" -#include "span.h" - -#include // for NULL, memset - -#include "internal_logging.h" // for ASSERT -#include "page_heap_allocator.h" // for PageHeapAllocator -#include "static_vars.h" // for Static - -namespace tcmalloc { - -#ifdef SPAN_HISTORY -void Event(Span* span, char op, int v = 0) { - span->history[span->nexthistory] = op; - span->value[span->nexthistory] = v; - span->nexthistory++; - if (span->nexthistory == sizeof(span->history)) span->nexthistory = 0; -} -#endif - -Span* NewSpan(PageID p, Length len) { - Span* result = Static::span_allocator()->New(); - memset(result, 0, sizeof(*result)); - result->start = p; - result->length = len; -#ifdef SPAN_HISTORY - result->nexthistory = 0; -#endif - return result; -} - -void DeleteSpan(Span* span) { -#ifndef NDEBUG - // In debug mode, trash the contents of deleted Spans - memset(span, 0x3f, sizeof(*span)); -#endif - Static::span_allocator()->Delete(span); -} - -void DLL_Init(Span* list) { - list->next = list; - list->prev = list; -} - -void DLL_Remove(Span* span) { - span->prev->next = span->next; - span->next->prev = span->prev; - span->prev = NULL; - span->next = NULL; -} - -int DLL_Length(const Span* list) { - int result = 0; - for (Span* s = list->next; s != list; s = s->next) { - result++; - } - return result; -} - -void DLL_Prepend(Span* list, Span* span) { - ASSERT(span->next == NULL); - ASSERT(span->prev == NULL); - span->next = list->next; - span->prev = list; - list->next->prev = span; - list->next = span; -} - -} // namespace tcmalloc diff --git a/contrib/libtcmalloc/src/span.h b/contrib/libtcmalloc/src/span.h deleted file mode 100644 index 3fe30ba33d0..00000000000 --- a/contrib/libtcmalloc/src/span.h +++ /dev/null @@ -1,102 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat -// -// A Span is a contiguous run of pages. - -#ifndef TCMALLOC_SPAN_H_ -#define TCMALLOC_SPAN_H_ - -#include "config.h" -#include "common.h" - -namespace tcmalloc { - -// Information kept for a span (a contiguous run of pages). -struct Span { - PageID start; // Starting page number - Length length; // Number of pages in span - Span* next; // Used when in link list - Span* prev; // Used when in link list - void* objects; // Linked list of free objects - unsigned int refcount : 16; // Number of non-free objects - unsigned int sizeclass : 8; // Size-class for small objects (or 0) - unsigned int location : 2; // Is the span on a freelist, and if so, which? - unsigned int sample : 1; // Sampled object? - -#undef SPAN_HISTORY -#ifdef SPAN_HISTORY - // For debugging, we can keep a log events per span - int nexthistory; - char history[64]; - int value[64]; -#endif - - // What freelist the span is on: IN_USE if on none, or normal or returned - enum { IN_USE, ON_NORMAL_FREELIST, ON_RETURNED_FREELIST }; -}; - -#ifdef SPAN_HISTORY -void Event(Span* span, char op, int v = 0); -#else -#define Event(s,o,v) ((void) 0) -#endif - -// Allocator/deallocator for spans -Span* NewSpan(PageID p, Length len); -void DeleteSpan(Span* span); - -// ------------------------------------------------------------------------- -// Doubly linked list of spans. -// ------------------------------------------------------------------------- - -// Initialize *list to an empty list. -void DLL_Init(Span* list); - -// Remove 'span' from the linked list in which it resides, updating the -// pointers of adjacent Spans and setting span's next and prev to NULL. -void DLL_Remove(Span* span); - -// Return true iff "list" is empty. -inline bool DLL_IsEmpty(const Span* list) { - return list->next == list; -} - -// Add span to the front of list. -void DLL_Prepend(Span* list, Span* span); - -// Return the length of the linked list. O(n) -int DLL_Length(const Span* list); - -} // namespace tcmalloc - -#endif // TCMALLOC_SPAN_H_ diff --git a/contrib/libtcmalloc/src/stack_trace_table.cc b/contrib/libtcmalloc/src/stack_trace_table.cc deleted file mode 100644 index 049cca524b5..00000000000 --- a/contrib/libtcmalloc/src/stack_trace_table.cc +++ /dev/null @@ -1,160 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2009, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Andrew Fikes - -#include "config.h" -#include "stack_trace_table.h" -#include // for NULL, memset -#include "base/spinlock.h" // for SpinLockHolder -#include "common.h" // for StackTrace -#include "internal_logging.h" // for ASSERT, Log -#include "page_heap_allocator.h" // for PageHeapAllocator -#include "static_vars.h" // for Static - -namespace tcmalloc { - -bool StackTraceTable::Bucket::KeyEqual(uintptr_t h, - const StackTrace& t) const { - const bool eq = (this->hash == h && this->trace.depth == t.depth); - for (int i = 0; eq && i < t.depth; ++i) { - if (this->trace.stack[i] != t.stack[i]) { - return false; - } - } - return eq; -} - -StackTraceTable::StackTraceTable() - : error_(false), - depth_total_(0), - bucket_total_(0), - table_(new Bucket*[kHashTableSize]()) { - memset(table_, 0, kHashTableSize * sizeof(Bucket*)); -} - -StackTraceTable::~StackTraceTable() { - delete[] table_; -} - -void StackTraceTable::AddTrace(const StackTrace& t) { - if (error_) { - return; - } - - // Hash function borrowed from base/heap-profile-table.cc - uintptr_t h = 0; - for (int i = 0; i < t.depth; ++i) { - h += reinterpret_cast(t.stack[i]); - h += h << 10; - h ^= h >> 6; - } - h += h << 3; - h ^= h >> 11; - - const int idx = h % kHashTableSize; - - Bucket* b = table_[idx]; - while (b != NULL && !b->KeyEqual(h, t)) { - b = b->next; - } - if (b != NULL) { - b->count++; - b->trace.size += t.size; // keep cumulative size - } else { - depth_total_ += t.depth; - bucket_total_++; - b = Static::bucket_allocator()->New(); - if (b == NULL) { - Log(kLog, __FILE__, __LINE__, - "tcmalloc: could not allocate bucket", sizeof(*b)); - error_ = true; - } else { - b->hash = h; - b->trace = t; - b->count = 1; - b->next = table_[idx]; - table_[idx] = b; - } - } -} - -void** StackTraceTable::ReadStackTracesAndClear() { - if (error_) { - return NULL; - } - - // Allocate output array - const int out_len = bucket_total_ * 3 + depth_total_ + 1; - void** out = new void*[out_len]; - if (out == NULL) { - Log(kLog, __FILE__, __LINE__, - "tcmalloc: allocation failed for stack traces", - out_len * sizeof(*out)); - return NULL; - } - - // Fill output array - int idx = 0; - for (int i = 0; i < kHashTableSize; ++i) { - Bucket* b = table_[i]; - while (b != NULL) { - out[idx++] = reinterpret_cast(static_cast(b->count)); - out[idx++] = reinterpret_cast(b->trace.size); // cumulative size - out[idx++] = reinterpret_cast(b->trace.depth); - for (int d = 0; d < b->trace.depth; ++d) { - out[idx++] = b->trace.stack[d]; - } - b = b->next; - } - } - out[idx++] = NULL; - ASSERT(idx == out_len); - - // Clear state - error_ = false; - depth_total_ = 0; - bucket_total_ = 0; - SpinLockHolder h(Static::pageheap_lock()); - for (int i = 0; i < kHashTableSize; ++i) { - Bucket* b = table_[i]; - while (b != NULL) { - Bucket* next = b->next; - Static::bucket_allocator()->Delete(b); - b = next; - } - table_[i] = NULL; - } - - return out; -} - -} // namespace tcmalloc diff --git a/contrib/libtcmalloc/src/stack_trace_table.h b/contrib/libtcmalloc/src/stack_trace_table.h deleted file mode 100644 index 66ed5d92822..00000000000 --- a/contrib/libtcmalloc/src/stack_trace_table.h +++ /dev/null @@ -1,92 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2009, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Andrew Fikes -// -// Utility class for coalescing sampled stack traces. Not thread-safe. - -#ifndef TCMALLOC_STACK_TRACE_TABLE_H_ -#define TCMALLOC_STACK_TRACE_TABLE_H_ - -#include "config.h" -#ifdef HAVE_STDINT_H -#include // for uintptr_t -#endif -#include "common.h" - -namespace tcmalloc { - -class PERFTOOLS_DLL_DECL StackTraceTable { - public: - // REQUIRES: L < pageheap_lock - StackTraceTable(); - ~StackTraceTable(); - - // Adds stack trace "t" to table. - // - // REQUIRES: L >= pageheap_lock - void AddTrace(const StackTrace& t); - - // Returns stack traces formatted per MallocExtension guidelines. - // May return NULL on error. Clears state before returning. - // - // REQUIRES: L < pageheap_lock - void** ReadStackTracesAndClear(); - - // Exposed for PageHeapAllocator - struct Bucket { - // Key - uintptr_t hash; - StackTrace trace; - - // Payload - int count; - Bucket* next; - - bool KeyEqual(uintptr_t h, const StackTrace& t) const; - }; - - // For testing - int depth_total() const { return depth_total_; } - int bucket_total() const { return bucket_total_; } - - private: - static const int kHashTableSize = 1 << 14; // => table_ is 128k - - bool error_; - int depth_total_; - int bucket_total_; - Bucket** table_; -}; - -} // namespace tcmalloc - -#endif // TCMALLOC_STACK_TRACE_TABLE_H_ diff --git a/contrib/libtcmalloc/src/stacktrace.cc b/contrib/libtcmalloc/src/stacktrace.cc deleted file mode 100644 index 88c8b15946d..00000000000 --- a/contrib/libtcmalloc/src/stacktrace.cc +++ /dev/null @@ -1,339 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat -// -// Produce stack trace. -// -// There are three different ways we can try to get the stack trace: -// -// 1) Our hand-coded stack-unwinder. This depends on a certain stack -// layout, which is used by gcc (and those systems using a -// gcc-compatible ABI) on x86 systems, at least since gcc 2.95. -// It uses the frame pointer to do its work. -// -// 2) The libunwind library. This is still in development, and as a -// separate library adds a new dependency, abut doesn't need a frame -// pointer. It also doesn't call malloc. -// -// 3) The gdb unwinder -- also the one used by the c++ exception code. -// It's obviously well-tested, but has a fatal flaw: it can call -// malloc() from the unwinder. This is a problem because we're -// trying to use the unwinder to instrument malloc(). -// -// Note: if you add a new implementation here, make sure it works -// correctly when GetStackTrace() is called with max_depth == 0. -// Some code may do that. - -#include "config.h" -#include // for getenv -#include // for strcmp -#include // for fprintf -#include "gperftools/stacktrace.h" -#include "base/commandlineflags.h" -#include "base/googleinit.h" - - -// we're using plain struct and not class to avoid any possible issues -// during initialization. Struct of pointers is easy to init at -// link-time. -struct GetStackImplementation { - int (*GetStackFramesPtr)(void** result, int* sizes, int max_depth, - int skip_count); - - int (*GetStackFramesWithContextPtr)(void** result, int* sizes, int max_depth, - int skip_count, const void *uc); - - int (*GetStackTracePtr)(void** result, int max_depth, - int skip_count); - - int (*GetStackTraceWithContextPtr)(void** result, int max_depth, - int skip_count, const void *uc); - - const char *name; -}; - -#if HAVE_DECL_BACKTRACE -#define STACKTRACE_INL_HEADER "stacktrace_generic-inl.h" -#define GST_SUFFIX generic -#include "stacktrace_impl_setup-inl.h" -#undef GST_SUFFIX -#undef STACKTRACE_INL_HEADER -#define HAVE_GST_generic -#endif - -#ifdef HAVE_UNWIND_BACKTRACE -#define STACKTRACE_INL_HEADER "stacktrace_libgcc-inl.h" -#define GST_SUFFIX libgcc -#include "stacktrace_impl_setup-inl.h" -#undef GST_SUFFIX -#undef STACKTRACE_INL_HEADER -#define HAVE_GST_libgcc -#endif - -// libunwind uses __thread so we check for both libunwind.h and -// __thread support -#if defined(HAVE_LIBUNWIND_H) && defined(HAVE_TLS) -#define STACKTRACE_INL_HEADER "stacktrace_libunwind-inl.h" -#define GST_SUFFIX libunwind -#include "stacktrace_impl_setup-inl.h" -#undef GST_SUFFIX -#undef STACKTRACE_INL_HEADER -#define HAVE_GST_libunwind -#endif // HAVE_LIBUNWIND_H - -#if defined(__i386__) || defined(__x86_64__) -#define STACKTRACE_INL_HEADER "stacktrace_x86-inl.h" -#define GST_SUFFIX x86 -#include "stacktrace_impl_setup-inl.h" -#undef GST_SUFFIX -#undef STACKTRACE_INL_HEADER -#define HAVE_GST_x86 -#endif // i386 || x86_64 - -#if defined(__ppc__) || defined(__PPC__) -#if defined(__linux__) -#define STACKTRACE_INL_HEADER "stacktrace_powerpc-linux-inl.h" -#else -#define STACKTRACE_INL_HEADER "stacktrace_powerpc-darwin-inl.h" -#endif -#define GST_SUFFIX ppc -#include "stacktrace_impl_setup-inl.h" -#undef GST_SUFFIX -#undef STACKTRACE_INL_HEADER -#define HAVE_GST_ppc -#endif - -#if defined(__arm__) -#define STACKTRACE_INL_HEADER "stacktrace_arm-inl.h" -#define GST_SUFFIX arm -#include "stacktrace_impl_setup-inl.h" -#undef GST_SUFFIX -#undef STACKTRACE_INL_HEADER -#define HAVE_GST_arm -#endif - -#ifdef TCMALLOC_ENABLE_INSTRUMENT_STACKTRACE -#define STACKTRACE_INL_HEADER "stacktrace_instrument-inl.h" -#define GST_SUFFIX instrument -#include "stacktrace_impl_setup-inl.h" -#undef GST_SUFFIX -#undef STACKTRACE_INL_HEADER -#define HAVE_GST_instrument -#endif - -// The Windows case -- probably cygwin and mingw will use one of the -// x86-includes above, but if not, we can fall back to windows intrinsics. -#if defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__MINGW32__) -#define STACKTRACE_INL_HEADER "stacktrace_win32-inl.h" -#define GST_SUFFIX win32 -#include "stacktrace_impl_setup-inl.h" -#undef GST_SUFFIX -#undef STACKTRACE_INL_HEADER -#define HAVE_GST_win32 -#endif - -static GetStackImplementation *all_impls[] = { -#ifdef HAVE_GST_libgcc - &impl__libgcc, -#endif -#ifdef HAVE_GST_generic - &impl__generic, -#endif -#ifdef HAVE_GST_libunwind - &impl__libunwind, -#endif -#ifdef HAVE_GST_x86 - &impl__x86, -#endif -#ifdef HAVE_GST_arm - &impl__arm, -#endif -#ifdef HAVE_GST_ppc - &impl__ppc, -#endif -#ifdef HAVE_GST_instrument - &impl__instrument, -#endif -#ifdef HAVE_GST_win32 - &impl__win32, -#endif - NULL -}; - -// ppc and i386 implementations prefer arch-specific asm implementations. -// arm's asm implementation is broken -#if defined(__i386__) || defined(__x86_64__) || defined(__ppc__) || defined(__PPC__) -#if !defined(NO_FRAME_POINTER) -#define TCMALLOC_DONT_PREFER_LIBUNWIND -#endif -#endif - -static bool get_stack_impl_inited; - -#if defined(HAVE_GST_instrument) -static GetStackImplementation *get_stack_impl = &impl__instrument; -#elif defined(HAVE_GST_win32) -static GetStackImplementation *get_stack_impl = &impl__win32; -#elif defined(HAVE_GST_x86) && defined(TCMALLOC_DONT_PREFER_LIBUNWIND) -static GetStackImplementation *get_stack_impl = &impl__x86; -#elif defined(HAVE_GST_ppc) && defined(TCMALLOC_DONT_PREFER_LIBUNWIND) -static GetStackImplementation *get_stack_impl = &impl__ppc; -#elif defined(HAVE_GST_libunwind) -static GetStackImplementation *get_stack_impl = &impl__libunwind; -#elif defined(HAVE_GST_libgcc) -static GetStackImplementation *get_stack_impl = &impl__libgcc; -#elif defined(HAVE_GST_generic) -static GetStackImplementation *get_stack_impl = &impl__generic; -#elif defined(HAVE_GST_arm) -static GetStackImplementation *get_stack_impl = &impl__arm; -#elif 0 -// This is for the benefit of code analysis tools that may have -// trouble with the computed #include above. -# include "stacktrace_x86-inl.h" -# include "stacktrace_libunwind-inl.h" -# include "stacktrace_generic-inl.h" -# include "stacktrace_powerpc-inl.h" -# include "stacktrace_win32-inl.h" -# include "stacktrace_arm-inl.h" -# include "stacktrace_instrument-inl.h" -#else -#error Cannot calculate stack trace: will need to write for your environment -#endif - -static int ATTRIBUTE_NOINLINE frame_forcer(int rv) { - return rv; -} - -static void init_default_stack_impl_inner(void); - -namespace tcmalloc { - bool EnterStacktraceScope(void); - void LeaveStacktraceScope(void); -} - -namespace { - using tcmalloc::EnterStacktraceScope; - using tcmalloc::LeaveStacktraceScope; - - class StacktraceScope { - bool stacktrace_allowed; - public: - StacktraceScope() { - stacktrace_allowed = true; - stacktrace_allowed = EnterStacktraceScope(); - } - bool IsStacktraceAllowed() { - return stacktrace_allowed; - } - ~StacktraceScope() { - if (stacktrace_allowed) { - LeaveStacktraceScope(); - } - } - }; -} - -PERFTOOLS_DLL_DECL int GetStackFrames(void** result, int* sizes, int max_depth, - int skip_count) { - StacktraceScope scope; - if (!scope.IsStacktraceAllowed()) { - return 0; - } - init_default_stack_impl_inner(); - return frame_forcer(get_stack_impl->GetStackFramesPtr(result, sizes, max_depth, skip_count)); -} - -PERFTOOLS_DLL_DECL int GetStackFramesWithContext(void** result, int* sizes, int max_depth, - int skip_count, const void *uc) { - StacktraceScope scope; - if (!scope.IsStacktraceAllowed()) { - return 0; - } - init_default_stack_impl_inner(); - return frame_forcer(get_stack_impl->GetStackFramesWithContextPtr( - result, sizes, max_depth, - skip_count, uc)); -} - -PERFTOOLS_DLL_DECL int GetStackTrace(void** result, int max_depth, - int skip_count) { - StacktraceScope scope; - if (!scope.IsStacktraceAllowed()) { - return 0; - } - init_default_stack_impl_inner(); - return frame_forcer(get_stack_impl->GetStackTracePtr(result, max_depth, skip_count)); -} - -PERFTOOLS_DLL_DECL int GetStackTraceWithContext(void** result, int max_depth, - int skip_count, const void *uc) { - StacktraceScope scope; - if (!scope.IsStacktraceAllowed()) { - return 0; - } - init_default_stack_impl_inner(); - return frame_forcer(get_stack_impl->GetStackTraceWithContextPtr( - result, max_depth, skip_count, uc)); -} - -static void init_default_stack_impl_inner(void) { - if (get_stack_impl_inited) { - return; - } - get_stack_impl_inited = true; - char *val = getenv("TCMALLOC_STACKTRACE_METHOD"); - if (!val || !*val) { - return; - } - for (GetStackImplementation **p = all_impls; *p; p++) { - GetStackImplementation *c = *p; - if (strcmp(c->name, val) == 0) { - get_stack_impl = c; - return; - } - } - fprintf(stderr, "Unknown or unsupported stacktrace method requested: %s. Ignoring it\n", val); -} - -static void init_default_stack_impl(void) { - init_default_stack_impl_inner(); - if (EnvToBool("TCMALLOC_STACKTRACE_METHOD_VERBOSE", false)) { - fprintf(stderr, "Chosen stacktrace method is %s\nSupported methods:\n", get_stack_impl->name); - for (GetStackImplementation **p = all_impls; *p; p++) { - GetStackImplementation *c = *p; - fprintf(stderr, "* %s\n", c->name); - } - fputs("\n", stderr); - } -} - -REGISTER_MODULE_INITIALIZER(stacktrace_init_default_stack_impl, init_default_stack_impl()); diff --git a/contrib/libtcmalloc/src/stacktrace_arm-inl.h b/contrib/libtcmalloc/src/stacktrace_arm-inl.h deleted file mode 100644 index 1586b8fec62..00000000000 --- a/contrib/libtcmalloc/src/stacktrace_arm-inl.h +++ /dev/null @@ -1,148 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2011, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Doug Kwan -// This is inspired by Craig Silverstein's PowerPC stacktrace code. -// - -#ifndef BASE_STACKTRACE_ARM_INL_H_ -#define BASE_STACKTRACE_ARM_INL_H_ -// Note: this file is included into stacktrace.cc more than once. -// Anything that should only be defined once should be here: - -#include // for uintptr_t -#include "base/basictypes.h" // for NULL -#include - -// WARNING: -// This only works if all your code is in either ARM or THUMB mode. With -// interworking, the frame pointer of the caller can either be in r11 (ARM -// mode) or r7 (THUMB mode). A callee only saves the frame pointer of its -// mode in a fixed location on its stack frame. If the caller is a different -// mode, there is no easy way to find the frame pointer. It can either be -// still in the designated register or saved on stack along with other callee -// saved registers. - -// Given a pointer to a stack frame, locate and return the calling -// stackframe, or return NULL if no stackframe can be found. Perform sanity -// checks (the strictness of which is controlled by the boolean parameter -// "STRICT_UNWINDING") to reduce the chance that a bad pointer is returned. -template -static void **NextStackFrame(void **old_sp) { - void **new_sp = (void**) old_sp[-1]; - - // Check that the transition from frame pointer old_sp to frame - // pointer new_sp isn't clearly bogus - if (STRICT_UNWINDING) { - // With the stack growing downwards, older stack frame must be - // at a greater address that the current one. - if (new_sp <= old_sp) return NULL; - // Assume stack frames larger than 100,000 bytes are bogus. - if ((uintptr_t)new_sp - (uintptr_t)old_sp > 100000) return NULL; - } else { - // In the non-strict mode, allow discontiguous stack frames. - // (alternate-signal-stacks for example). - if (new_sp == old_sp) return NULL; - // And allow frames upto about 1MB. - if ((new_sp > old_sp) - && ((uintptr_t)new_sp - (uintptr_t)old_sp > 1000000)) return NULL; - } - if ((uintptr_t)new_sp & (sizeof(void *) - 1)) return NULL; - return new_sp; -} - -// This ensures that GetStackTrace stes up the Link Register properly. -#ifdef __GNUC__ -void StacktraceArmDummyFunction() __attribute__((noinline)); -void StacktraceArmDummyFunction() { __asm__ volatile(""); } -#else -# error StacktraceArmDummyFunction() needs to be ported to this platform. -#endif -#endif // BASE_STACKTRACE_ARM_INL_H_ - -// Note: this part of the file is included several times. -// Do not put globals below. - -// The following 4 functions are generated from the code below: -// GetStack{Trace,Frames}() -// GetStack{Trace,Frames}WithContext() -// -// These functions take the following args: -// void** result: the stack-trace, as an array -// int* sizes: the size of each stack frame, as an array -// (GetStackFrames* only) -// int max_depth: the size of the result (and sizes) array(s) -// int skip_count: how many stack pointers to skip before storing in result -// void* ucp: a ucontext_t* (GetStack{Trace,Frames}WithContext only) -static int GET_STACK_TRACE_OR_FRAMES { -#ifdef __GNUC__ - void **sp = reinterpret_cast(__builtin_frame_address(0)); -#else -# error reading stack point not yet supported on this platform. -#endif - - // On ARM, the return address is stored in the link register (r14). - // This is not saved on the stack frame of a leaf function. To - // simplify code that reads return addresses, we call a dummy - // function so that the return address of this function is also - // stored in the stack frame. This works at least for gcc. - StacktraceArmDummyFunction(); - - skip_count++; // skip parent frame due to indirection in stacktrace.cc - - int n = 0; - while (sp && n < max_depth) { - // The GetStackFrames routine is called when we are in some - // informational context (the failure signal handler for example). - // Use the non-strict unwinding rules to produce a stack trace - // that is as complete as possible (even if it contains a few bogus - // entries in some rare cases). - void **next_sp = NextStackFrame(sp); - - if (skip_count > 0) { - skip_count--; - } else { - result[n] = *sp; - -#if IS_STACK_FRAMES - if (next_sp > sp) { - sizes[n] = (uintptr_t)next_sp - (uintptr_t)sp; - } else { - // A frame-size of 0 is used to indicate unknown frame size. - sizes[n] = 0; - } -#endif - n++; - } - sp = next_sp; - } - return n; -} diff --git a/contrib/libtcmalloc/src/stacktrace_generic-inl.h b/contrib/libtcmalloc/src/stacktrace_generic-inl.h deleted file mode 100644 index 7d7c22d9e45..00000000000 --- a/contrib/libtcmalloc/src/stacktrace_generic-inl.h +++ /dev/null @@ -1,84 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat -// -// Portable implementation - just use glibc -// -// Note: The glibc implementation may cause a call to malloc. -// This can cause a deadlock in HeapProfiler. - -#ifndef BASE_STACKTRACE_GENERIC_INL_H_ -#define BASE_STACKTRACE_GENERIC_INL_H_ -// Note: this file is included into stacktrace.cc more than once. -// Anything that should only be defined once should be here: - -#include -#include -#include "gperftools/stacktrace.h" -#endif // BASE_STACKTRACE_GENERIC_INL_H_ - -// Note: this part of the file is included several times. -// Do not put globals below. - -// The following 4 functions are generated from the code below: -// GetStack{Trace,Frames}() -// GetStack{Trace,Frames}WithContext() -// -// These functions take the following args: -// void** result: the stack-trace, as an array -// int* sizes: the size of each stack frame, as an array -// (GetStackFrames* only) -// int max_depth: the size of the result (and sizes) array(s) -// int skip_count: how many stack pointers to skip before storing in result -// void* ucp: a ucontext_t* (GetStack{Trace,Frames}WithContext only) -static int GET_STACK_TRACE_OR_FRAMES { - static const int kStackLength = 64; - void * stack[kStackLength]; - int size; - - size = backtrace(stack, kStackLength); - skip_count += 2; // we want to skip the current and it's parent frame as well - int result_count = size - skip_count; - if (result_count < 0) - result_count = 0; - if (result_count > max_depth) - result_count = max_depth; - for (int i = 0; i < result_count; i++) - result[i] = stack[i + skip_count]; - -#if IS_STACK_FRAMES - // No implementation for finding out the stack frame sizes yet. - memset(sizes, 0, sizeof(*sizes) * result_count); -#endif - - return result_count; -} diff --git a/contrib/libtcmalloc/src/stacktrace_impl_setup-inl.h b/contrib/libtcmalloc/src/stacktrace_impl_setup-inl.h deleted file mode 100644 index 698c5b38196..00000000000 --- a/contrib/libtcmalloc/src/stacktrace_impl_setup-inl.h +++ /dev/null @@ -1,94 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// NOTE: this is NOT to be #include-d normally. It's internal -// implementation detail of stacktrace.cc -// - -// Copyright (c) 2014, gperftools Contributors. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Aliaksey Kandratsenka -// -// based on stacktrace.cc and stacktrace_config.h by Sanjay Ghemawat -// and Paul Pluzhnikov from Google Inc - -#define SIS_CONCAT2(a, b) a##b -#define SIS_CONCAT(a, b) SIS_CONCAT2(a,b) - -#define SIS_STRINGIFY(a) SIS_STRINGIFY2(a) -#define SIS_STRINGIFY2(a) #a - -#define IS_STACK_FRAMES 0 -#define IS_WITH_CONTEXT 0 -#define GET_STACK_TRACE_OR_FRAMES \ - SIS_CONCAT(GetStackTrace_, GST_SUFFIX)(void **result, int max_depth, int skip_count) -#include STACKTRACE_INL_HEADER -#undef IS_STACK_FRAMES -#undef IS_WITH_CONTEXT -#undef GET_STACK_TRACE_OR_FRAMES - -#define IS_STACK_FRAMES 1 -#define IS_WITH_CONTEXT 0 -#define GET_STACK_TRACE_OR_FRAMES \ - SIS_CONCAT(GetStackFrames_, GST_SUFFIX)(void **result, int *sizes, int max_depth, int skip_count) -#include STACKTRACE_INL_HEADER -#undef IS_STACK_FRAMES -#undef IS_WITH_CONTEXT -#undef GET_STACK_TRACE_OR_FRAMES - -#define IS_STACK_FRAMES 0 -#define IS_WITH_CONTEXT 1 -#define GET_STACK_TRACE_OR_FRAMES \ - SIS_CONCAT(GetStackTraceWithContext_, GST_SUFFIX)(void **result, int max_depth, \ - int skip_count, const void *ucp) -#include STACKTRACE_INL_HEADER -#undef IS_STACK_FRAMES -#undef IS_WITH_CONTEXT -#undef GET_STACK_TRACE_OR_FRAMES - -#define IS_STACK_FRAMES 1 -#define IS_WITH_CONTEXT 1 -#define GET_STACK_TRACE_OR_FRAMES \ - SIS_CONCAT(GetStackFramesWithContext_, GST_SUFFIX)(void **result, int *sizes, int max_depth, \ - int skip_count, const void *ucp) -#include STACKTRACE_INL_HEADER -#undef IS_STACK_FRAMES -#undef IS_WITH_CONTEXT -#undef GET_STACK_TRACE_OR_FRAMES - -static GetStackImplementation SIS_CONCAT(impl__,GST_SUFFIX) = { - SIS_CONCAT(GetStackFrames_, GST_SUFFIX), - SIS_CONCAT(GetStackFramesWithContext_, GST_SUFFIX), - SIS_CONCAT(GetStackTrace_, GST_SUFFIX), - SIS_CONCAT(GetStackTraceWithContext_, GST_SUFFIX), - SIS_STRINGIFY(GST_SUFFIX) -}; - -#undef SIS_CONCAT2 -#undef SIS_CONCAT diff --git a/contrib/libtcmalloc/src/stacktrace_instrument-inl.h b/contrib/libtcmalloc/src/stacktrace_instrument-inl.h deleted file mode 100644 index c631765c8a2..00000000000 --- a/contrib/libtcmalloc/src/stacktrace_instrument-inl.h +++ /dev/null @@ -1,155 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2013, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Jean Lee -// based on gcc Code-Gen-Options "-finstrument-functions" listed in -// http://gcc.gnu.org/onlinedocs/gcc/Code-Gen-Options.html . -// Should run configure with CXXFLAGS="-finstrument-functions". - -// This file is a backtrace implementation for systems : -// * The glibc implementation of backtrace() may cause a call to malloc, -// and cause a deadlock in HeapProfiler. -// * The libunwind implementation prints no backtrace. - -// The backtrace arrays are stored in "thread_back_trace" variable. -// Maybe to use thread local storage is better and should save memorys. - -#ifndef BASE_STACKTRACE_INSTRUMENT_INL_H_ -#define BASE_STACKTRACE_INSTRUMENT_INL_H_ -// Note: this file is included into stacktrace.cc more than once. -// Anything that should only be defined once should be here: - -#include -#include -#include -#include -#include "gperftools/stacktrace.h" - -#define gettid() syscall(__NR_gettid) -#ifndef __x86_64__ -#define MAX_THREAD (32768) -#else -#define MAX_THREAD (65536) -#endif -#define MAX_DEPTH (30) -#define ATTRIBUTE_NOINSTRUMENT __attribute__ ((no_instrument_function)) - -typedef struct { - int stack_depth; - void* frame[MAX_DEPTH]; -}BACK_TRACE; - -static BACK_TRACE thread_back_trace[MAX_THREAD]; -extern "C" { -void __cyg_profile_func_enter(void *func_address, - void *call_site) ATTRIBUTE_NOINSTRUMENT; -void __cyg_profile_func_enter(void *func_address, void *call_site) { - (void)func_address; - - BACK_TRACE* backtrace = thread_back_trace + gettid(); - int stack_depth = backtrace->stack_depth; - backtrace->stack_depth = stack_depth + 1; - if ( stack_depth >= MAX_DEPTH ) { - return; - } - backtrace->frame[stack_depth] = call_site; -} - -void __cyg_profile_func_exit(void *func_address, - void *call_site) ATTRIBUTE_NOINSTRUMENT; -void __cyg_profile_func_exit(void *func_address, void *call_site) { - (void)func_address; - (void)call_site; - - BACK_TRACE* backtrace = thread_back_trace + gettid(); - int stack_depth = backtrace->stack_depth; - backtrace->stack_depth = stack_depth - 1; - if ( stack_depth >= MAX_DEPTH ) { - return; - } - backtrace->frame[stack_depth] = 0; -} -} // extern "C" - -static int cyg_backtrace(void **buffer, int size) { - BACK_TRACE* backtrace = thread_back_trace + gettid(); - int stack_depth = backtrace->stack_depth; - if ( stack_depth >= MAX_DEPTH ) { - stack_depth = MAX_DEPTH; - } - int nSize = (size > stack_depth) ? stack_depth : size; - for (int i = 0; i < nSize; i++) { - buffer[i] = backtrace->frame[nSize - i - 1]; - } - - return nSize; -} - -#endif // BASE_STACKTRACE_INSTRUMENT_INL_H_ - - -// Note: this part of the file is included several times. -// Do not put globals below. - -// The following 4 functions are generated from the code below: -// GetStack{Trace,Frames}() -// GetStack{Trace,Frames}WithContext() -// -// These functions take the following args: -// void** result: the stack-trace, as an array -// int* sizes: the size of each stack frame, as an array -// (GetStackFrames* only) -// int max_depth: the size of the result (and sizes) array(s) -// int skip_count: how many stack pointers to skip before storing in result -// void* ucp: a ucontext_t* (GetStack{Trace,Frames}WithContext only) -static int GET_STACK_TRACE_OR_FRAMES { - static const int kStackLength = 64; - void * stack[kStackLength]; - int size; - memset(stack, 0, sizeof(stack)); - - size = cyg_backtrace(stack, kStackLength); - skip_count += 2; // we want to skip the current and parent frame as well - int result_count = size - skip_count; - if (result_count < 0) - result_count = 0; - if (result_count > max_depth) - result_count = max_depth; - for (int i = 0; i < result_count; i++) - result[i] = stack[i + skip_count]; - -#if IS_STACK_FRAMES - // No implementation for finding out the stack frame sizes yet. - memset(sizes, 0, sizeof(*sizes) * result_count); -#endif - - return result_count; -} diff --git a/contrib/libtcmalloc/src/stacktrace_libgcc-inl.h b/contrib/libtcmalloc/src/stacktrace_libgcc-inl.h deleted file mode 100644 index ce9cf5196ad..00000000000 --- a/contrib/libtcmalloc/src/stacktrace_libgcc-inl.h +++ /dev/null @@ -1,111 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2016, gperftools Contributors -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// This file implements backtrace capturing via libgcc's -// _Unwind_Backtrace. This generally works almost always. It will fail -// sometimes when we're trying to capture backtrace from signal -// handler (i.e. in cpu profiler) while some C++ code is throwing -// exception. - -#ifndef BASE_STACKTRACE_LIBGCC_INL_H_ -#define BASE_STACKTRACE_LIBGCC_INL_H_ -// Note: this file is included into stacktrace.cc more than once. -// Anything that should only be defined once should be here: - -extern "C" { -#include -#include // for memset() -} - -#include - -#include "gperftools/stacktrace.h" - -struct libgcc_backtrace_data { - void **array; - int skip; - int pos; - int limit; -}; - -static _Unwind_Reason_Code libgcc_backtrace_helper(struct _Unwind_Context *ctx, - void *_data) { - libgcc_backtrace_data *data = - reinterpret_cast(_data); - - if (data->skip > 0) { - data->skip--; - return _URC_NO_REASON; - } - - if (data->pos < data->limit) { - void *ip = reinterpret_cast(_Unwind_GetIP(ctx));; - data->array[data->pos++] = ip; - } - - return _URC_NO_REASON; -} - -#endif // BASE_STACKTRACE_LIBGCC_INL_H_ - -// Note: this part of the file is included several times. -// Do not put globals below. - -// The following 4 functions are generated from the code below: -// GetStack{Trace,Frames}() -// GetStack{Trace,Frames}WithContext() -// -// These functions take the following args: -// void** result: the stack-trace, as an array -// int* sizes: the size of each stack frame, as an array -// (GetStackFrames* only) -// int max_depth: the size of the result (and sizes) array(s) -// int skip_count: how many stack pointers to skip before storing in result -// void* ucp: a ucontext_t* (GetStack{Trace,Frames}WithContext only) -static int GET_STACK_TRACE_OR_FRAMES { - libgcc_backtrace_data data; - data.array = result; - // we're also skipping current and parent's frame - data.skip = skip_count + 2; - data.pos = 0; - data.limit = max_depth; - - _Unwind_Backtrace(libgcc_backtrace_helper, &data); - - if (data.pos > 1 && data.array[data.pos - 1] == NULL) - --data.pos; - -#if IS_STACK_FRAMES - // No implementation for finding out the stack frame sizes. - memset(sizes, 0, sizeof(*sizes) * data.pos); -#endif - - return data.pos; -} diff --git a/contrib/libtcmalloc/src/stacktrace_libunwind-inl.h b/contrib/libtcmalloc/src/stacktrace_libunwind-inl.h deleted file mode 100644 index e8257af6c2c..00000000000 --- a/contrib/libtcmalloc/src/stacktrace_libunwind-inl.h +++ /dev/null @@ -1,152 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Arun Sharma -// -// Produce stack trace using libunwind - -#ifndef BASE_STACKTRACE_LIBINWIND_INL_H_ -#define BASE_STACKTRACE_LIBINWIND_INL_H_ -// Note: this file is included into stacktrace.cc more than once. -// Anything that should only be defined once should be here: - -// We only need local unwinder. -#define UNW_LOCAL_ONLY - -extern "C" { -#include -#include // for memset() -#include -} -#include "gperftools/stacktrace.h" - -#include "base/basictypes.h" -#include "base/logging.h" - -// Sometimes, we can try to get a stack trace from within a stack -// trace, because libunwind can call mmap (maybe indirectly via an -// internal mmap based memory allocator), and that mmap gets trapped -// and causes a stack-trace request. If were to try to honor that -// recursive request, we'd end up with infinite recursion or deadlock. -// Luckily, it's safe to ignore those subsequent traces. In such -// cases, we return 0 to indicate the situation. -static __thread int recursive ATTR_INITIAL_EXEC; - -#if defined(TCMALLOC_ENABLE_UNWIND_FROM_UCONTEXT) && (defined(__i386__) || defined(__x86_64__)) && defined(__GNU_LIBRARY__) -#define BASE_STACKTRACE_UNW_CONTEXT_IS_UCONTEXT 1 -#endif - -#endif // BASE_STACKTRACE_LIBINWIND_INL_H_ - -// Note: this part of the file is included several times. -// Do not put globals below. - -// The following 4 functions are generated from the code below: -// GetStack{Trace,Frames}() -// GetStack{Trace,Frames}WithContext() -// -// These functions take the following args: -// void** result: the stack-trace, as an array -// int* sizes: the size of each stack frame, as an array -// (GetStackFrames* only) -// int max_depth: the size of the result (and sizes) array(s) -// int skip_count: how many stack pointers to skip before storing in result -// void* ucp: a ucontext_t* (GetStack{Trace,Frames}WithContext only) -static int GET_STACK_TRACE_OR_FRAMES { - void *ip; - int n = 0; - unw_cursor_t cursor; - unw_context_t uc; -#if IS_STACK_FRAMES - unw_word_t sp = 0, next_sp = 0; -#endif - - if (recursive) { - return 0; - } - ++recursive; - -#if (IS_WITH_CONTEXT && defined(BASE_STACKTRACE_UNW_CONTEXT_IS_UCONTEXT)) - if (ucp) { - uc = *(static_cast(const_cast(ucp))); - /* this is a bit weird. profiler.cc calls us with signal's ucontext - * yet passing us 2 as skip_count and essentially assuming we won't - * use ucontext. */ - /* In order to fix that I'm going to assume that if ucp is - * non-null we're asked to ignore skip_count in case we're - * able to use ucp */ - skip_count = 0; - } else { - unw_getcontext(&uc); - skip_count += 2; // Do not include current and parent frame - } -#else - unw_getcontext(&uc); - skip_count += 2; // Do not include current and parent frame -#endif - - /*int ret =*/ unw_init_local(&cursor, &uc); - //assert(ret >= 0); - - while (skip_count--) { - if (unw_step(&cursor) <= 0) { - goto out; - } -#if IS_STACK_FRAMES - if (unw_get_reg(&cursor, UNW_REG_SP, &next_sp)) { - goto out; - } -#endif - } - - while (n < max_depth) { - if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) { - break; - } -#if IS_STACK_FRAMES - sizes[n] = 0; -#endif - result[n++] = ip; - if (unw_step(&cursor) <= 0) { - break; - } -#if IS_STACK_FRAMES - sp = next_sp; - if (unw_get_reg(&cursor, UNW_REG_SP, &next_sp) , 0) { - break; - } - sizes[n - 1] = next_sp - sp; -#endif - } -out: - --recursive; - return n; -} diff --git a/contrib/libtcmalloc/src/stacktrace_x86-inl.h b/contrib/libtcmalloc/src/stacktrace_x86-inl.h deleted file mode 100644 index 46eb5d82d71..00000000000 --- a/contrib/libtcmalloc/src/stacktrace_x86-inl.h +++ /dev/null @@ -1,354 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat -// -// Produce stack trace - -#ifndef BASE_STACKTRACE_X86_INL_H_ -#define BASE_STACKTRACE_X86_INL_H_ -// Note: this file is included into stacktrace.cc more than once. -// Anything that should only be defined once should be here: - -#include "config.h" -#include // for NULL -#include -#if defined(HAVE_SYS_UCONTEXT_H) -#include -#elif defined(HAVE_UCONTEXT_H) -#include // for ucontext_t -#elif defined(HAVE_CYGWIN_SIGNAL_H) -// cygwin/signal.h has a buglet where it uses pthread_attr_t without -// #including itself. So we have to do it. -# ifdef HAVE_PTHREAD -# include -# endif -#include -typedef ucontext ucontext_t; -#endif -#ifdef HAVE_STDINT_H -#include // for uintptr_t -#endif -#ifdef HAVE_UNISTD_H -#include -#endif -#ifdef HAVE_MMAP -#include // for msync -#include "base/vdso_support.h" -#endif - -#include "gperftools/stacktrace.h" - -#if defined(__linux__) && defined(__i386__) && defined(__ELF__) && defined(HAVE_MMAP) -// Count "push %reg" instructions in VDSO __kernel_vsyscall(), -// preceding "syscall" or "sysenter". -// If __kernel_vsyscall uses frame pointer, answer 0. -// -// kMaxBytes tells how many instruction bytes of __kernel_vsyscall -// to analyze before giving up. Up to kMaxBytes+1 bytes of -// instructions could be accessed. -// -// Here are known __kernel_vsyscall instruction sequences: -// -// SYSENTER (linux-2.6.26/arch/x86/vdso/vdso32/sysenter.S). -// Used on Intel. -// 0xffffe400 <__kernel_vsyscall+0>: push %ecx -// 0xffffe401 <__kernel_vsyscall+1>: push %edx -// 0xffffe402 <__kernel_vsyscall+2>: push %ebp -// 0xffffe403 <__kernel_vsyscall+3>: mov %esp,%ebp -// 0xffffe405 <__kernel_vsyscall+5>: sysenter -// -// SYSCALL (see linux-2.6.26/arch/x86/vdso/vdso32/syscall.S). -// Used on AMD. -// 0xffffe400 <__kernel_vsyscall+0>: push %ebp -// 0xffffe401 <__kernel_vsyscall+1>: mov %ecx,%ebp -// 0xffffe403 <__kernel_vsyscall+3>: syscall -// -// i386 (see linux-2.6.26/arch/x86/vdso/vdso32/int80.S) -// 0xffffe400 <__kernel_vsyscall+0>: int $0x80 -// 0xffffe401 <__kernel_vsyscall+1>: ret -// -static const int kMaxBytes = 10; - -// We use assert()s instead of DCHECK()s -- this is too low level -// for DCHECK(). - -static int CountPushInstructions(const unsigned char *const addr) { - int result = 0; - for (int i = 0; i < kMaxBytes; ++i) { - if (addr[i] == 0x89) { - // "mov reg,reg" - if (addr[i + 1] == 0xE5) { - // Found "mov %esp,%ebp". - return 0; - } - ++i; // Skip register encoding byte. - } else if (addr[i] == 0x0F && - (addr[i + 1] == 0x34 || addr[i + 1] == 0x05)) { - // Found "sysenter" or "syscall". - return result; - } else if ((addr[i] & 0xF0) == 0x50) { - // Found "push %reg". - ++result; - } else if (addr[i] == 0xCD && addr[i + 1] == 0x80) { - // Found "int $0x80" - assert(result == 0); - return 0; - } else { - // Unexpected instruction. - assert(0 == "unexpected instruction in __kernel_vsyscall"); - return 0; - } - } - // Unexpected: didn't find SYSENTER or SYSCALL in - // [__kernel_vsyscall, __kernel_vsyscall + kMaxBytes) interval. - assert(0 == "did not find SYSENTER or SYSCALL in __kernel_vsyscall"); - return 0; -} -#endif - -// Given a pointer to a stack frame, locate and return the calling -// stackframe, or return NULL if no stackframe can be found. Perform sanity -// checks (the strictness of which is controlled by the boolean parameter -// "STRICT_UNWINDING") to reduce the chance that a bad pointer is returned. -template -static void **NextStackFrame(void **old_sp, const void *uc) { - void **new_sp = (void **) *old_sp; - -#if defined(__linux__) && defined(__i386__) && defined(HAVE_VDSO_SUPPORT) - if (WITH_CONTEXT && uc != NULL) { - // How many "push %reg" instructions are there at __kernel_vsyscall? - // This is constant for a given kernel and processor, so compute - // it only once. - static int num_push_instructions = -1; // Sentinel: not computed yet. - // Initialize with sentinel value: __kernel_rt_sigreturn can not possibly - // be there. - static const unsigned char *kernel_rt_sigreturn_address = NULL; - static const unsigned char *kernel_vsyscall_address = NULL; - if (num_push_instructions == -1) { - base::VDSOSupport vdso; - if (vdso.IsPresent()) { - base::VDSOSupport::SymbolInfo rt_sigreturn_symbol_info; - base::VDSOSupport::SymbolInfo vsyscall_symbol_info; - if (!vdso.LookupSymbol("__kernel_rt_sigreturn", "LINUX_2.5", - STT_FUNC, &rt_sigreturn_symbol_info) || - !vdso.LookupSymbol("__kernel_vsyscall", "LINUX_2.5", - STT_FUNC, &vsyscall_symbol_info) || - rt_sigreturn_symbol_info.address == NULL || - vsyscall_symbol_info.address == NULL) { - // Unexpected: 32-bit VDSO is present, yet one of the expected - // symbols is missing or NULL. - assert(0 == "VDSO is present, but doesn't have expected symbols"); - num_push_instructions = 0; - } else { - kernel_rt_sigreturn_address = - reinterpret_cast( - rt_sigreturn_symbol_info.address); - kernel_vsyscall_address = - reinterpret_cast( - vsyscall_symbol_info.address); - num_push_instructions = - CountPushInstructions(kernel_vsyscall_address); - } - } else { - num_push_instructions = 0; - } - } - if (num_push_instructions != 0 && kernel_rt_sigreturn_address != NULL && - old_sp[1] == kernel_rt_sigreturn_address) { - const ucontext_t *ucv = static_cast(uc); - // This kernel does not use frame pointer in its VDSO code, - // and so %ebp is not suitable for unwinding. - void **const reg_ebp = - reinterpret_cast(ucv->uc_mcontext.gregs[REG_EBP]); - const unsigned char *const reg_eip = - reinterpret_cast(ucv->uc_mcontext.gregs[REG_EIP]); - if (new_sp == reg_ebp && - kernel_vsyscall_address <= reg_eip && - reg_eip - kernel_vsyscall_address < kMaxBytes) { - // We "stepped up" to __kernel_vsyscall, but %ebp is not usable. - // Restore from 'ucv' instead. - void **const reg_esp = - reinterpret_cast(ucv->uc_mcontext.gregs[REG_ESP]); - // Check that alleged %esp is not NULL and is reasonably aligned. - if (reg_esp && - ((uintptr_t)reg_esp & (sizeof(reg_esp) - 1)) == 0) { - // Check that alleged %esp is actually readable. This is to prevent - // "double fault" in case we hit the first fault due to e.g. stack - // corruption. - // - // page_size is linker-initalized to avoid async-unsafe locking - // that GCC would otherwise insert (__cxa_guard_acquire etc). - static int page_size; - if (page_size == 0) { - // First time through. - page_size = getpagesize(); - } - void *const reg_esp_aligned = - reinterpret_cast( - (uintptr_t)(reg_esp + num_push_instructions - 1) & - ~(page_size - 1)); - if (msync(reg_esp_aligned, page_size, MS_ASYNC) == 0) { - // Alleged %esp is readable, use it for further unwinding. - new_sp = reinterpret_cast( - reg_esp[num_push_instructions - 1]); - } - } - } - } - } -#endif - - // Check that the transition from frame pointer old_sp to frame - // pointer new_sp isn't clearly bogus - if (STRICT_UNWINDING) { - // With the stack growing downwards, older stack frame must be - // at a greater address that the current one. - if (new_sp <= old_sp) return NULL; - // Assume stack frames larger than 100,000 bytes are bogus. - if ((uintptr_t)new_sp - (uintptr_t)old_sp > 100000) return NULL; - } else { - // In the non-strict mode, allow discontiguous stack frames. - // (alternate-signal-stacks for example). - if (new_sp == old_sp) return NULL; - if (new_sp > old_sp) { - // And allow frames upto about 1MB. - const uintptr_t delta = (uintptr_t)new_sp - (uintptr_t)old_sp; - const uintptr_t acceptable_delta = 1000000; - if (delta > acceptable_delta) { - return NULL; - } - } - } - if ((uintptr_t)new_sp & (sizeof(void *) - 1)) return NULL; -#ifdef __i386__ - // On 64-bit machines, the stack pointer can be very close to - // 0xffffffff, so we explicitly check for a pointer into the - // last two pages in the address space - if ((uintptr_t)new_sp >= 0xffffe000) return NULL; -#endif -#ifdef HAVE_MMAP - if (!STRICT_UNWINDING) { - // Lax sanity checks cause a crash on AMD-based machines with - // VDSO-enabled kernels. - // Make an extra sanity check to insure new_sp is readable. - // Note: NextStackFrame() is only called while the program - // is already on its last leg, so it's ok to be slow here. - static int page_size = getpagesize(); - void *new_sp_aligned = (void *)((uintptr_t)new_sp & ~(page_size - 1)); - if (msync(new_sp_aligned, page_size, MS_ASYNC) == -1) - return NULL; - } -#endif - return new_sp; -} - -#endif // BASE_STACKTRACE_X86_INL_H_ - -// Note: this part of the file is included several times. -// Do not put globals below. - -// The following 4 functions are generated from the code below: -// GetStack{Trace,Frames}() -// GetStack{Trace,Frames}WithContext() -// -// These functions take the following args: -// void** result: the stack-trace, as an array -// int* sizes: the size of each stack frame, as an array -// (GetStackFrames* only) -// int max_depth: the size of the result (and sizes) array(s) -// int skip_count: how many stack pointers to skip before storing in result -// void* ucp: a ucontext_t* (GetStack{Trace,Frames}WithContext only) - -static int GET_STACK_TRACE_OR_FRAMES { - void **sp; -#if (__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 2) || __llvm__ - // __builtin_frame_address(0) can return the wrong address on gcc-4.1.0-k8. - // It's always correct on llvm, and the techniques below aren't (in - // particular, llvm-gcc will make a copy of pcs, so it's not in sp[2]), - // so we also prefer __builtin_frame_address when running under llvm. - sp = reinterpret_cast(__builtin_frame_address(0)); -#elif defined(__i386__) - // Stack frame format: - // sp[0] pointer to previous frame - // sp[1] caller address - // sp[2] first argument - // ... - // NOTE: This will break under llvm, since result is a copy and not in sp[2] - sp = (void **)&result - 2; -#elif defined(__x86_64__) - unsigned long rbp; - // Move the value of the register %rbp into the local variable rbp. - // We need 'volatile' to prevent this instruction from getting moved - // around during optimization to before function prologue is done. - // An alternative way to achieve this - // would be (before this __asm__ instruction) to call Noop() defined as - // static void Noop() __attribute__ ((noinline)); // prevent inlining - // static void Noop() { asm(""); } // prevent optimizing-away - __asm__ volatile ("mov %%rbp, %0" : "=r" (rbp)); - // Arguments are passed in registers on x86-64, so we can't just - // offset from &result - sp = (void **) rbp; -#else -# error Using stacktrace_x86-inl.h on a non x86 architecture! -#endif - - skip_count++; // skip parent's frame due to indirection in stacktrace.cc - - int n = 0; - while (sp && n < max_depth) { - if (*(sp+1) == reinterpret_cast(0)) { - // In 64-bit code, we often see a frame that - // points to itself and has a return address of 0. - break; - } -#if !IS_WITH_CONTEXT - const void *const ucp = NULL; -#endif - void **next_sp = NextStackFrame(sp, ucp); - if (skip_count > 0) { - skip_count--; - } else { - result[n] = *(sp+1); -#if IS_STACK_FRAMES - if (next_sp > sp) { - sizes[n] = (uintptr_t)next_sp - (uintptr_t)sp; - } else { - // A frame-size of 0 is used to indicate unknown frame size. - sizes[n] = 0; - } -#endif - n++; - } - sp = next_sp; - } - return n; -} diff --git a/contrib/libtcmalloc/src/static_vars.cc b/contrib/libtcmalloc/src/static_vars.cc deleted file mode 100644 index 1e29d339996..00000000000 --- a/contrib/libtcmalloc/src/static_vars.cc +++ /dev/null @@ -1,125 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Ken Ashcraft - -#include "config.h" -#include "static_vars.h" -#include // for NULL -#include // for operator new -#ifdef HAVE_PTHREAD -#include // for pthread_atfork -#endif -#include "internal_logging.h" // for CHECK_CONDITION -#include "common.h" -#include "sampler.h" // for Sampler -#include "getenv_safe.h" // TCMallocGetenvSafe -#include "base/googleinit.h" -#include "maybe_threads.h" - -namespace tcmalloc { - -#if defined(HAVE_FORK) && defined(HAVE_PTHREAD) -// These following two functions are registered via pthread_atfork to make -// sure the central_cache locks remain in a consisten state in the forked -// version of the thread. - -void CentralCacheLockAll() -{ - Static::pageheap_lock()->Lock(); - for (int i = 0; i < kNumClasses; ++i) - Static::central_cache()[i].Lock(); -} - -void CentralCacheUnlockAll() -{ - for (int i = 0; i < kNumClasses; ++i) - Static::central_cache()[i].Unlock(); - Static::pageheap_lock()->Unlock(); -} -#endif - -SpinLock Static::pageheap_lock_(SpinLock::LINKER_INITIALIZED); -SizeMap Static::sizemap_; -CentralFreeListPadded Static::central_cache_[kNumClasses]; -PageHeapAllocator Static::span_allocator_; -PageHeapAllocator Static::stacktrace_allocator_; -Span Static::sampled_objects_; -PageHeapAllocator Static::bucket_allocator_; -StackTrace* Static::growth_stacks_ = NULL; -PageHeap* Static::pageheap_ = NULL; - - -void Static::InitStaticVars() { - sizemap_.Init(); - span_allocator_.Init(); - span_allocator_.New(); // Reduce cache conflicts - span_allocator_.New(); // Reduce cache conflicts - stacktrace_allocator_.Init(); - bucket_allocator_.Init(); - // Do a bit of sanitizing: make sure central_cache is aligned properly - CHECK_CONDITION((sizeof(central_cache_[0]) % 64) == 0); - for (int i = 0; i < kNumClasses; ++i) { - central_cache_[i].Init(i); - } - - // It's important to have PageHeap allocated, not in static storage, - // so that HeapLeakChecker does not consider all the byte patterns stored - // in is caches as pointers that are sources of heap object liveness, - // which leads to it missing some memory leaks. - pageheap_ = new (MetaDataAlloc(sizeof(PageHeap))) PageHeap; - - bool aggressive_decommit = - tcmalloc::commandlineflags::StringToBool( - TCMallocGetenvSafe("TCMALLOC_AGGRESSIVE_DECOMMIT"), true); - - pageheap_->SetAggressiveDecommit(aggressive_decommit); - - DLL_Init(&sampled_objects_); - Sampler::InitStatics(); -} - - -#if defined(HAVE_FORK) && defined(HAVE_PTHREAD) && !defined(__APPLE__) - -static inline -void SetupAtForkLocksHandler() -{ - perftools_pthread_atfork( - CentralCacheLockAll, // parent calls before fork - CentralCacheUnlockAll, // parent calls after fork - CentralCacheUnlockAll); // child calls after fork -} -REGISTER_MODULE_INITIALIZER(tcmalloc_fork_handler, SetupAtForkLocksHandler()); - -#endif - -} // namespace tcmalloc diff --git a/contrib/libtcmalloc/src/static_vars.h b/contrib/libtcmalloc/src/static_vars.h deleted file mode 100644 index d6dfa334ab2..00000000000 --- a/contrib/libtcmalloc/src/static_vars.h +++ /dev/null @@ -1,115 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Ken Ashcraft -// -// Static variables shared by multiple classes. - -#ifndef TCMALLOC_STATIC_VARS_H_ -#define TCMALLOC_STATIC_VARS_H_ - -#include "config.h" -#include "base/spinlock.h" -#include "central_freelist.h" -#include "common.h" -#include "page_heap.h" -#include "page_heap_allocator.h" -#include "span.h" -#include "stack_trace_table.h" - -namespace tcmalloc { - -class Static { - public: - // Linker initialized, so this lock can be accessed at any time. - static SpinLock* pageheap_lock() { return &pageheap_lock_; } - - // Must be called before calling any of the accessors below. - static void InitStaticVars(); - - // Central cache -- an array of free-lists, one per size-class. - // We have a separate lock per free-list to reduce contention. - static CentralFreeListPadded* central_cache() { return central_cache_; } - - static SizeMap* sizemap() { return &sizemap_; } - - ////////////////////////////////////////////////////////////////////// - // In addition to the explicit initialization comment, the variables below - // must be protected by pageheap_lock. - - // Page-level allocator. - static PageHeap* pageheap() { return pageheap_; } - - static PageHeapAllocator* span_allocator() { return &span_allocator_; } - - static PageHeapAllocator* stacktrace_allocator() { - return &stacktrace_allocator_; - } - - static StackTrace* growth_stacks() { return growth_stacks_; } - static void set_growth_stacks(StackTrace* s) { growth_stacks_ = s; } - - // State kept for sampled allocations (/pprof/heap support) - static Span* sampled_objects() { return &sampled_objects_; } - static PageHeapAllocator* bucket_allocator() { - return &bucket_allocator_; - } - - // Check if InitStaticVars() has been run. - static bool IsInited() { return pageheap() != NULL; } - - private: - static SpinLock pageheap_lock_; - - // These static variables require explicit initialization. We cannot - // count on their constructors to do any initialization because other - // static variables may try to allocate memory before these variables - // can run their constructors. - - static SizeMap sizemap_; - static CentralFreeListPadded central_cache_[kNumClasses]; - static PageHeapAllocator span_allocator_; - static PageHeapAllocator stacktrace_allocator_; - static Span sampled_objects_; - static PageHeapAllocator bucket_allocator_; - - // Linked list of stack traces recorded every time we allocated memory - // from the system. Useful for finding allocation sites that cause - // increase in the footprint of the system. The linked list pointer - // is stored in trace->stack[kMaxStackDepth-1]. - static StackTrace* growth_stacks_; - - static PageHeap* pageheap_; -}; - -} // namespace tcmalloc - -#endif // TCMALLOC_STATIC_VARS_H_ diff --git a/contrib/libtcmalloc/src/symbolize.cc b/contrib/libtcmalloc/src/symbolize.cc deleted file mode 100644 index a27106e8bce..00000000000 --- a/contrib/libtcmalloc/src/symbolize.cc +++ /dev/null @@ -1,285 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2009, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Craig Silverstein -// -// This forks out to pprof to do the actual symbolizing. We might -// be better off writing our own in C++. - -#include "config.h" -#include "symbolize.h" -#include -#ifdef HAVE_UNISTD_H -#include // for write() -#endif -#ifdef HAVE_SYS_SOCKET_H -#include // for socketpair() -- needed by Symbolize -#endif -#ifdef HAVE_SYS_WAIT_H -#include // for wait() -- needed by Symbolize -#endif -#ifdef HAVE_POLL_H -#include -#endif -#ifdef __MACH__ -#include // for GetProgramInvocationName() -#include // for PATH_MAX -#endif -#if defined(__CYGWIN__) || defined(__CYGWIN32__) -#include // for get_osfhandle() -#endif -#include -#include "base/commandlineflags.h" -#include "base/logging.h" -#include "base/sysinfo.h" - -using std::string; -using tcmalloc::DumpProcSelfMaps; // from sysinfo.h - - -DEFINE_string(symbolize_pprof, - EnvToString("PPROF_PATH", "pprof"), - "Path to pprof to call for reporting function names."); - -// heap_profile_table_pprof may be referenced after destructors are -// called (since that's when leak-checking is done), so we make -// a more-permanent copy that won't ever get destroyed. -static string* g_pprof_path = new string(FLAGS_symbolize_pprof); - -// Returns NULL if we're on an OS where we can't get the invocation name. -// Using a static var is ok because we're not called from a thread. -static const char* GetProgramInvocationName() { -#if defined(HAVE_PROGRAM_INVOCATION_NAME) -#ifdef __UCLIBC__ - extern const char* program_invocation_name; // uclibc provides this -#else - extern char* program_invocation_name; // gcc provides this -#endif - return program_invocation_name; -#elif defined(__MACH__) - // We don't want to allocate memory for this since we may be - // calculating it when memory is corrupted. - static char program_invocation_name[PATH_MAX]; - if (program_invocation_name[0] == '\0') { // first time calculating - uint32_t length = sizeof(program_invocation_name); - if (_NSGetExecutablePath(program_invocation_name, &length)) - return NULL; - } - return program_invocation_name; -#else - return NULL; // figure out a way to get argv[0] -#endif -} - -// Prints an error message when you can't run Symbolize(). -static void PrintError(const char* reason) { - RAW_LOG(ERROR, - "*** WARNING: Cannot convert addresses to symbols in output below.\n" - "*** Reason: %s\n" - "*** If you cannot fix this, try running pprof directly.\n", - reason); -} - -void SymbolTable::Add(const void* addr) { - symbolization_table_[addr] = ""; -} - -const char* SymbolTable::GetSymbol(const void* addr) { - return symbolization_table_[addr]; -} - -// Updates symbolization_table with the pointers to symbol names corresponding -// to its keys. The symbol names are stored in out, which is allocated and -// freed by the caller of this routine. -// Note that the forking/etc is not thread-safe or re-entrant. That's -// ok for the purpose we need -- reporting leaks detected by heap-checker -// -- but be careful if you decide to use this routine for other purposes. -// Returns number of symbols read on error. If can't symbolize, returns 0 -// and emits an error message about why. -int SymbolTable::Symbolize() { -#if !defined(HAVE_UNISTD_H) || !defined(HAVE_SYS_SOCKET_H) || !defined(HAVE_SYS_WAIT_H) - PrintError("Perftools does not know how to call a sub-process on this O/S"); - return 0; -#else - const char* argv0 = GetProgramInvocationName(); - if (argv0 == NULL) { // can't call symbolize if we can't figure out our name - PrintError("Cannot figure out the name of this executable (argv0)"); - return 0; - } - if (access(g_pprof_path->c_str(), R_OK) != 0) { - PrintError("Cannot find 'pprof' (is PPROF_PATH set correctly?)"); - return 0; - } - - // All this work is to do two-way communication. ugh. - int *child_in = NULL; // file descriptors - int *child_out = NULL; // for now, we don't worry about child_err - int child_fds[5][2]; // socketpair may be called up to five times below - - // The client program may close its stdin and/or stdout and/or stderr - // thus allowing socketpair to reuse file descriptors 0, 1 or 2. - // In this case the communication between the forked processes may be broken - // if either the parent or the child tries to close or duplicate these - // descriptors. The loop below produces two pairs of file descriptors, each - // greater than 2 (stderr). - for (int i = 0; i < 5; i++) { - if (socketpair(AF_UNIX, SOCK_STREAM, 0, child_fds[i]) == -1) { - for (int j = 0; j < i; j++) { - close(child_fds[j][0]); - close(child_fds[j][1]); - PrintError("Cannot create a socket pair"); - } - return 0; - } else { - if ((child_fds[i][0] > 2) && (child_fds[i][1] > 2)) { - if (child_in == NULL) { - child_in = child_fds[i]; - } else { - child_out = child_fds[i]; - for (int j = 0; j < i; j++) { - if (child_fds[j] == child_in) continue; - close(child_fds[j][0]); - close(child_fds[j][1]); - } - break; - } - } - } - } - - switch (fork()) { - case -1: { // error - close(child_in[0]); - close(child_in[1]); - close(child_out[0]); - close(child_out[1]); - PrintError("Unknown error calling fork()"); - return 0; - } - case 0: { // child - close(child_in[1]); // child uses the 0's, parent uses the 1's - close(child_out[1]); // child uses the 0's, parent uses the 1's - close(0); - close(1); - if (dup2(child_in[0], 0) == -1) _exit(1); - if (dup2(child_out[0], 1) == -1) _exit(2); - // Unset vars that might cause trouble when we fork - unsetenv("CPUPROFILE"); - unsetenv("HEAPPROFILE"); - unsetenv("HEAPCHECK"); - unsetenv("PERFTOOLS_VERBOSE"); - execlp(g_pprof_path->c_str(), g_pprof_path->c_str(), - "--symbols", argv0, NULL); - _exit(3); // if execvp fails, it's bad news for us - } - default: { // parent - close(child_in[0]); // child uses the 0's, parent uses the 1's - close(child_out[0]); // child uses the 0's, parent uses the 1's -#ifdef HAVE_POLL_H - // Waiting for 1ms seems to give the OS time to notice any errors. - poll(0, 0, 1); - // For maximum safety, we check to make sure the execlp - // succeeded before trying to write. (Otherwise we'll get a - // SIGPIPE.) For systems without poll.h, we'll just skip this - // check, and trust that the user set PPROF_PATH correctly! - struct pollfd pfd = { child_in[1], POLLOUT, 0 }; - if (!poll(&pfd, 1, 0) || !(pfd.revents & POLLOUT) || - (pfd.revents & (POLLHUP|POLLERR))) { - PrintError("Cannot run 'pprof' (is PPROF_PATH set correctly?)"); - return 0; - } -#endif -#if defined(__CYGWIN__) || defined(__CYGWIN32__) - // On cygwin, DumpProcSelfMaps() takes a HANDLE, not an fd. Convert. - const HANDLE symbols_handle = (HANDLE) get_osfhandle(child_in[1]); - DumpProcSelfMaps(symbols_handle); -#else - DumpProcSelfMaps(child_in[1]); // what pprof expects on stdin -#endif - - // Allocate 24 bytes = ("0x" + 8 bytes + "\n" + overhead) for each - // address to feed to pprof. - const int kOutBufSize = 24 * symbolization_table_.size(); - char *pprof_buffer = new char[kOutBufSize]; - int written = 0; - for (SymbolMap::const_iterator iter = symbolization_table_.begin(); - iter != symbolization_table_.end(); ++iter) { - written += snprintf(pprof_buffer + written, kOutBufSize - written, - // pprof expects format to be 0xXXXXXX - "0x%" PRIxPTR "\n", reinterpret_cast(iter->first)); - } - write(child_in[1], pprof_buffer, strlen(pprof_buffer)); - close(child_in[1]); // that's all we need to write - - const int kSymbolBufferSize = kSymbolSize * symbolization_table_.size(); - int total_bytes_read = 0; - delete[] symbol_buffer_; - symbol_buffer_ = new char[kSymbolBufferSize]; - memset(symbol_buffer_, '\0', kSymbolBufferSize); - while (1) { - int bytes_read = read(child_out[1], symbol_buffer_ + total_bytes_read, - kSymbolBufferSize - total_bytes_read); - if (bytes_read < 0) { - close(child_out[1]); - PrintError("Cannot read data from pprof"); - return 0; - } else if (bytes_read == 0) { - close(child_out[1]); - wait(NULL); - break; - } else { - total_bytes_read += bytes_read; - } - } - // We have successfully read the output of pprof into out. Make sure - // the last symbol is full (we can tell because it ends with a \n). - if (total_bytes_read == 0 || symbol_buffer_[total_bytes_read - 1] != '\n') - return 0; - // make the symbolization_table_ values point to the output vector - SymbolMap::iterator fill = symbolization_table_.begin(); - int num_symbols = 0; - const char *current_name = symbol_buffer_; - for (int i = 0; i < total_bytes_read; i++) { - if (symbol_buffer_[i] == '\n') { - fill->second = current_name; - symbol_buffer_[i] = '\0'; - current_name = symbol_buffer_ + i + 1; - fill++; - num_symbols++; - } - } - return num_symbols; - } - } - PrintError("Unkown error (should never occur!)"); - return 0; // shouldn't be reachable -#endif -} diff --git a/contrib/libtcmalloc/src/symbolize.h b/contrib/libtcmalloc/src/symbolize.h deleted file mode 100644 index 728d073308a..00000000000 --- a/contrib/libtcmalloc/src/symbolize.h +++ /dev/null @@ -1,84 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2009, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Craig Silverstein - -#ifndef TCMALLOC_SYMBOLIZE_H_ -#define TCMALLOC_SYMBOLIZE_H_ - -#include "config.h" -#ifdef HAVE_STDINT_H -#include // for uintptr_t -#endif -#include // for NULL -#include - -using std::map; - -// SymbolTable encapsulates the address operations necessary for stack trace -// symbolization. A common use-case is to Add() the addresses from one or -// several stack traces to a table, call Symbolize() once and use GetSymbol() -// to get the symbol names for pretty-printing the stack traces. -class SymbolTable { - public: - SymbolTable() - : symbol_buffer_(NULL) {} - ~SymbolTable() { - delete[] symbol_buffer_; - } - - // Adds an address to the table. This may overwrite a currently known symbol - // name, so Add() should not generally be called after Symbolize(). - void Add(const void* addr); - - // Returns the symbol name for addr, if the given address was added before - // the last successful call to Symbolize(). Otherwise may return an empty - // c-string. - const char* GetSymbol(const void* addr); - - // Obtains the symbol names for the addresses stored in the table and returns - // the number of addresses actually symbolized. - int Symbolize(); - - private: - typedef map SymbolMap; - - // An average size of memory allocated for a stack trace symbol. - static const int kSymbolSize = 1024; - - // Map from addresses to symbol names. - SymbolMap symbolization_table_; - - // Pointer to the buffer that stores the symbol names. - char *symbol_buffer_; -}; - -#endif // TCMALLOC_SYMBOLIZE_H_ diff --git a/contrib/libtcmalloc/src/system-alloc.cc b/contrib/libtcmalloc/src/system-alloc.cc deleted file mode 100644 index 1356513c1ce..00000000000 --- a/contrib/libtcmalloc/src/system-alloc.cc +++ /dev/null @@ -1,567 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat - -#include "config.h" -#include // for EAGAIN, errno -#include // for open, O_RDWR -#include // for size_t, NULL, ptrdiff_t -#if defined HAVE_STDINT_H -#include // for uintptr_t, intptr_t -#elif defined HAVE_INTTYPES_H -#include -#else -#include -#endif -#ifdef HAVE_MMAP -#include // for munmap, mmap, MADV_DONTNEED, etc -#endif -#ifdef HAVE_UNISTD_H -#include // for sbrk, getpagesize, off_t -#endif -#include // for operator new -#include -#include "base/basictypes.h" -#include "base/commandlineflags.h" -#include "base/spinlock.h" // for SpinLockHolder, SpinLock, etc -#include "common.h" -#include "internal_logging.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" - -// On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old -// form of the name instead. -#ifndef MAP_ANONYMOUS -# define MAP_ANONYMOUS MAP_ANON -#endif - -// Linux added support for MADV_FREE in 4.5 but we aren't ready to use it -// yet. Among other things, using compile-time detection leads to poor -// results when compiling on a system with MADV_FREE and running on a -// system without it. See https://github.com/gperftools/gperftools/issues/780. -#if defined(__linux__) && defined(MADV_FREE) && !defined(TCMALLOC_USE_MADV_FREE) -# undef MADV_FREE -#endif - -// MADV_FREE is specifically designed for use by malloc(), but only -// FreeBSD supports it; in linux we fall back to the somewhat inferior -// MADV_DONTNEED. -#if !defined(MADV_FREE) && defined(MADV_DONTNEED) -# define MADV_FREE MADV_DONTNEED -#endif - -// Solaris has a bug where it doesn't declare madvise() for C++. -// http://www.opensolaris.org/jive/thread.jspa?threadID=21035&tstart=0 -#if defined(__sun) && defined(__SVR4) -# include // for caddr_t - extern "C" { extern int madvise(caddr_t, size_t, int); } -#endif - -// Set kDebugMode mode so that we can have use C++ conditionals -// instead of preprocessor conditionals. -#ifdef NDEBUG -static const bool kDebugMode = false; -#else -static const bool kDebugMode = true; -#endif - -// TODO(sanjay): Move the code below into the tcmalloc namespace -using tcmalloc::kLog; -using tcmalloc::Log; - -// Anonymous namespace to avoid name conflicts on "CheckAddressBits". -namespace { - -// Check that no bit is set at position ADDRESS_BITS or higher. -template bool CheckAddressBits(uintptr_t ptr) { - return (ptr >> ADDRESS_BITS) == 0; -} - -// Specialize for the bit width of a pointer to avoid undefined shift. -template <> bool CheckAddressBits<8 * sizeof(void*)>(uintptr_t ptr) { - return true; -} - -} // Anonymous namespace to avoid name conflicts on "CheckAddressBits". - -COMPILE_ASSERT(kAddressBits <= 8 * sizeof(void*), - address_bits_larger_than_pointer_size); - -static SpinLock spinlock(SpinLock::LINKER_INITIALIZED); - -#if defined(HAVE_MMAP) || defined(MADV_FREE) -// Page size is initialized on demand (only needed for mmap-based allocators) -static size_t pagesize = 0; -#endif - -// The current system allocator -SysAllocator* sys_alloc = NULL; - -// Number of bytes taken from system. -size_t TCMalloc_SystemTaken = 0; - -// Configuration parameters. -DEFINE_int32(malloc_devmem_start, - EnvToInt("TCMALLOC_DEVMEM_START", 0), - "Physical memory starting location in MB for /dev/mem allocation." - " Setting this to 0 disables /dev/mem allocation"); -DEFINE_int32(malloc_devmem_limit, - EnvToInt("TCMALLOC_DEVMEM_LIMIT", 0), - "Physical memory limit location in MB for /dev/mem allocation." - " Setting this to 0 means no limit."); -DEFINE_bool(malloc_skip_sbrk, - EnvToBool("TCMALLOC_SKIP_SBRK", false), - "Whether sbrk can be used to obtain memory."); -DEFINE_bool(malloc_skip_mmap, - EnvToBool("TCMALLOC_SKIP_MMAP", false), - "Whether mmap can be used to obtain memory."); -DEFINE_bool(malloc_disable_memory_release, - EnvToBool("TCMALLOC_DISABLE_MEMORY_RELEASE", false), - "Whether MADV_FREE/MADV_DONTNEED should be used" - " to return unused memory to the system."); - -// static allocators -class SbrkSysAllocator : public SysAllocator { -public: - SbrkSysAllocator() : SysAllocator() { - } - void* Alloc(size_t size, size_t *actual_size, size_t alignment); -}; -static union { - char buf[sizeof(SbrkSysAllocator)]; - void *ptr; -} sbrk_space; - -class MmapSysAllocator : public SysAllocator { -public: - MmapSysAllocator() : SysAllocator() { - } - void* Alloc(size_t size, size_t *actual_size, size_t alignment); -}; -static union { - char buf[sizeof(MmapSysAllocator)]; - void *ptr; -} mmap_space; - -class DevMemSysAllocator : public SysAllocator { -public: - DevMemSysAllocator() : SysAllocator() { - } - void* Alloc(size_t size, size_t *actual_size, size_t alignment); -}; - -class DefaultSysAllocator : public SysAllocator { - public: - DefaultSysAllocator() : SysAllocator() { - for (int i = 0; i < kMaxAllocators; i++) { - failed_[i] = true; - allocs_[i] = NULL; - names_[i] = NULL; - } - } - void SetChildAllocator(SysAllocator* alloc, unsigned int index, - const char* name) { - if (index < kMaxAllocators && alloc != NULL) { - allocs_[index] = alloc; - failed_[index] = false; - names_[index] = name; - } - } - void* Alloc(size_t size, size_t *actual_size, size_t alignment); - - private: - static const int kMaxAllocators = 2; - bool failed_[kMaxAllocators]; - SysAllocator* allocs_[kMaxAllocators]; - const char* names_[kMaxAllocators]; -}; -static union { - char buf[sizeof(DefaultSysAllocator)]; - void *ptr; -} default_space; -static const char sbrk_name[] = "SbrkSysAllocator"; -static const char mmap_name[] = "MmapSysAllocator"; - - -void* SbrkSysAllocator::Alloc(size_t size, size_t *actual_size, - size_t alignment) { -#if !defined(HAVE_SBRK) || defined(__UCLIBC__) - return NULL; -#else - // Check if we should use sbrk allocation. - // FLAGS_malloc_skip_sbrk starts out as false (its uninitialized - // state) and eventually gets initialized to the specified value. Note - // that this code runs for a while before the flags are initialized. - // That means that even if this flag is set to true, some (initial) - // memory will be allocated with sbrk before the flag takes effect. - if (FLAGS_malloc_skip_sbrk) { - return NULL; - } - - // sbrk will release memory if passed a negative number, so we do - // a strict check here - if (static_cast(size + alignment) < 0) return NULL; - - // This doesn't overflow because TCMalloc_SystemAlloc has already - // tested for overflow at the alignment boundary. - size = ((size + alignment - 1) / alignment) * alignment; - - // "actual_size" indicates that the bytes from the returned pointer - // p up to and including (p + actual_size - 1) have been allocated. - if (actual_size) { - *actual_size = size; - } - - // Check that we we're not asking for so much more memory that we'd - // wrap around the end of the virtual address space. (This seems - // like something sbrk() should check for us, and indeed opensolaris - // does, but glibc does not: - // http://src.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/lib/libc/port/sys/sbrk.c?a=true - // http://sourceware.org/cgi-bin/cvsweb.cgi/~checkout~/libc/misc/sbrk.c?rev=1.1.2.1&content-type=text/plain&cvsroot=glibc - // Without this check, sbrk may succeed when it ought to fail.) - if (reinterpret_cast(sbrk(0)) + size < size) { - return NULL; - } - - void* result = sbrk(size); - if (result == reinterpret_cast(-1)) { - return NULL; - } - - // Is it aligned? - uintptr_t ptr = reinterpret_cast(result); - if ((ptr & (alignment-1)) == 0) return result; - - // Try to get more memory for alignment - size_t extra = alignment - (ptr & (alignment-1)); - void* r2 = sbrk(extra); - if (reinterpret_cast(r2) == (ptr + size)) { - // Contiguous with previous result - return reinterpret_cast(ptr + extra); - } - - // Give up and ask for "size + alignment - 1" bytes so - // that we can find an aligned region within it. - result = sbrk(size + alignment - 1); - if (result == reinterpret_cast(-1)) { - return NULL; - } - ptr = reinterpret_cast(result); - if ((ptr & (alignment-1)) != 0) { - ptr += alignment - (ptr & (alignment-1)); - } - return reinterpret_cast(ptr); -#endif // HAVE_SBRK -} - -void* MmapSysAllocator::Alloc(size_t size, size_t *actual_size, - size_t alignment) { -#ifndef HAVE_MMAP - return NULL; -#else - // Check if we should use mmap allocation. - // FLAGS_malloc_skip_mmap starts out as false (its uninitialized - // state) and eventually gets initialized to the specified value. Note - // that this code runs for a while before the flags are initialized. - // Chances are we never get here before the flags are initialized since - // sbrk is used until the heap is exhausted (before mmap is used). - if (FLAGS_malloc_skip_mmap) { - return NULL; - } - - // Enforce page alignment - if (pagesize == 0) pagesize = getpagesize(); - if (alignment < pagesize) alignment = pagesize; - size_t aligned_size = ((size + alignment - 1) / alignment) * alignment; - if (aligned_size < size) { - return NULL; - } - size = aligned_size; - - // "actual_size" indicates that the bytes from the returned pointer - // p up to and including (p + actual_size - 1) have been allocated. - if (actual_size) { - *actual_size = size; - } - - // Ask for extra memory if alignment > pagesize - size_t extra = 0; - if (alignment > pagesize) { - extra = alignment - pagesize; - } - - // Note: size + extra does not overflow since: - // size + alignment < (1<(MAP_FAILED)) { - return NULL; - } - - // Adjust the return memory so it is aligned - uintptr_t ptr = reinterpret_cast(result); - size_t adjust = 0; - if ((ptr & (alignment - 1)) != 0) { - adjust = alignment - (ptr & (alignment - 1)); - } - - // Return the unused memory to the system - if (adjust > 0) { - munmap(reinterpret_cast(ptr), adjust); - } - if (adjust < extra) { - munmap(reinterpret_cast(ptr + adjust + size), extra - adjust); - } - - ptr += adjust; - return reinterpret_cast(ptr); -#endif // HAVE_MMAP -} - -void* DevMemSysAllocator::Alloc(size_t size, size_t *actual_size, - size_t alignment) { -#ifndef HAVE_MMAP - return NULL; -#else - static bool initialized = false; - static off_t physmem_base; // next physical memory address to allocate - static off_t physmem_limit; // maximum physical address allowed - static int physmem_fd; // file descriptor for /dev/mem - - // Check if we should use /dev/mem allocation. Note that it may take - // a while to get this flag initialized, so meanwhile we fall back to - // the next allocator. (It looks like 7MB gets allocated before - // this flag gets initialized -khr.) - if (FLAGS_malloc_devmem_start == 0) { - // NOTE: not a devmem_failure - we'd like TCMalloc_SystemAlloc to - // try us again next time. - return NULL; - } - - if (!initialized) { - physmem_fd = open("/dev/mem", O_RDWR); - if (physmem_fd < 0) { - return NULL; - } - physmem_base = FLAGS_malloc_devmem_start*1024LL*1024LL; - physmem_limit = FLAGS_malloc_devmem_limit*1024LL*1024LL; - initialized = true; - } - - // Enforce page alignment - if (pagesize == 0) pagesize = getpagesize(); - if (alignment < pagesize) alignment = pagesize; - size_t aligned_size = ((size + alignment - 1) / alignment) * alignment; - if (aligned_size < size) { - return NULL; - } - size = aligned_size; - - // "actual_size" indicates that the bytes from the returned pointer - // p up to and including (p + actual_size - 1) have been allocated. - if (actual_size) { - *actual_size = size; - } - - // Ask for extra memory if alignment > pagesize - size_t extra = 0; - if (alignment > pagesize) { - extra = alignment - pagesize; - } - - // check to see if we have any memory left - if (physmem_limit != 0 && - ((size + extra) > (physmem_limit - physmem_base))) { - return NULL; - } - - // Note: size + extra does not overflow since: - // size + alignment < (1<(MAP_FAILED)) { - return NULL; - } - uintptr_t ptr = reinterpret_cast(result); - - // Adjust the return memory so it is aligned - size_t adjust = 0; - if ((ptr & (alignment - 1)) != 0) { - adjust = alignment - (ptr & (alignment - 1)); - } - - // Return the unused virtual memory to the system - if (adjust > 0) { - munmap(reinterpret_cast(ptr), adjust); - } - if (adjust < extra) { - munmap(reinterpret_cast(ptr + adjust + size), extra - adjust); - } - - ptr += adjust; - physmem_base += adjust + size; - - return reinterpret_cast(ptr); -#endif // HAVE_MMAP -} - -void* DefaultSysAllocator::Alloc(size_t size, size_t *actual_size, - size_t alignment) { - for (int i = 0; i < kMaxAllocators; i++) { - if (!failed_[i] && allocs_[i] != NULL) { - void* result = allocs_[i]->Alloc(size, actual_size, alignment); - if (result != NULL) { - return result; - } - failed_[i] = true; - } - } - // After both failed, reset "failed_" to false so that a single failed - // allocation won't make the allocator never work again. - for (int i = 0; i < kMaxAllocators; i++) { - failed_[i] = false; - } - return NULL; -} - -ATTRIBUTE_WEAK ATTRIBUTE_NOINLINE -SysAllocator *tc_get_sysalloc_override(SysAllocator *def) -{ - return def; -} - -static bool system_alloc_inited = false; -void InitSystemAllocators(void) { - MmapSysAllocator *mmap = new (mmap_space.buf) MmapSysAllocator(); - SbrkSysAllocator *sbrk = new (sbrk_space.buf) SbrkSysAllocator(); - - // In 64-bit debug mode, place the mmap allocator first since it - // allocates pointers that do not fit in 32 bits and therefore gives - // us better testing of code's 64-bit correctness. It also leads to - // less false negatives in heap-checking code. (Numbers are less - // likely to look like pointers and therefore the conservative gc in - // the heap-checker is less likely to misinterpret a number as a - // pointer). - DefaultSysAllocator *sdef = new (default_space.buf) DefaultSysAllocator(); - if (kDebugMode && sizeof(void*) > 4) { - sdef->SetChildAllocator(mmap, 0, mmap_name); - sdef->SetChildAllocator(sbrk, 1, sbrk_name); - } else { - sdef->SetChildAllocator(sbrk, 0, sbrk_name); - sdef->SetChildAllocator(mmap, 1, mmap_name); - } - - sys_alloc = tc_get_sysalloc_override(sdef); -} - -void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size, - size_t alignment) { - // Discard requests that overflow - if (size + alignment < size) return NULL; - - SpinLockHolder lock_holder(&spinlock); - - if (!system_alloc_inited) { - InitSystemAllocators(); - system_alloc_inited = true; - } - - // Enforce minimum alignment - if (alignment < sizeof(MemoryAligner)) alignment = sizeof(MemoryAligner); - - size_t actual_size_storage; - if (actual_size == NULL) { - actual_size = &actual_size_storage; - } - - void* result = sys_alloc->Alloc(size, actual_size, alignment); - if (result != NULL) { - CHECK_CONDITION( - CheckAddressBits( - reinterpret_cast(result) + *actual_size - 1)); - TCMalloc_SystemTaken += *actual_size; - } - return result; -} - -bool TCMalloc_SystemRelease(void* start, size_t length) { -#ifdef MADV_FREE - if (FLAGS_malloc_devmem_start) { - // It's not safe to use MADV_FREE/MADV_DONTNEED if we've been - // mapping /dev/mem for heap memory. - return false; - } - if (FLAGS_malloc_disable_memory_release) return false; - if (pagesize == 0) pagesize = getpagesize(); - const size_t pagemask = pagesize - 1; - - size_t new_start = reinterpret_cast(start); - size_t end = new_start + length; - size_t new_end = end; - - // Round up the starting address and round down the ending address - // to be page aligned: - new_start = (new_start + pagesize - 1) & ~pagemask; - new_end = new_end & ~pagemask; - - ASSERT((new_start & pagemask) == 0); - ASSERT((new_end & pagemask) == 0); - ASSERT(new_start >= reinterpret_cast(start)); - ASSERT(new_end <= end); - - if (new_end > new_start) { - int result; - do { - result = madvise(reinterpret_cast(new_start), - new_end - new_start, MADV_FREE); - } while (result == -1 && errno == EAGAIN); - - return result != -1; - } -#endif - return false; -} - -void TCMalloc_SystemCommit(void* start, size_t length) { - // Nothing to do here. TCMalloc_SystemRelease does not alter pages - // such that they need to be re-committed before they can be used by the - // application. -} - -#pragma GCC diagnostic pop diff --git a/contrib/libtcmalloc/src/system-alloc.h b/contrib/libtcmalloc/src/system-alloc.h deleted file mode 100644 index 2c06c183d63..00000000000 --- a/contrib/libtcmalloc/src/system-alloc.h +++ /dev/null @@ -1,92 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat -// -// Routine that uses sbrk/mmap to allocate memory from the system. -// Useful for implementing malloc. - -#ifndef TCMALLOC_SYSTEM_ALLOC_H_ -#define TCMALLOC_SYSTEM_ALLOC_H_ - -#include "config.h" -#include // for size_t - -class SysAllocator; - -// REQUIRES: "alignment" is a power of two or "0" to indicate default alignment -// -// Allocate and return "N" bytes of zeroed memory. -// -// If actual_bytes is NULL then the returned memory is exactly the -// requested size. If actual bytes is non-NULL then the allocator -// may optionally return more bytes than asked for (i.e. return an -// entire "huge" page if a huge page allocator is in use). -// -// The returned pointer is a multiple of "alignment" if non-zero. The -// returned pointer will always be aligned suitably for holding a -// void*, double, or size_t. In addition, if this platform defines -// CACHELINE_ALIGNED, the return pointer will always be cacheline -// aligned. -// -// Returns NULL when out of memory. -extern PERFTOOLS_DLL_DECL -void* TCMalloc_SystemAlloc(size_t bytes, size_t *actual_bytes, - size_t alignment = 0); - -// This call is a hint to the operating system that the pages -// contained in the specified range of memory will not be used for a -// while, and can be released for use by other processes or the OS. -// Pages which are released in this way may be destroyed (zeroed) by -// the OS. The benefit of this function is that it frees memory for -// use by the system, the cost is that the pages are faulted back into -// the address space next time they are touched, which can impact -// performance. (Only pages fully covered by the memory region will -// be released, partial pages will not.) -// -// Returns false if release failed or not supported. -extern PERFTOOLS_DLL_DECL -bool TCMalloc_SystemRelease(void* start, size_t length); - -// Called to ressurect memory which has been previously released -// to the system via TCMalloc_SystemRelease. An attempt to -// commit a page that is already committed does not cause this -// function to fail. -extern PERFTOOLS_DLL_DECL -void TCMalloc_SystemCommit(void* start, size_t length); - -// The current system allocator. -extern PERFTOOLS_DLL_DECL SysAllocator* sys_alloc; - -// Number of bytes taken from system. -extern PERFTOOLS_DLL_DECL size_t TCMalloc_SystemTaken; - -#endif /* TCMALLOC_SYSTEM_ALLOC_H_ */ diff --git a/contrib/libtcmalloc/src/tcmalloc.cc b/contrib/libtcmalloc/src/tcmalloc.cc deleted file mode 100644 index b52524b1361..00000000000 --- a/contrib/libtcmalloc/src/tcmalloc.cc +++ /dev/null @@ -1,1842 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat -// -// A malloc that uses a per-thread cache to satisfy small malloc requests. -// (The time for malloc/free of a small object drops from 300 ns to 50 ns.) -// -// See doc/tcmalloc.html for a high-level -// description of how this malloc works. -// -// SYNCHRONIZATION -// 1. The thread-specific lists are accessed without acquiring any locks. -// This is safe because each such list is only accessed by one thread. -// 2. We have a lock per central free-list, and hold it while manipulating -// the central free list for a particular size. -// 3. The central page allocator is protected by "pageheap_lock". -// 4. The pagemap (which maps from page-number to descriptor), -// can be read without holding any locks, and written while holding -// the "pageheap_lock". -// 5. To improve performance, a subset of the information one can get -// from the pagemap is cached in a data structure, pagemap_cache_, -// that atomically reads and writes its entries. This cache can be -// read and written without locking. -// -// This multi-threaded access to the pagemap is safe for fairly -// subtle reasons. We basically assume that when an object X is -// allocated by thread A and deallocated by thread B, there must -// have been appropriate synchronization in the handoff of object -// X from thread A to thread B. The same logic applies to pagemap_cache_. -// -// THE PAGEID-TO-SIZECLASS CACHE -// Hot PageID-to-sizeclass mappings are held by pagemap_cache_. If this cache -// returns 0 for a particular PageID then that means "no information," not that -// the sizeclass is 0. The cache may have stale information for pages that do -// not hold the beginning of any free()'able object. Staleness is eliminated -// in Populate() for pages with sizeclass > 0 objects, and in do_malloc() and -// do_memalign() for all other relevant pages. -// -// PAGEMAP -// ------- -// Page map contains a mapping from page id to Span. -// -// If Span s occupies pages [p..q], -// pagemap[p] == s -// pagemap[q] == s -// pagemap[p+1..q-1] are undefined -// pagemap[p-1] and pagemap[q+1] are defined: -// NULL if the corresponding page is not yet in the address space. -// Otherwise it points to a Span. This span may be free -// or allocated. If free, it is in one of pageheap's freelist. -// -// TODO: Bias reclamation to larger addresses -// TODO: implement mallinfo/mallopt -// TODO: Better testing -// -// 9/28/2003 (new page-level allocator replaces ptmalloc2): -// * malloc/free of small objects goes from ~300 ns to ~50 ns. -// * allocation of a reasonably complicated struct -// goes from about 1100 ns to about 300 ns. - -#include "config.h" -#include - -#include // for ENOMEM, EINVAL, errno -#if defined HAVE_STDINT_H -#include -#elif defined HAVE_INTTYPES_H -#include -#else -#include -#endif -#include // for size_t, NULL -#include // for getenv -#include // for strcmp, memset, strlen, etc -#ifdef HAVE_UNISTD_H -#include // for getpagesize, write, etc -#endif -#include // for max, min -#include // for numeric_limits -#include // for nothrow_t (ptr only), etc -#include // for vector - -#include -#include // for MallocHook -#include "base/basictypes.h" // for int64 -#include "base/commandlineflags.h" // for RegisterFlagValidator, etc -#include "base/dynamic_annotations.h" // for RunningOnValgrind -#include "base/spinlock.h" // for SpinLockHolder -#include "central_freelist.h" // for CentralFreeListPadded -#include "common.h" // for StackTrace, kPageShift, etc -#include "internal_logging.h" // for ASSERT, TCMalloc_Printer, etc -#include "linked_list.h" // for SLL_SetNext -#include "malloc_hook-inl.h" // for MallocHook::InvokeNewHook, etc -#include "page_heap.h" // for PageHeap, PageHeap::Stats -#include "page_heap_allocator.h" // for PageHeapAllocator -#include "span.h" // for Span, DLL_Prepend, etc -#include "stack_trace_table.h" // for StackTraceTable -#include "static_vars.h" // for Static -#include "system-alloc.h" // for DumpSystemAllocatorStats, etc -#include "tcmalloc_guard.h" // for TCMallocGuard -#include "thread_cache.h" // for ThreadCache - -#ifdef __clang__ -// clang's apparent focus on code size somehow causes it to ignore -// normal inline directives even for few functions which inlining is -// key for performance. In order to get performance of clang's -// generated code closer to normal, we're forcing inlining via -// attribute. -#define ALWAYS_INLINE inline __attribute__((always_inline)) -#else -#define ALWAYS_INLINE inline -#endif - -#include "maybe_emergency_malloc.h" - -#if (defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__)) && !defined(WIN32_OVERRIDE_ALLOCATORS) -# define WIN32_DO_PATCHING 1 -#endif - -// Some windows file somewhere (at least on cygwin) #define's small (!) -#undef small - -using STL_NAMESPACE::max; -using STL_NAMESPACE::numeric_limits; -using STL_NAMESPACE::vector; - -#include "libc_override.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" - -using tcmalloc::AlignmentForSize; -using tcmalloc::kLog; -using tcmalloc::kCrash; -using tcmalloc::kCrashWithStats; -using tcmalloc::Log; -using tcmalloc::PageHeap; -using tcmalloc::PageHeapAllocator; -using tcmalloc::SizeMap; -using tcmalloc::Span; -using tcmalloc::StackTrace; -using tcmalloc::Static; -using tcmalloc::ThreadCache; - -DECLARE_double(tcmalloc_release_rate); - -// For windows, the printf we use to report large allocs is -// potentially dangerous: it could cause a malloc that would cause an -// infinite loop. So by default we set the threshold to a huge number -// on windows, so this bad situation will never trigger. You can -// always set TCMALLOC_LARGE_ALLOC_REPORT_THRESHOLD manually if you -// want this functionality. -#ifdef _WIN32 -const int64 kDefaultLargeAllocReportThreshold = static_cast(1) << 62; -#else -const int64 kDefaultLargeAllocReportThreshold = static_cast(1) << 30; -#endif -DEFINE_int64(tcmalloc_large_alloc_report_threshold, - EnvToInt64("TCMALLOC_LARGE_ALLOC_REPORT_THRESHOLD", - kDefaultLargeAllocReportThreshold), - "Allocations larger than this value cause a stack " - "trace to be dumped to stderr. The threshold for " - "dumping stack traces is increased by a factor of 1.125 " - "every time we print a message so that the threshold " - "automatically goes up by a factor of ~1000 every 60 " - "messages. This bounds the amount of extra logging " - "generated by this flag. Default value of this flag " - "is very large and therefore you should see no extra " - "logging unless the flag is overridden. Set to 0 to " - "disable reporting entirely."); - - -// We already declared these functions in tcmalloc.h, but we have to -// declare them again to give them an ATTRIBUTE_SECTION: we want to -// put all callers of MallocHook::Invoke* in this module into -// ATTRIBUTE_SECTION(google_malloc) section, so that -// MallocHook::GetCallerStackTrace can function accurately. -#ifndef _WIN32 // windows doesn't have attribute_section, so don't bother -extern "C" { - void* tc_malloc(size_t size) PERFTOOLS_THROW - ATTRIBUTE_SECTION(google_malloc); - void tc_free(void* ptr) PERFTOOLS_THROW - ATTRIBUTE_SECTION(google_malloc); - void* tc_realloc(void* ptr, size_t size) PERFTOOLS_THROW - ATTRIBUTE_SECTION(google_malloc); - void* tc_calloc(size_t nmemb, size_t size) PERFTOOLS_THROW - ATTRIBUTE_SECTION(google_malloc); - void tc_cfree(void* ptr) PERFTOOLS_THROW - ATTRIBUTE_SECTION(google_malloc); - - void* tc_memalign(size_t __alignment, size_t __size) PERFTOOLS_THROW - ATTRIBUTE_SECTION(google_malloc); - int tc_posix_memalign(void** ptr, size_t align, size_t size) PERFTOOLS_THROW - ATTRIBUTE_SECTION(google_malloc); - void* tc_valloc(size_t __size) PERFTOOLS_THROW - ATTRIBUTE_SECTION(google_malloc); - void* tc_pvalloc(size_t __size) PERFTOOLS_THROW - ATTRIBUTE_SECTION(google_malloc); - - void tc_malloc_stats(void) PERFTOOLS_THROW - ATTRIBUTE_SECTION(google_malloc); - int tc_mallopt(int cmd, int value) PERFTOOLS_THROW - ATTRIBUTE_SECTION(google_malloc); -#ifdef HAVE_STRUCT_MALLINFO - struct mallinfo tc_mallinfo(void) PERFTOOLS_THROW - ATTRIBUTE_SECTION(google_malloc); -#endif - - void* tc_new(size_t size) - ATTRIBUTE_SECTION(google_malloc); - void tc_delete(void* p) PERFTOOLS_THROW - ATTRIBUTE_SECTION(google_malloc); - void* tc_newarray(size_t size) - ATTRIBUTE_SECTION(google_malloc); - void tc_deletearray(void* p) PERFTOOLS_THROW - ATTRIBUTE_SECTION(google_malloc); - - // And the nothrow variants of these: - void* tc_new_nothrow(size_t size, const std::nothrow_t&) PERFTOOLS_THROW - ATTRIBUTE_SECTION(google_malloc); - void* tc_newarray_nothrow(size_t size, const std::nothrow_t&) PERFTOOLS_THROW - ATTRIBUTE_SECTION(google_malloc); - // Surprisingly, standard C++ library implementations use a - // nothrow-delete internally. See, eg: - // http://www.dinkumware.com/manuals/?manual=compleat&page=new.html - void tc_delete_nothrow(void* ptr, const std::nothrow_t&) PERFTOOLS_THROW - ATTRIBUTE_SECTION(google_malloc); - void tc_deletearray_nothrow(void* ptr, const std::nothrow_t&) PERFTOOLS_THROW - ATTRIBUTE_SECTION(google_malloc); - - // Some non-standard extensions that we support. - - // This is equivalent to - // OS X: malloc_size() - // glibc: malloc_usable_size() - // Windows: _msize() - size_t tc_malloc_size(void* p) PERFTOOLS_THROW - ATTRIBUTE_SECTION(google_malloc); -} // extern "C" -#endif // #ifndef _WIN32 - -// ----------------------- IMPLEMENTATION ------------------------------- - -static int tc_new_mode = 0; // See tc_set_new_mode(). - -// Routines such as free() and realloc() catch some erroneous pointers -// passed to them, and invoke the below when they do. (An erroneous pointer -// won't be caught if it's within a valid span or a stale span for which -// the pagemap cache has a non-zero sizeclass.) This is a cheap (source-editing -// required) kind of exception handling for these routines. -namespace { -void InvalidFree(void* ptr) { - if (tcmalloc::IsEmergencyPtr(ptr)) { - tcmalloc::EmergencyFree(ptr); - return; - } - Log(kCrash, __FILE__, __LINE__, "Attempt to free invalid pointer", ptr); -} - -size_t InvalidGetSizeForRealloc(const void* old_ptr) { - Log(kCrash, __FILE__, __LINE__, - "Attempt to realloc invalid pointer", old_ptr); - return 0; -} - -size_t InvalidGetAllocatedSize(const void* ptr) { - Log(kCrash, __FILE__, __LINE__, - "Attempt to get the size of an invalid pointer", ptr); - return 0; -} -} // unnamed namespace - -// Extract interesting stats -struct TCMallocStats { - uint64_t thread_bytes; // Bytes in thread caches - uint64_t central_bytes; // Bytes in central cache - uint64_t transfer_bytes; // Bytes in central transfer cache - uint64_t metadata_bytes; // Bytes alloced for metadata - PageHeap::Stats pageheap; // Stats from page heap -}; - -// Get stats into "r". Also, if class_count != NULL, class_count[k] -// will be set to the total number of objects of size class k in the -// central cache, transfer cache, and per-thread caches. If small_spans -// is non-NULL, it is filled. Same for large_spans. -static void ExtractStats(TCMallocStats* r, uint64_t* class_count, - PageHeap::SmallSpanStats* small_spans, - PageHeap::LargeSpanStats* large_spans) { - r->central_bytes = 0; - r->transfer_bytes = 0; - for (int cl = 0; cl < kNumClasses; ++cl) { - const int length = Static::central_cache()[cl].length(); - const int tc_length = Static::central_cache()[cl].tc_length(); - const size_t cache_overhead = Static::central_cache()[cl].OverheadBytes(); - const size_t size = static_cast( - Static::sizemap()->ByteSizeForClass(cl)); - r->central_bytes += (size * length) + cache_overhead; - r->transfer_bytes += (size * tc_length); - if (class_count) { - // Sum the lengths of all per-class freelists, except the per-thread - // freelists, which get counted when we call GetThreadStats(), below. - class_count[cl] = length + tc_length; - } - - } - - // Add stats from per-thread heaps - r->thread_bytes = 0; - { // scope - SpinLockHolder h(Static::pageheap_lock()); - ThreadCache::GetThreadStats(&r->thread_bytes, class_count); - r->metadata_bytes = tcmalloc::metadata_system_bytes(); - r->pageheap = Static::pageheap()->stats(); - if (small_spans != NULL) { - Static::pageheap()->GetSmallSpanStats(small_spans); - } - if (large_spans != NULL) { - Static::pageheap()->GetLargeSpanStats(large_spans); - } - } -} - -static double PagesToMiB(uint64_t pages) { - return (pages << kPageShift) / 1048576.0; -} - -// WRITE stats to "out" -static void DumpStats(TCMalloc_Printer* out, int level) { - TCMallocStats stats; - uint64_t class_count[kNumClasses]; - PageHeap::SmallSpanStats small; - PageHeap::LargeSpanStats large; - if (level >= 2) { - ExtractStats(&stats, class_count, &small, &large); - } else { - ExtractStats(&stats, NULL, NULL, NULL); - } - - static const double MiB = 1048576.0; - - const uint64_t virtual_memory_used = (stats.pageheap.system_bytes - + stats.metadata_bytes); - const uint64_t physical_memory_used = (virtual_memory_used - - stats.pageheap.unmapped_bytes); - const uint64_t bytes_in_use_by_app = (physical_memory_used - - stats.metadata_bytes - - stats.pageheap.free_bytes - - stats.central_bytes - - stats.transfer_bytes - - stats.thread_bytes); - -#ifdef TCMALLOC_SMALL_BUT_SLOW - out->printf( - "NOTE: SMALL MEMORY MODEL IS IN USE, PERFORMANCE MAY SUFFER.\n"); -#endif - out->printf( - "------------------------------------------------\n" - "MALLOC: %12" PRIu64 " (%7.1f MiB) Bytes in use by application\n" - "MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in page heap freelist\n" - "MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in central cache freelist\n" - "MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in transfer cache freelist\n" - "MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in thread cache freelists\n" - "MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in malloc metadata\n" - "MALLOC: ------------\n" - "MALLOC: = %12" PRIu64 " (%7.1f MiB) Actual memory used (physical + swap)\n" - "MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes released to OS (aka unmapped)\n" - "MALLOC: ------------\n" - "MALLOC: = %12" PRIu64 " (%7.1f MiB) Virtual address space used\n" - "MALLOC:\n" - "MALLOC: %12" PRIu64 " Spans in use\n" - "MALLOC: %12" PRIu64 " Thread heaps in use\n" - "MALLOC: %12" PRIu64 " Tcmalloc page size\n" - "------------------------------------------------\n" - "Call ReleaseFreeMemory() to release freelist memory to the OS" - " (via madvise()).\n" - "Bytes released to the OS take up virtual address space" - " but no physical memory.\n", - bytes_in_use_by_app, bytes_in_use_by_app / MiB, - stats.pageheap.free_bytes, stats.pageheap.free_bytes / MiB, - stats.central_bytes, stats.central_bytes / MiB, - stats.transfer_bytes, stats.transfer_bytes / MiB, - stats.thread_bytes, stats.thread_bytes / MiB, - stats.metadata_bytes, stats.metadata_bytes / MiB, - physical_memory_used, physical_memory_used / MiB, - stats.pageheap.unmapped_bytes, stats.pageheap.unmapped_bytes / MiB, - virtual_memory_used, virtual_memory_used / MiB, - uint64_t(Static::span_allocator()->inuse()), - uint64_t(ThreadCache::HeapsInUse()), - uint64_t(kPageSize)); - - if (level >= 2) { - out->printf("------------------------------------------------\n"); - out->printf("Total size of freelists for per-thread caches,\n"); - out->printf("transfer cache, and central cache, by size class\n"); - out->printf("------------------------------------------------\n"); - uint64_t cumulative = 0; - for (int cl = 0; cl < kNumClasses; ++cl) { - if (class_count[cl] > 0) { - uint64_t class_bytes = - class_count[cl] * Static::sizemap()->ByteSizeForClass(cl); - cumulative += class_bytes; - out->printf("class %3d [ %8" PRIuS " bytes ] : " - "%8" PRIu64 " objs; %5.1f MiB; %5.1f cum MiB\n", - cl, Static::sizemap()->ByteSizeForClass(cl), - class_count[cl], - class_bytes / MiB, - cumulative / MiB); - } - } - - // append page heap info - int nonempty_sizes = 0; - for (int s = 0; s < kMaxPages; s++) { - if (small.normal_length[s] + small.returned_length[s] > 0) { - nonempty_sizes++; - } - } - out->printf("------------------------------------------------\n"); - out->printf("PageHeap: %d sizes; %6.1f MiB free; %6.1f MiB unmapped\n", - nonempty_sizes, stats.pageheap.free_bytes / MiB, - stats.pageheap.unmapped_bytes / MiB); - out->printf("------------------------------------------------\n"); - uint64_t total_normal = 0; - uint64_t total_returned = 0; - for (int s = 0; s < kMaxPages; s++) { - const int n_length = small.normal_length[s]; - const int r_length = small.returned_length[s]; - if (n_length + r_length > 0) { - uint64_t n_pages = s * n_length; - uint64_t r_pages = s * r_length; - total_normal += n_pages; - total_returned += r_pages; - out->printf("%6u pages * %6u spans ~ %6.1f MiB; %6.1f MiB cum" - "; unmapped: %6.1f MiB; %6.1f MiB cum\n", - s, - (n_length + r_length), - PagesToMiB(n_pages + r_pages), - PagesToMiB(total_normal + total_returned), - PagesToMiB(r_pages), - PagesToMiB(total_returned)); - } - } - - total_normal += large.normal_pages; - total_returned += large.returned_pages; - out->printf(">255 large * %6u spans ~ %6.1f MiB; %6.1f MiB cum" - "; unmapped: %6.1f MiB; %6.1f MiB cum\n", - static_cast(large.spans), - PagesToMiB(large.normal_pages + large.returned_pages), - PagesToMiB(total_normal + total_returned), - PagesToMiB(large.returned_pages), - PagesToMiB(total_returned)); - } -} - -static void PrintStats(int level) { - const int kBufferSize = 16 << 10; - char* buffer = new char[kBufferSize]; - TCMalloc_Printer printer(buffer, kBufferSize); - DumpStats(&printer, level); - write(STDERR_FILENO, buffer, strlen(buffer)); - delete[] buffer; -} - -static void** DumpHeapGrowthStackTraces() { - // Count how much space we need - int needed_slots = 0; - { - SpinLockHolder h(Static::pageheap_lock()); - for (StackTrace* t = Static::growth_stacks(); - t != NULL; - t = reinterpret_cast( - t->stack[tcmalloc::kMaxStackDepth-1])) { - needed_slots += 3 + t->depth; - } - needed_slots += 100; // Slop in case list grows - needed_slots += needed_slots/8; // An extra 12.5% slop - } - - void** result = new void*[needed_slots]; - if (result == NULL) { - Log(kLog, __FILE__, __LINE__, - "tcmalloc: allocation failed for stack trace slots", - needed_slots * sizeof(*result)); - return NULL; - } - - SpinLockHolder h(Static::pageheap_lock()); - int used_slots = 0; - for (StackTrace* t = Static::growth_stacks(); - t != NULL; - t = reinterpret_cast( - t->stack[tcmalloc::kMaxStackDepth-1])) { - ASSERT(used_slots < needed_slots); // Need to leave room for terminator - if (used_slots + 3 + t->depth >= needed_slots) { - // No more room - break; - } - - result[used_slots+0] = reinterpret_cast(static_cast(1)); - result[used_slots+1] = reinterpret_cast(t->size); - result[used_slots+2] = reinterpret_cast(t->depth); - for (int d = 0; d < t->depth; d++) { - result[used_slots+3+d] = t->stack[d]; - } - used_slots += 3 + t->depth; - } - result[used_slots] = reinterpret_cast(static_cast(0)); - return result; -} - -static void IterateOverRanges(void* arg, MallocExtension::RangeFunction func) { - PageID page = 1; // Some code may assume that page==0 is never used - bool done = false; - while (!done) { - // Accumulate a small number of ranges in a local buffer - static const int kNumRanges = 16; - static base::MallocRange ranges[kNumRanges]; - int n = 0; - { - SpinLockHolder h(Static::pageheap_lock()); - while (n < kNumRanges) { - if (!Static::pageheap()->GetNextRange(page, &ranges[n])) { - done = true; - break; - } else { - uintptr_t limit = ranges[n].address + ranges[n].length; - page = (limit + kPageSize - 1) >> kPageShift; - n++; - } - } - } - - for (int i = 0; i < n; i++) { - (*func)(arg, &ranges[i]); - } - } -} - -// TCMalloc's support for extra malloc interfaces -class TCMallocImplementation : public MallocExtension { - private: - // ReleaseToSystem() might release more than the requested bytes because - // the page heap releases at the span granularity, and spans are of wildly - // different sizes. This member keeps track of the extra bytes bytes - // released so that the app can periodically call ReleaseToSystem() to - // release memory at a constant rate. - // NOTE: Protected by Static::pageheap_lock(). - size_t extra_bytes_released_; - - public: - TCMallocImplementation() - : extra_bytes_released_(0) { - } - - virtual void GetStats(char* buffer, int buffer_length) { - ASSERT(buffer_length > 0); - TCMalloc_Printer printer(buffer, buffer_length); - - // Print level one stats unless lots of space is available - if (buffer_length < 10000) { - DumpStats(&printer, 1); - } else { - DumpStats(&printer, 2); - } - } - - // We may print an extra, tcmalloc-specific warning message here. - virtual void GetHeapSample(MallocExtensionWriter* writer) { - if (FLAGS_tcmalloc_sample_parameter == 0) { - const char* const kWarningMsg = - "%warn\n" - "%warn This heap profile does not have any data in it, because\n" - "%warn the application was run with heap sampling turned off.\n" - "%warn To get useful data from GetHeapSample(), you must\n" - "%warn set the environment variable TCMALLOC_SAMPLE_PARAMETER to\n" - "%warn a positive sampling period, such as 524288.\n" - "%warn\n"; - writer->append(kWarningMsg, strlen(kWarningMsg)); - } - MallocExtension::GetHeapSample(writer); - } - - virtual void** ReadStackTraces(int* sample_period) { - tcmalloc::StackTraceTable table; - { - SpinLockHolder h(Static::pageheap_lock()); - Span* sampled = Static::sampled_objects(); - for (Span* s = sampled->next; s != sampled; s = s->next) { - table.AddTrace(*reinterpret_cast(s->objects)); - } - } - *sample_period = ThreadCache::GetCache()->GetSamplePeriod(); - return table.ReadStackTracesAndClear(); // grabs and releases pageheap_lock - } - - virtual void** ReadHeapGrowthStackTraces() { - return DumpHeapGrowthStackTraces(); - } - - virtual size_t GetThreadCacheSize() { - ThreadCache* tc = ThreadCache::GetCacheIfPresent(); - if (!tc) - return 0; - return tc->Size(); - } - - virtual void MarkThreadTemporarilyIdle() { - ThreadCache::BecomeTemporarilyIdle(); - } - - virtual void Ranges(void* arg, RangeFunction func) { - IterateOverRanges(arg, func); - } - - virtual bool GetNumericProperty(const char* name, size_t* value) { - ASSERT(name != NULL); - - if (strcmp(name, "generic.current_allocated_bytes") == 0) { - TCMallocStats stats; - ExtractStats(&stats, NULL, NULL, NULL); - *value = stats.pageheap.system_bytes - - stats.thread_bytes - - stats.central_bytes - - stats.transfer_bytes - - stats.pageheap.free_bytes - - stats.pageheap.unmapped_bytes; - return true; - } - - if (strcmp(name, "generic.heap_size") == 0) { - TCMallocStats stats; - ExtractStats(&stats, NULL, NULL, NULL); - *value = stats.pageheap.system_bytes; - return true; - } - - if (strcmp(name, "tcmalloc.slack_bytes") == 0) { - // Kept for backwards compatibility. Now defined externally as: - // pageheap_free_bytes + pageheap_unmapped_bytes. - SpinLockHolder l(Static::pageheap_lock()); - PageHeap::Stats stats = Static::pageheap()->stats(); - *value = stats.free_bytes + stats.unmapped_bytes; - return true; - } - - if (strcmp(name, "tcmalloc.central_cache_free_bytes") == 0) { - TCMallocStats stats; - ExtractStats(&stats, NULL, NULL, NULL); - *value = stats.central_bytes; - return true; - } - - if (strcmp(name, "tcmalloc.transfer_cache_free_bytes") == 0) { - TCMallocStats stats; - ExtractStats(&stats, NULL, NULL, NULL); - *value = stats.transfer_bytes; - return true; - } - - if (strcmp(name, "tcmalloc.thread_cache_free_bytes") == 0) { - TCMallocStats stats; - ExtractStats(&stats, NULL, NULL, NULL); - *value = stats.thread_bytes; - return true; - } - - if (strcmp(name, "tcmalloc.pageheap_free_bytes") == 0) { - SpinLockHolder l(Static::pageheap_lock()); - *value = Static::pageheap()->stats().free_bytes; - return true; - } - - if (strcmp(name, "tcmalloc.pageheap_unmapped_bytes") == 0) { - SpinLockHolder l(Static::pageheap_lock()); - *value = Static::pageheap()->stats().unmapped_bytes; - return true; - } - - if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) { - SpinLockHolder l(Static::pageheap_lock()); - *value = ThreadCache::overall_thread_cache_size(); - return true; - } - - if (strcmp(name, "tcmalloc.current_total_thread_cache_bytes") == 0) { - TCMallocStats stats; - ExtractStats(&stats, NULL, NULL, NULL); - *value = stats.thread_bytes; - return true; - } - - if (strcmp(name, "tcmalloc.aggressive_memory_decommit") == 0) { - *value = size_t(Static::pageheap()->GetAggressiveDecommit()); - return true; - } - - return false; - } - - virtual bool SetNumericProperty(const char* name, size_t value) { - ASSERT(name != NULL); - - if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) { - SpinLockHolder l(Static::pageheap_lock()); - ThreadCache::set_overall_thread_cache_size(value); - return true; - } - - if (strcmp(name, "tcmalloc.aggressive_memory_decommit") == 0) { - Static::pageheap()->SetAggressiveDecommit(value != 0); - return true; - } - - return false; - } - - virtual void MarkThreadIdle() { - ThreadCache::BecomeIdle(); - } - - virtual void MarkThreadBusy(); // Implemented below - - virtual SysAllocator* GetSystemAllocator() { - SpinLockHolder h(Static::pageheap_lock()); - return sys_alloc; - } - - virtual void SetSystemAllocator(SysAllocator* alloc) { - SpinLockHolder h(Static::pageheap_lock()); - sys_alloc = alloc; - } - - virtual void ReleaseToSystem(size_t num_bytes) { - SpinLockHolder h(Static::pageheap_lock()); - if (num_bytes <= extra_bytes_released_) { - // We released too much on a prior call, so don't release any - // more this time. - extra_bytes_released_ = extra_bytes_released_ - num_bytes; - return; - } - num_bytes = num_bytes - extra_bytes_released_; - // num_bytes might be less than one page. If we pass zero to - // ReleaseAtLeastNPages, it won't do anything, so we release a whole - // page now and let extra_bytes_released_ smooth it out over time. - Length num_pages = max(num_bytes >> kPageShift, 1); - size_t bytes_released = Static::pageheap()->ReleaseAtLeastNPages( - num_pages) << kPageShift; - if (bytes_released > num_bytes) { - extra_bytes_released_ = bytes_released - num_bytes; - } else { - // The PageHeap wasn't able to release num_bytes. Don't try to - // compensate with a big release next time. Specifically, - // ReleaseFreeMemory() calls ReleaseToSystem(LONG_MAX). - extra_bytes_released_ = 0; - } - } - - virtual void SetMemoryReleaseRate(double rate) { - FLAGS_tcmalloc_release_rate = rate; - } - - virtual double GetMemoryReleaseRate() { - return FLAGS_tcmalloc_release_rate; - } - virtual size_t GetEstimatedAllocatedSize(size_t size) { - if (size <= kMaxSize) { - const size_t cl = Static::sizemap()->SizeClass(size); - const size_t alloc_size = Static::sizemap()->ByteSizeForClass(cl); - return alloc_size; - } else { - return tcmalloc::pages(size) << kPageShift; - } - } - - // This just calls GetSizeWithCallback, but because that's in an - // unnamed namespace, we need to move the definition below it in the - // file. - virtual size_t GetAllocatedSize(const void* ptr); - - // This duplicates some of the logic in GetSizeWithCallback, but is - // faster. This is important on OS X, where this function is called - // on every allocation operation. - virtual Ownership GetOwnership(const void* ptr) { - const PageID p = reinterpret_cast(ptr) >> kPageShift; - // The rest of tcmalloc assumes that all allocated pointers use at - // most kAddressBits bits. If ptr doesn't, then it definitely - // wasn't alloacted by tcmalloc. - if ((p >> (kAddressBits - kPageShift)) > 0) { - return kNotOwned; - } - size_t cl = Static::pageheap()->GetSizeClassIfCached(p); - if (cl != 0) { - return kOwned; - } - const Span *span = Static::pageheap()->GetDescriptor(p); - return span ? kOwned : kNotOwned; - } - - virtual void GetFreeListSizes(vector* v) { - static const char* kCentralCacheType = "tcmalloc.central"; - static const char* kTransferCacheType = "tcmalloc.transfer"; - static const char* kThreadCacheType = "tcmalloc.thread"; - static const char* kPageHeapType = "tcmalloc.page"; - static const char* kPageHeapUnmappedType = "tcmalloc.page_unmapped"; - static const char* kLargeSpanType = "tcmalloc.large"; - static const char* kLargeUnmappedSpanType = "tcmalloc.large_unmapped"; - - v->clear(); - - // central class information - int64 prev_class_size = 0; - for (int cl = 1; cl < kNumClasses; ++cl) { - size_t class_size = Static::sizemap()->ByteSizeForClass(cl); - MallocExtension::FreeListInfo i; - i.min_object_size = prev_class_size + 1; - i.max_object_size = class_size; - i.total_bytes_free = - Static::central_cache()[cl].length() * class_size; - i.type = kCentralCacheType; - v->push_back(i); - - // transfer cache - i.total_bytes_free = - Static::central_cache()[cl].tc_length() * class_size; - i.type = kTransferCacheType; - v->push_back(i); - - prev_class_size = Static::sizemap()->ByteSizeForClass(cl); - } - - // Add stats from per-thread heaps - uint64_t class_count[kNumClasses]; - memset(class_count, 0, sizeof(class_count)); - { - SpinLockHolder h(Static::pageheap_lock()); - uint64_t thread_bytes = 0; - ThreadCache::GetThreadStats(&thread_bytes, class_count); - } - - prev_class_size = 0; - for (int cl = 1; cl < kNumClasses; ++cl) { - MallocExtension::FreeListInfo i; - i.min_object_size = prev_class_size + 1; - i.max_object_size = Static::sizemap()->ByteSizeForClass(cl); - i.total_bytes_free = - class_count[cl] * Static::sizemap()->ByteSizeForClass(cl); - i.type = kThreadCacheType; - v->push_back(i); - } - - // append page heap info - PageHeap::SmallSpanStats small; - PageHeap::LargeSpanStats large; - { - SpinLockHolder h(Static::pageheap_lock()); - Static::pageheap()->GetSmallSpanStats(&small); - Static::pageheap()->GetLargeSpanStats(&large); - } - - // large spans: mapped - MallocExtension::FreeListInfo span_info; - span_info.type = kLargeSpanType; - span_info.max_object_size = (numeric_limits::max)(); - span_info.min_object_size = kMaxPages << kPageShift; - span_info.total_bytes_free = large.normal_pages << kPageShift; - v->push_back(span_info); - - // large spans: unmapped - span_info.type = kLargeUnmappedSpanType; - span_info.total_bytes_free = large.returned_pages << kPageShift; - v->push_back(span_info); - - // small spans - for (int s = 1; s < kMaxPages; s++) { - MallocExtension::FreeListInfo i; - i.max_object_size = (s << kPageShift); - i.min_object_size = ((s - 1) << kPageShift); - - i.type = kPageHeapType; - i.total_bytes_free = (s << kPageShift) * small.normal_length[s]; - v->push_back(i); - - i.type = kPageHeapUnmappedType; - i.total_bytes_free = (s << kPageShift) * small.returned_length[s]; - v->push_back(i); - } - } -}; - -// The constructor allocates an object to ensure that initialization -// runs before main(), and therefore we do not have a chance to become -// multi-threaded before initialization. We also create the TSD key -// here. Presumably by the time this constructor runs, glibc is in -// good enough shape to handle pthread_key_create(). -// -// The constructor also takes the opportunity to tell STL to use -// tcmalloc. We want to do this early, before construct time, so -// all user STL allocations go through tcmalloc (which works really -// well for STL). -// -// The destructor prints stats when the program exits. -static int tcmallocguard_refcount = 0; // no lock needed: runs before main() -TCMallocGuard::TCMallocGuard() { - if (tcmallocguard_refcount++ == 0) { - ReplaceSystemAlloc(); // defined in libc_override_*.h - tc_free(tc_malloc(1)); - ThreadCache::InitTSD(); - tc_free(tc_malloc(1)); - // Either we, or debugallocation.cc, or valgrind will control memory - // management. We register our extension if we're the winner. -#ifdef TCMALLOC_USING_DEBUGALLOCATION - // Let debugallocation register its extension. -#else - if (RunningOnValgrind()) { - // Let Valgrind uses its own malloc (so don't register our extension). - } else { - MallocExtension::Register(new TCMallocImplementation); - } -#endif - } -} - -TCMallocGuard::~TCMallocGuard() { - if (--tcmallocguard_refcount == 0) { - const char* env = NULL; - if (!RunningOnValgrind()) { - // Valgrind uses it's own malloc so we cannot do MALLOCSTATS - env = getenv("MALLOCSTATS"); - } - if (env != NULL) { - int level = atoi(env); - if (level < 1) level = 1; - PrintStats(level); - } - } -} -#ifndef WIN32_OVERRIDE_ALLOCATORS -static TCMallocGuard module_enter_exit_hook; -#endif - -//------------------------------------------------------------------- -// Helpers for the exported routines below -//------------------------------------------------------------------- - -static inline bool CheckCachedSizeClass(void *ptr) { - PageID p = reinterpret_cast(ptr) >> kPageShift; - size_t cached_value = Static::pageheap()->GetSizeClassIfCached(p); - return cached_value == 0 || - cached_value == Static::pageheap()->GetDescriptor(p)->sizeclass; -} - -static inline void* CheckedMallocResult(void *result) { - ASSERT(result == NULL || CheckCachedSizeClass(result)); - return result; -} - -static inline void* SpanToMallocResult(Span *span) { - Static::pageheap()->CacheSizeClass(span->start, 0); - return - CheckedMallocResult(reinterpret_cast(span->start << kPageShift)); -} - -static void* DoSampledAllocation(size_t size) { -#ifndef NO_TCMALLOC_SAMPLES - // Grab the stack trace outside the heap lock - StackTrace tmp; - tmp.depth = GetStackTrace(tmp.stack, tcmalloc::kMaxStackDepth, 1); - tmp.size = size; - - SpinLockHolder h(Static::pageheap_lock()); - // Allocate span - Span *span = Static::pageheap()->New(tcmalloc::pages(size == 0 ? 1 : size)); - if (UNLIKELY(span == NULL)) { - return NULL; - } - - // Allocate stack trace - StackTrace *stack = Static::stacktrace_allocator()->New(); - if (UNLIKELY(stack == NULL)) { - // Sampling failed because of lack of memory - return span; - } - *stack = tmp; - span->sample = 1; - span->objects = stack; - tcmalloc::DLL_Prepend(Static::sampled_objects(), span); - - return SpanToMallocResult(span); -#else - abort(); -#endif -} - -namespace { - -typedef void* (*malloc_fn)(void *arg); - -SpinLock set_new_handler_lock(SpinLock::LINKER_INITIALIZED); - -void* handle_oom(malloc_fn retry_fn, - void* retry_arg, - bool from_operator, - bool nothrow) { - if (!from_operator && !tc_new_mode) { - // we're out of memory in C library function (malloc etc) and no - // "new mode" forced on us. Just return NULL - return NULL; - } - // we're OOM in operator new or "new mode" is set. We might have to - // call new_handle and maybe retry allocation. - - for (;;) { - // Get the current new handler. NB: this function is not - // thread-safe. We make a feeble stab at making it so here, but - // this lock only protects against tcmalloc interfering with - // itself, not with other libraries calling set_new_handler. - std::new_handler nh; - { - SpinLockHolder h(&set_new_handler_lock); - nh = std::set_new_handler(0); - (void) std::set_new_handler(nh); - } -#if (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS) - if (!nh) { - return NULL; - } - // Since exceptions are disabled, we don't really know if new_handler - // failed. Assume it will abort if it fails. - (*nh)(); -#else - // If no new_handler is established, the allocation failed. - if (!nh) { - if (nothrow) { - return NULL; - } - throw std::bad_alloc(); - } - // Otherwise, try the new_handler. If it returns, retry the - // allocation. If it throws std::bad_alloc, fail the allocation. - // if it throws something else, don't interfere. - try { - (*nh)(); - } catch (const std::bad_alloc&) { - if (!nothrow) throw; - return NULL; - } -#endif // (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS) - - // we get here if new_handler returns successfully. So we retry - // allocation. - void* rv = retry_fn(retry_arg); - if (rv != NULL) { - return rv; - } - - // if allocation failed again we go to next loop iteration - } -} - -// Copy of FLAGS_tcmalloc_large_alloc_report_threshold with -// automatic increases factored in. -static int64_t large_alloc_threshold = - (kPageSize > FLAGS_tcmalloc_large_alloc_report_threshold - ? kPageSize : FLAGS_tcmalloc_large_alloc_report_threshold); - -static void ReportLargeAlloc(Length num_pages, void* result) { - StackTrace stack; - stack.depth = GetStackTrace(stack.stack, tcmalloc::kMaxStackDepth, 1); - - static const int N = 1000; - char buffer[N]; - TCMalloc_Printer printer(buffer, N); - printer.printf("tcmalloc: large alloc %" PRIu64 " bytes == %p @ ", - static_cast(num_pages) << kPageShift, - result); - for (int i = 0; i < stack.depth; i++) { - printer.printf(" %p", stack.stack[i]); - } - printer.printf("\n"); - write(STDERR_FILENO, buffer, strlen(buffer)); -} - -void* do_memalign(size_t align, size_t size); - -struct retry_memaligh_data { - size_t align; - size_t size; -}; - -static void *retry_do_memalign(void *arg) { - retry_memaligh_data *data = static_cast(arg); - return do_memalign(data->align, data->size); -} - -static void *maybe_do_cpp_memalign_slow(size_t align, size_t size) { - retry_memaligh_data data; - data.align = align; - data.size = size; - return handle_oom(retry_do_memalign, &data, - false, true); -} - -inline void* do_memalign_or_cpp_memalign(size_t align, size_t size) { - void *rv = do_memalign(align, size); - if (LIKELY(rv != NULL)) { - return rv; - } - return maybe_do_cpp_memalign_slow(align, size); -} - -// Must be called with the page lock held. -inline bool should_report_large(Length num_pages) { - const int64 threshold = large_alloc_threshold; - if (threshold > 0 && num_pages >= (threshold >> kPageShift)) { - // Increase the threshold by 1/8 every time we generate a report. - // We cap the threshold at 8GiB to avoid overflow problems. - large_alloc_threshold = (threshold + threshold/8 < 8ll<<30 - ? threshold + threshold/8 : 8ll<<30); - return true; - } - return false; -} - -// Helper for do_malloc(). -inline void* do_malloc_pages(ThreadCache* heap, size_t size) { - void* result; - bool report_large; - - Length num_pages = tcmalloc::pages(size); - - // NOTE: we're passing original size here as opposed to rounded-up - // size as we do in do_malloc_small. The difference is small here - // (at most 4k out of at least 256k). And not rounding up saves us - // from possibility of overflow, which rounding up could produce. - // - // See https://github.com/gperftools/gperftools/issues/723 - if (heap->SampleAllocation(size)) { - result = DoSampledAllocation(size); - - SpinLockHolder h(Static::pageheap_lock()); - report_large = should_report_large(num_pages); - } else { - SpinLockHolder h(Static::pageheap_lock()); - Span* span = Static::pageheap()->New(num_pages); - result = (UNLIKELY(span == NULL) ? NULL : SpanToMallocResult(span)); - report_large = should_report_large(num_pages); - } - - if (report_large) { - ReportLargeAlloc(num_pages, result); - } - return result; -} - -ALWAYS_INLINE void* do_malloc_small(ThreadCache* heap, size_t size) { - ASSERT(Static::IsInited()); - ASSERT(heap != NULL); - size_t cl = Static::sizemap()->SizeClass(size); - size = Static::sizemap()->class_to_size(cl); - - if (UNLIKELY(heap->SampleAllocation(size))) { - return DoSampledAllocation(size); - } else { - // The common case, and also the simplest. This just pops the - // size-appropriate freelist, after replenishing it if it's empty. - return CheckedMallocResult(heap->Allocate(size, cl)); - } -} - -ALWAYS_INLINE void* do_malloc(size_t size) { - if (ThreadCache::have_tls) { - if (LIKELY(size < ThreadCache::MinSizeForSlowPath())) { - return do_malloc_small(ThreadCache::GetCacheWhichMustBePresent(), size); - } - if (UNLIKELY(ThreadCache::IsUseEmergencyMalloc())) { - return tcmalloc::EmergencyMalloc(size); - } - } - - if (size <= kMaxSize) { - return do_malloc_small(ThreadCache::GetCache(), size); - } else { - return do_malloc_pages(ThreadCache::GetCache(), size); - } -} - -static void *retry_malloc(void* size) { - return do_malloc(reinterpret_cast(size)); -} - -ALWAYS_INLINE void* do_malloc_or_cpp_alloc(size_t size) { - void *rv = do_malloc(size); - if (LIKELY(rv != NULL)) { - return rv; - } - return handle_oom(retry_malloc, reinterpret_cast(size), - false, true); -} - -ALWAYS_INLINE void* do_calloc(size_t n, size_t elem_size) { - // Overflow check - const size_t size = n * elem_size; - if (elem_size != 0 && size / elem_size != n) return NULL; - - void* result = do_malloc_or_cpp_alloc(size); - if (result != NULL) { - memset(result, 0, size); - } - return result; -} - -// If ptr is NULL, do nothing. Otherwise invoke the given function. -inline void free_null_or_invalid(void* ptr, void (*invalid_free_fn)(void*)) { - if (ptr != NULL) { - (*invalid_free_fn)(ptr); - } -} - -// Helper for do_free_with_callback(), below. Inputs: -// ptr is object to be freed -// invalid_free_fn is a function that gets invoked on certain "bad frees" -// heap is the ThreadCache for this thread, or NULL if it isn't known -// heap_must_be_valid is whether heap is known to be non-NULL -// -// This function may only be used after Static::IsInited() is true. -// -// We can usually detect the case where ptr is not pointing to a page that -// tcmalloc is using, and in those cases we invoke invalid_free_fn. -// -// To maximize speed in the common case, we usually get here with -// heap_must_be_valid being a manifest constant equal to true. -ALWAYS_INLINE void do_free_helper(void* ptr, - void (*invalid_free_fn)(void*), - ThreadCache* heap, - bool heap_must_be_valid, - bool use_hint, - size_t size_hint) { - ASSERT((Static::IsInited() && heap != NULL) || !heap_must_be_valid); - if (!heap_must_be_valid && !Static::IsInited()) { - // We called free() before malloc(). This can occur if the - // (system) malloc() is called before tcmalloc is loaded, and then - // free() is called after tcmalloc is loaded (and tc_free has - // replaced free), but before the global constructor has run that - // sets up the tcmalloc data structures. - free_null_or_invalid(ptr, invalid_free_fn); - return; - } - Span* span = NULL; - const PageID p = reinterpret_cast(ptr) >> kPageShift; - size_t cl; - if (use_hint && Static::sizemap()->MaybeSizeClass(size_hint, &cl)) { - goto non_zero; - } - - cl = Static::pageheap()->GetSizeClassIfCached(p); - if (UNLIKELY(cl == 0)) { - span = Static::pageheap()->GetDescriptor(p); - if (UNLIKELY(!span)) { - // span can be NULL because the pointer passed in is NULL or invalid - // (not something returned by malloc or friends), or because the - // pointer was allocated with some other allocator besides - // tcmalloc. The latter can happen if tcmalloc is linked in via - // a dynamic library, but is not listed last on the link line. - // In that case, libraries after it on the link line will - // allocate with libc malloc, but free with tcmalloc's free. - free_null_or_invalid(ptr, invalid_free_fn); - return; - } - cl = span->sizeclass; - Static::pageheap()->CacheSizeClass(p, cl); - } - - ASSERT(ptr != NULL); - if (LIKELY(cl != 0)) { - non_zero: - ASSERT(!Static::pageheap()->GetDescriptor(p)->sample); - if (heap_must_be_valid || heap != NULL) { - heap->Deallocate(ptr, cl); - } else { - // Delete directly into central cache - tcmalloc::SLL_SetNext(ptr, NULL); - Static::central_cache()[cl].InsertRange(ptr, ptr, 1); - } - } else { - SpinLockHolder h(Static::pageheap_lock()); - ASSERT(reinterpret_cast(ptr) % kPageSize == 0); - ASSERT(span != NULL && span->start == p); - if (span->sample) { - StackTrace* st = reinterpret_cast(span->objects); - tcmalloc::DLL_Remove(span); - Static::stacktrace_allocator()->Delete(st); - span->objects = NULL; - } - Static::pageheap()->Delete(span); - } -} - -// Helper for the object deletion (free, delete, etc.). Inputs: -// ptr is object to be freed -// invalid_free_fn is a function that gets invoked on certain "bad frees" -// -// We can usually detect the case where ptr is not pointing to a page that -// tcmalloc is using, and in those cases we invoke invalid_free_fn. -ALWAYS_INLINE void do_free_with_callback(void* ptr, - void (*invalid_free_fn)(void*), - bool use_hint, size_t size_hint) { - ThreadCache* heap = NULL; - heap = ThreadCache::GetCacheIfPresent(); - if (LIKELY(heap)) { - do_free_helper(ptr, invalid_free_fn, heap, true, use_hint, size_hint); - } else { - do_free_helper(ptr, invalid_free_fn, heap, false, use_hint, size_hint); - } -} - -// The default "do_free" that uses the default callback. -ALWAYS_INLINE void do_free(void* ptr) { - return do_free_with_callback(ptr, &InvalidFree, false, 0); -} - -// NOTE: some logic here is duplicated in GetOwnership (above), for -// speed. If you change this function, look at that one too. -inline size_t GetSizeWithCallback(const void* ptr, - size_t (*invalid_getsize_fn)(const void*)) { - if (ptr == NULL) - return 0; - const PageID p = reinterpret_cast(ptr) >> kPageShift; - size_t cl = Static::pageheap()->GetSizeClassIfCached(p); - if (cl != 0) { - return Static::sizemap()->ByteSizeForClass(cl); - } else { - const Span *span = Static::pageheap()->GetDescriptor(p); - if (UNLIKELY(span == NULL)) { // means we do not own this memory - return (*invalid_getsize_fn)(ptr); - } else if (span->sizeclass != 0) { - Static::pageheap()->CacheSizeClass(p, span->sizeclass); - return Static::sizemap()->ByteSizeForClass(span->sizeclass); - } else { - return span->length << kPageShift; - } - } -} - -// This lets you call back to a given function pointer if ptr is invalid. -// It is used primarily by windows code which wants a specialized callback. -ALWAYS_INLINE void* do_realloc_with_callback( - void* old_ptr, size_t new_size, - void (*invalid_free_fn)(void*), - size_t (*invalid_get_size_fn)(const void*)) { - // Get the size of the old entry - const size_t old_size = GetSizeWithCallback(old_ptr, invalid_get_size_fn); - - // Reallocate if the new size is larger than the old size, - // or if the new size is significantly smaller than the old size. - // We do hysteresis to avoid resizing ping-pongs: - // . If we need to grow, grow to max(new_size, old_size * 1.X) - // . Don't shrink unless new_size < old_size * 0.Y - // X and Y trade-off time for wasted space. For now we do 1.25 and 0.5. - const size_t lower_bound_to_grow = old_size + old_size / 4ul; - const size_t upper_bound_to_shrink = old_size / 2ul; - if ((new_size > old_size) || (new_size < upper_bound_to_shrink)) { - // Need to reallocate. - void* new_ptr = NULL; - - if (new_size > old_size && new_size < lower_bound_to_grow) { - new_ptr = do_malloc_or_cpp_alloc(lower_bound_to_grow); - } - if (new_ptr == NULL) { - // Either new_size is not a tiny increment, or last do_malloc failed. - new_ptr = do_malloc_or_cpp_alloc(new_size); - } - if (UNLIKELY(new_ptr == NULL)) { - return NULL; - } - MallocHook::InvokeNewHook(new_ptr, new_size); - memcpy(new_ptr, old_ptr, ((old_size < new_size) ? old_size : new_size)); - MallocHook::InvokeDeleteHook(old_ptr); - // We could use a variant of do_free() that leverages the fact - // that we already know the sizeclass of old_ptr. The benefit - // would be small, so don't bother. - do_free_with_callback(old_ptr, invalid_free_fn, false, 0); - return new_ptr; - } else { - // We still need to call hooks to report the updated size: - MallocHook::InvokeDeleteHook(old_ptr); - MallocHook::InvokeNewHook(old_ptr, new_size); - return old_ptr; - } -} - -ALWAYS_INLINE void* do_realloc(void* old_ptr, size_t new_size) { - return do_realloc_with_callback(old_ptr, new_size, - &InvalidFree, &InvalidGetSizeForRealloc); -} - -// For use by exported routines below that want specific alignments -// -// Note: this code can be slow for alignments > 16, and can -// significantly fragment memory. The expectation is that -// memalign/posix_memalign/valloc/pvalloc will not be invoked very -// often. This requirement simplifies our implementation and allows -// us to tune for expected allocation patterns. -void* do_memalign(size_t align, size_t size) { - ASSERT((align & (align - 1)) == 0); - ASSERT(align > 0); - if (size + align < size) return NULL; // Overflow - - // Fall back to malloc if we would already align this memory access properly. - if (align <= AlignmentForSize(size)) { - void* p = do_malloc(size); - ASSERT((reinterpret_cast(p) % align) == 0); - return p; - } - - if (UNLIKELY(Static::pageheap() == NULL)) ThreadCache::InitModule(); - - // Allocate at least one byte to avoid boundary conditions below - if (size == 0) size = 1; - - if (size <= kMaxSize && align < kPageSize) { - // Search through acceptable size classes looking for one with - // enough alignment. This depends on the fact that - // InitSizeClasses() currently produces several size classes that - // are aligned at powers of two. We will waste time and space if - // we miss in the size class array, but that is deemed acceptable - // since memalign() should be used rarely. - int cl = Static::sizemap()->SizeClass(size); - while (cl < kNumClasses && - ((Static::sizemap()->class_to_size(cl) & (align - 1)) != 0)) { - cl++; - } - if (cl < kNumClasses) { - ThreadCache* heap = ThreadCache::GetCache(); - size = Static::sizemap()->class_to_size(cl); - return CheckedMallocResult(heap->Allocate(size, cl)); - } - } - - // We will allocate directly from the page heap - SpinLockHolder h(Static::pageheap_lock()); - - if (align <= kPageSize) { - // Any page-level allocation will be fine - // TODO: We could put the rest of this page in the appropriate - // TODO: cache but it does not seem worth it. - Span* span = Static::pageheap()->New(tcmalloc::pages(size)); - return UNLIKELY(span == NULL) ? NULL : SpanToMallocResult(span); - } - - // Allocate extra pages and carve off an aligned portion - const Length alloc = tcmalloc::pages(size + align); - Span* span = Static::pageheap()->New(alloc); - if (UNLIKELY(span == NULL)) return NULL; - - // Skip starting portion so that we end up aligned - Length skip = 0; - while ((((span->start+skip) << kPageShift) & (align - 1)) != 0) { - skip++; - } - ASSERT(skip < alloc); - if (skip > 0) { - Span* rest = Static::pageheap()->Split(span, skip); - Static::pageheap()->Delete(span); - span = rest; - } - - // Skip trailing portion that we do not need to return - const Length needed = tcmalloc::pages(size); - ASSERT(span->length >= needed); - if (span->length > needed) { - Span* trailer = Static::pageheap()->Split(span, needed); - Static::pageheap()->Delete(trailer); - } - return SpanToMallocResult(span); -} - -// Helpers for use by exported routines below: - -inline void do_malloc_stats() { - PrintStats(1); -} - -inline int do_mallopt(int cmd, int value) { - return 1; // Indicates error -} - -#ifdef HAVE_STRUCT_MALLINFO -inline struct mallinfo do_mallinfo() { - TCMallocStats stats; - ExtractStats(&stats, NULL, NULL, NULL); - - // Just some of the fields are filled in. - struct mallinfo info; - memset(&info, 0, sizeof(info)); - - // Unfortunately, the struct contains "int" field, so some of the - // size values will be truncated. - info.arena = static_cast(stats.pageheap.system_bytes); - info.fsmblks = static_cast(stats.thread_bytes - + stats.central_bytes - + stats.transfer_bytes); - info.fordblks = static_cast(stats.pageheap.free_bytes + - stats.pageheap.unmapped_bytes); - info.uordblks = static_cast(stats.pageheap.system_bytes - - stats.thread_bytes - - stats.central_bytes - - stats.transfer_bytes - - stats.pageheap.free_bytes - - stats.pageheap.unmapped_bytes); - - return info; -} -#endif // HAVE_STRUCT_MALLINFO - -inline void* cpp_alloc(size_t size, bool nothrow) { - void* p = do_malloc(size); - if (LIKELY(p)) { - return p; - } - return handle_oom(retry_malloc, reinterpret_cast(size), - true, nothrow); -} - -} // end unnamed namespace - -// As promised, the definition of this function, declared above. -size_t TCMallocImplementation::GetAllocatedSize(const void* ptr) { - if (ptr == NULL) - return 0; - ASSERT(TCMallocImplementation::GetOwnership(ptr) - != TCMallocImplementation::kNotOwned); - return GetSizeWithCallback(ptr, &InvalidGetAllocatedSize); -} - -void TCMallocImplementation::MarkThreadBusy() { - // Allocate to force the creation of a thread cache, but avoid - // invoking any hooks. - do_free(do_malloc(0)); -} - -//------------------------------------------------------------------- -// Exported routines -//------------------------------------------------------------------- - -extern "C" PERFTOOLS_DLL_DECL const char* tc_version( - int* major, int* minor, const char** patch) PERFTOOLS_THROW { - if (major) *major = TC_VERSION_MAJOR; - if (minor) *minor = TC_VERSION_MINOR; - if (patch) *patch = TC_VERSION_PATCH; - return TC_VERSION_STRING; -} - -// This function behaves similarly to MSVC's _set_new_mode. -// If flag is 0 (default), calls to malloc will behave normally. -// If flag is 1, calls to malloc will behave like calls to new, -// and the std_new_handler will be invoked on failure. -// Returns the previous mode. -extern "C" PERFTOOLS_DLL_DECL int tc_set_new_mode(int flag) PERFTOOLS_THROW { - int old_mode = tc_new_mode; - tc_new_mode = flag; - return old_mode; -} - -#ifndef TCMALLOC_USING_DEBUGALLOCATION // debugallocation.cc defines its own - -#if defined(__GNUC__) && defined(__ELF__) && !defined(TCMALLOC_NO_ALIASES) -#define TC_ALIAS(name) __attribute__((alias(#name))) -#endif - -// CAVEAT: The code structure below ensures that MallocHook methods are always -// called from the stack frame of the invoked allocation function. -// heap-checker.cc depends on this to start a stack trace from -// the call to the (de)allocation function. - -extern "C" PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) PERFTOOLS_THROW { - void* result = do_malloc_or_cpp_alloc(size); - MallocHook::InvokeNewHook(result, size); - return result; -} - -extern "C" PERFTOOLS_DLL_DECL void tc_free(void* ptr) PERFTOOLS_THROW { - MallocHook::InvokeDeleteHook(ptr); - do_free(ptr); -} - -extern "C" PERFTOOLS_DLL_DECL void tc_free_sized(void *ptr, size_t size) PERFTOOLS_THROW { - if ((reinterpret_cast(ptr) & (kPageSize-1)) == 0) { - tc_free(ptr); - return; - } - MallocHook::InvokeDeleteHook(ptr); - do_free_with_callback(ptr, &InvalidFree, true, size); -} - -#ifdef TC_ALIAS - -extern "C" PERFTOOLS_DLL_DECL void tc_delete_sized(void *p, size_t size) throw() - TC_ALIAS(tc_free_sized); -extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_sized(void *p, size_t size) throw() - TC_ALIAS(tc_free_sized); - -#else - -extern "C" PERFTOOLS_DLL_DECL void tc_delete_sized(void *p, size_t size) throw() { - tc_free_sized(p, size); -} -extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_sized(void *p, size_t size) throw() { - tc_free_sized(p, size); -} - -#endif - -extern "C" PERFTOOLS_DLL_DECL void* tc_calloc(size_t n, - size_t elem_size) PERFTOOLS_THROW { - if (ThreadCache::IsUseEmergencyMalloc()) { - return tcmalloc::EmergencyCalloc(n, elem_size); - } - void* result = do_calloc(n, elem_size); - MallocHook::InvokeNewHook(result, n * elem_size); - return result; -} - -extern "C" PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) PERFTOOLS_THROW -#ifdef TC_ALIAS -TC_ALIAS(tc_free); -#else -{ - MallocHook::InvokeDeleteHook(ptr); - do_free(ptr); -} -#endif - -extern "C" PERFTOOLS_DLL_DECL void* tc_realloc(void* old_ptr, - size_t new_size) PERFTOOLS_THROW { - if (old_ptr == NULL) { - void* result = do_malloc_or_cpp_alloc(new_size); - MallocHook::InvokeNewHook(result, new_size); - return result; - } - if (new_size == 0) { - MallocHook::InvokeDeleteHook(old_ptr); - do_free(old_ptr); - return NULL; - } - if (UNLIKELY(tcmalloc::IsEmergencyPtr(old_ptr))) { - return tcmalloc::EmergencyRealloc(old_ptr, new_size); - } - return do_realloc(old_ptr, new_size); -} - -extern "C" PERFTOOLS_DLL_DECL void* tc_new(size_t size) { - void* p = cpp_alloc(size, false); - // We keep this next instruction out of cpp_alloc for a reason: when - // it's in, and new just calls cpp_alloc, the optimizer may fold the - // new call into cpp_alloc, which messes up our whole section-based - // stacktracing (see ATTRIBUTE_SECTION, above). This ensures cpp_alloc - // isn't the last thing this fn calls, and prevents the folding. - MallocHook::InvokeNewHook(p, size); - return p; -} - -extern "C" PERFTOOLS_DLL_DECL void* tc_new_nothrow(size_t size, const std::nothrow_t&) PERFTOOLS_THROW { - void* p = cpp_alloc(size, true); - MallocHook::InvokeNewHook(p, size); - return p; -} - -extern "C" PERFTOOLS_DLL_DECL void tc_delete(void* p) PERFTOOLS_THROW -#ifdef TC_ALIAS -TC_ALIAS(tc_free); -#else -{ - MallocHook::InvokeDeleteHook(p); - do_free(p); -} -#endif - -// Standard C++ library implementations define and use this -// (via ::operator delete(ptr, nothrow)). -// But it's really the same as normal delete, so we just do the same thing. -extern "C" PERFTOOLS_DLL_DECL void tc_delete_nothrow(void* p, const std::nothrow_t&) PERFTOOLS_THROW -#ifdef TC_ALIAS -TC_ALIAS(tc_free); -#else -{ - MallocHook::InvokeDeleteHook(p); - do_free(p); -} -#endif - -extern "C" PERFTOOLS_DLL_DECL void* tc_newarray(size_t size) -#ifdef TC_ALIAS -TC_ALIAS(tc_new); -#else -{ - void* p = cpp_alloc(size, false); - // We keep this next instruction out of cpp_alloc for a reason: when - // it's in, and new just calls cpp_alloc, the optimizer may fold the - // new call into cpp_alloc, which messes up our whole section-based - // stacktracing (see ATTRIBUTE_SECTION, above). This ensures cpp_alloc - // isn't the last thing this fn calls, and prevents the folding. - MallocHook::InvokeNewHook(p, size); - return p; -} -#endif - -extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_nothrow(size_t size, const std::nothrow_t&) - PERFTOOLS_THROW -#ifdef TC_ALIAS -TC_ALIAS(tc_new_nothrow); -#else -{ - void* p = cpp_alloc(size, true); - MallocHook::InvokeNewHook(p, size); - return p; -} -#endif - -extern "C" PERFTOOLS_DLL_DECL void tc_deletearray(void* p) PERFTOOLS_THROW -#ifdef TC_ALIAS -TC_ALIAS(tc_free); -#else -{ - MallocHook::InvokeDeleteHook(p); - do_free(p); -} -#endif - -extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_nothrow(void* p, const std::nothrow_t&) PERFTOOLS_THROW -#ifdef TC_ALIAS -TC_ALIAS(tc_free); -#else -{ - MallocHook::InvokeDeleteHook(p); - do_free(p); -} -#endif - -extern "C" PERFTOOLS_DLL_DECL void* tc_memalign(size_t align, - size_t size) PERFTOOLS_THROW { - void* result = do_memalign_or_cpp_memalign(align, size); - MallocHook::InvokeNewHook(result, size); - return result; -} - -extern "C" PERFTOOLS_DLL_DECL int tc_posix_memalign( - void** result_ptr, size_t align, size_t size) PERFTOOLS_THROW { - if (((align % sizeof(void*)) != 0) || - ((align & (align - 1)) != 0) || - (align == 0)) { - return EINVAL; - } - - void* result = do_memalign_or_cpp_memalign(align, size); - MallocHook::InvokeNewHook(result, size); - if (UNLIKELY(result == NULL)) { - return ENOMEM; - } else { - *result_ptr = result; - return 0; - } -} - -static size_t pagesize = 0; - -extern "C" PERFTOOLS_DLL_DECL void* tc_valloc(size_t size) PERFTOOLS_THROW { - // Allocate page-aligned object of length >= size bytes - if (pagesize == 0) pagesize = getpagesize(); - void* result = do_memalign_or_cpp_memalign(pagesize, size); - MallocHook::InvokeNewHook(result, size); - return result; -} - -extern "C" PERFTOOLS_DLL_DECL void* tc_pvalloc(size_t size) PERFTOOLS_THROW { - // Round up size to a multiple of pagesize - if (pagesize == 0) pagesize = getpagesize(); - if (size == 0) { // pvalloc(0) should allocate one page, according to - size = pagesize; // http://man.free4web.biz/man3/libmpatrol.3.html - } - size = (size + pagesize - 1) & ~(pagesize - 1); - void* result = do_memalign_or_cpp_memalign(pagesize, size); - MallocHook::InvokeNewHook(result, size); - return result; -} - -extern "C" PERFTOOLS_DLL_DECL void tc_malloc_stats(void) PERFTOOLS_THROW { - do_malloc_stats(); -} - -extern "C" PERFTOOLS_DLL_DECL int tc_mallopt(int cmd, int value) PERFTOOLS_THROW { - return do_mallopt(cmd, value); -} - -#ifdef HAVE_STRUCT_MALLINFO -extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) PERFTOOLS_THROW { - return do_mallinfo(); -} -#endif - -extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) PERFTOOLS_THROW { - return MallocExtension::instance()->GetAllocatedSize(ptr); -} - -extern "C" PERFTOOLS_DLL_DECL void* tc_malloc_skip_new_handler(size_t size) PERFTOOLS_THROW { - void* result = do_malloc(size); - MallocHook::InvokeNewHook(result, size); - return result; -} - -#pragma GCC diagnostic pop - -#endif // TCMALLOC_USING_DEBUGALLOCATION diff --git a/contrib/libtcmalloc/src/tcmalloc.h b/contrib/libtcmalloc/src/tcmalloc.h deleted file mode 100644 index 70d567268c2..00000000000 --- a/contrib/libtcmalloc/src/tcmalloc.h +++ /dev/null @@ -1,70 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2007, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Craig Silverstein -// -// Some obscure memory-allocation routines may not be declared on all -// systems. In those cases, we'll just declare them ourselves. -// This file is meant to be used only internally, for unittests. - -#include "config.h" - -#ifndef _XOPEN_SOURCE -# define _XOPEN_SOURCE 600 // for posix_memalign -#endif -#include // for posix_memalign -// FreeBSD has malloc.h, but complains if you use it -#if defined(HAVE_MALLOC_H) && !defined(__FreeBSD__) -#include // for memalign, valloc, pvalloc -#endif - -// __THROW is defined in glibc systems. It means, counter-intuitively, -// "This function will never throw an exception." It's an optional -// optimization tool, but we may need to use it to match glibc prototypes. -#ifndef __THROW // I guess we're not on a glibc system -# define __THROW // __THROW is just an optimization, so ok to make it "" -#endif - -#if !HAVE_CFREE_SYMBOL -extern "C" void cfree(void* ptr) __THROW; -#endif -#if !HAVE_POSIX_MEMALIGN_SYMBOL -extern "C" int posix_memalign(void** ptr, size_t align, size_t size) __THROW; -#endif -#if !HAVE_MEMALIGN_SYMBOL -extern "C" void* memalign(size_t __alignment, size_t __size) __THROW; -#endif -#if !HAVE_VALLOC_SYMBOL -extern "C" void* valloc(size_t __size) __THROW; -#endif -#if !HAVE_PVALLOC_SYMBOL -extern "C" void* pvalloc(size_t __size) __THROW; -#endif diff --git a/contrib/libtcmalloc/src/tcmalloc_guard.h b/contrib/libtcmalloc/src/tcmalloc_guard.h deleted file mode 100644 index 84952bac2ea..00000000000 --- a/contrib/libtcmalloc/src/tcmalloc_guard.h +++ /dev/null @@ -1,49 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Craig Silverstein -// -// We expose the TCMallocGuard class -- which initializes the tcmalloc -// allocator -- so classes that need to be sure tcmalloc is loaded -// before they do stuff -- notably heap-profiler -- can. To use this -// create a static TCMallocGuard instance at the top of a file where -// you need tcmalloc to be initialized before global constructors run. - -#ifndef TCMALLOC_TCMALLOC_GUARD_H_ -#define TCMALLOC_TCMALLOC_GUARD_H_ - -class TCMallocGuard { - public: - TCMallocGuard(); - ~TCMallocGuard(); -}; - -#endif // TCMALLOC_TCMALLOC_GUARD_H_ diff --git a/contrib/libtcmalloc/src/third_party/valgrind.h b/contrib/libtcmalloc/src/third_party/valgrind.h deleted file mode 100644 index 577c59ab0cd..00000000000 --- a/contrib/libtcmalloc/src/third_party/valgrind.h +++ /dev/null @@ -1,3924 +0,0 @@ -/* -*- c -*- - ---------------------------------------------------------------- - - Notice that the following BSD-style license applies to this one - file (valgrind.h) only. The rest of Valgrind is licensed under the - terms of the GNU General Public License, version 2, unless - otherwise indicated. See the COPYING file in the source - distribution for details. - - ---------------------------------------------------------------- - - This file is part of Valgrind, a dynamic binary instrumentation - framework. - - Copyright (C) 2000-2008 Julian Seward. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - 2. The origin of this software must not be misrepresented; you must - not claim that you wrote the original software. If you use this - software in a product, an acknowledgment in the product - documentation would be appreciated but is not required. - - 3. Altered source versions must be plainly marked as such, and must - not be misrepresented as being the original software. - - 4. The name of the author may not be used to endorse or promote - products derived from this software without specific prior written - permission. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS - OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE - GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - ---------------------------------------------------------------- - - Notice that the above BSD-style license applies to this one file - (valgrind.h) only. The entire rest of Valgrind is licensed under - the terms of the GNU General Public License, version 2. See the - COPYING file in the source distribution for details. - - ---------------------------------------------------------------- -*/ - - -/* This file is for inclusion into client (your!) code. - - You can use these macros to manipulate and query Valgrind's - execution inside your own programs. - - The resulting executables will still run without Valgrind, just a - little bit more slowly than they otherwise would, but otherwise - unchanged. When not running on valgrind, each client request - consumes very few (eg. 7) instructions, so the resulting performance - loss is negligible unless you plan to execute client requests - millions of times per second. Nevertheless, if that is still a - problem, you can compile with the NVALGRIND symbol defined (gcc - -DNVALGRIND) so that client requests are not even compiled in. */ - -#ifndef __VALGRIND_H -#define __VALGRIND_H - -#include - -/* Nb: this file might be included in a file compiled with -ansi. So - we can't use C++ style "//" comments nor the "asm" keyword (instead - use "__asm__"). */ - -/* Derive some tags indicating what the target platform is. Note - that in this file we're using the compiler's CPP symbols for - identifying architectures, which are different to the ones we use - within the rest of Valgrind. Note, __powerpc__ is active for both - 32 and 64-bit PPC, whereas __powerpc64__ is only active for the - latter (on Linux, that is). */ -#undef PLAT_x86_linux -#undef PLAT_amd64_linux -#undef PLAT_ppc32_linux -#undef PLAT_ppc64_linux -#undef PLAT_ppc32_aix5 -#undef PLAT_ppc64_aix5 - -#if !defined(_AIX) && defined(__i386__) -# define PLAT_x86_linux 1 -#elif !defined(_AIX) && defined(__x86_64__) -# define PLAT_amd64_linux 1 -#elif !defined(_AIX) && defined(__powerpc__) && !defined(__powerpc64__) -# define PLAT_ppc32_linux 1 -#elif !defined(_AIX) && defined(__powerpc__) && defined(__powerpc64__) -# define PLAT_ppc64_linux 1 -#elif defined(_AIX) && defined(__64BIT__) -# define PLAT_ppc64_aix5 1 -#elif defined(_AIX) && !defined(__64BIT__) -# define PLAT_ppc32_aix5 1 -#endif - - -/* If we're not compiling for our target platform, don't generate - any inline asms. */ -#if !defined(PLAT_x86_linux) && !defined(PLAT_amd64_linux) \ - && !defined(PLAT_ppc32_linux) && !defined(PLAT_ppc64_linux) \ - && !defined(PLAT_ppc32_aix5) && !defined(PLAT_ppc64_aix5) -# if !defined(NVALGRIND) -# define NVALGRIND 1 -# endif -#endif - - -/* ------------------------------------------------------------------ */ -/* ARCHITECTURE SPECIFICS for SPECIAL INSTRUCTIONS. There is nothing */ -/* in here of use to end-users -- skip to the next section. */ -/* ------------------------------------------------------------------ */ - -#if defined(NVALGRIND) - -/* Define NVALGRIND to completely remove the Valgrind magic sequence - from the compiled code (analogous to NDEBUG's effects on - assert()) */ -#define VALGRIND_DO_CLIENT_REQUEST( \ - _zzq_rlval, _zzq_default, _zzq_request, \ - _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ - { \ - (_zzq_rlval) = (_zzq_default); \ - } - -#else /* ! NVALGRIND */ - -/* The following defines the magic code sequences which the JITter - spots and handles magically. Don't look too closely at them as - they will rot your brain. - - The assembly code sequences for all architectures is in this one - file. This is because this file must be stand-alone, and we don't - want to have multiple files. - - For VALGRIND_DO_CLIENT_REQUEST, we must ensure that the default - value gets put in the return slot, so that everything works when - this is executed not under Valgrind. Args are passed in a memory - block, and so there's no intrinsic limit to the number that could - be passed, but it's currently five. - - The macro args are: - _zzq_rlval result lvalue - _zzq_default default value (result returned when running on real CPU) - _zzq_request request code - _zzq_arg1..5 request params - - The other two macros are used to support function wrapping, and are - a lot simpler. VALGRIND_GET_NR_CONTEXT returns the value of the - guest's NRADDR pseudo-register and whatever other information is - needed to safely run the call original from the wrapper: on - ppc64-linux, the R2 value at the divert point is also needed. This - information is abstracted into a user-visible type, OrigFn. - - VALGRIND_CALL_NOREDIR_* behaves the same as the following on the - guest, but guarantees that the branch instruction will not be - redirected: x86: call *%eax, amd64: call *%rax, ppc32/ppc64: - branch-and-link-to-r11. VALGRIND_CALL_NOREDIR is just text, not a - complete inline asm, since it needs to be combined with more magic - inline asm stuff to be useful. -*/ - -/* ------------------------- x86-linux ------------------------- */ - -#if defined(PLAT_x86_linux) - -typedef - struct { - unsigned int nraddr; /* where's the code? */ - } - OrigFn; - -#define __SPECIAL_INSTRUCTION_PREAMBLE \ - "roll $3, %%edi ; roll $13, %%edi\n\t" \ - "roll $29, %%edi ; roll $19, %%edi\n\t" - -#define VALGRIND_DO_CLIENT_REQUEST( \ - _zzq_rlval, _zzq_default, _zzq_request, \ - _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ - { volatile unsigned int _zzq_args[6]; \ - volatile unsigned int _zzq_result; \ - _zzq_args[0] = (unsigned int)(_zzq_request); \ - _zzq_args[1] = (unsigned int)(_zzq_arg1); \ - _zzq_args[2] = (unsigned int)(_zzq_arg2); \ - _zzq_args[3] = (unsigned int)(_zzq_arg3); \ - _zzq_args[4] = (unsigned int)(_zzq_arg4); \ - _zzq_args[5] = (unsigned int)(_zzq_arg5); \ - __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ - /* %EDX = client_request ( %EAX ) */ \ - "xchgl %%ebx,%%ebx" \ - : "=d" (_zzq_result) \ - : "a" (&_zzq_args[0]), "0" (_zzq_default) \ - : "cc", "memory" \ - ); \ - _zzq_rlval = _zzq_result; \ - } - -#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ - { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ - volatile unsigned int __addr; \ - __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ - /* %EAX = guest_NRADDR */ \ - "xchgl %%ecx,%%ecx" \ - : "=a" (__addr) \ - : \ - : "cc", "memory" \ - ); \ - _zzq_orig->nraddr = __addr; \ - } - -#define VALGRIND_CALL_NOREDIR_EAX \ - __SPECIAL_INSTRUCTION_PREAMBLE \ - /* call-noredir *%EAX */ \ - "xchgl %%edx,%%edx\n\t" -#endif /* PLAT_x86_linux */ - -/* ------------------------ amd64-linux ------------------------ */ - -#if defined(PLAT_amd64_linux) - -typedef - struct { - unsigned long long int nraddr; /* where's the code? */ - } - OrigFn; - -#define __SPECIAL_INSTRUCTION_PREAMBLE \ - "rolq $3, %%rdi ; rolq $13, %%rdi\n\t" \ - "rolq $61, %%rdi ; rolq $51, %%rdi\n\t" - -#define VALGRIND_DO_CLIENT_REQUEST( \ - _zzq_rlval, _zzq_default, _zzq_request, \ - _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ - { volatile unsigned long long int _zzq_args[6]; \ - volatile unsigned long long int _zzq_result; \ - _zzq_args[0] = (unsigned long long int)(_zzq_request); \ - _zzq_args[1] = (unsigned long long int)(_zzq_arg1); \ - _zzq_args[2] = (unsigned long long int)(_zzq_arg2); \ - _zzq_args[3] = (unsigned long long int)(_zzq_arg3); \ - _zzq_args[4] = (unsigned long long int)(_zzq_arg4); \ - _zzq_args[5] = (unsigned long long int)(_zzq_arg5); \ - __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ - /* %RDX = client_request ( %RAX ) */ \ - "xchgq %%rbx,%%rbx" \ - : "=d" (_zzq_result) \ - : "a" (&_zzq_args[0]), "0" (_zzq_default) \ - : "cc", "memory" \ - ); \ - _zzq_rlval = _zzq_result; \ - } - -#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ - { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ - volatile unsigned long long int __addr; \ - __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ - /* %RAX = guest_NRADDR */ \ - "xchgq %%rcx,%%rcx" \ - : "=a" (__addr) \ - : \ - : "cc", "memory" \ - ); \ - _zzq_orig->nraddr = __addr; \ - } - -#define VALGRIND_CALL_NOREDIR_RAX \ - __SPECIAL_INSTRUCTION_PREAMBLE \ - /* call-noredir *%RAX */ \ - "xchgq %%rdx,%%rdx\n\t" -#endif /* PLAT_amd64_linux */ - -/* ------------------------ ppc32-linux ------------------------ */ - -#if defined(PLAT_ppc32_linux) - -typedef - struct { - unsigned int nraddr; /* where's the code? */ - } - OrigFn; - -#define __SPECIAL_INSTRUCTION_PREAMBLE \ - "rlwinm 0,0,3,0,0 ; rlwinm 0,0,13,0,0\n\t" \ - "rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t" - -#define VALGRIND_DO_CLIENT_REQUEST( \ - _zzq_rlval, _zzq_default, _zzq_request, \ - _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ - \ - { unsigned int _zzq_args[6]; \ - unsigned int _zzq_result; \ - unsigned int* _zzq_ptr; \ - _zzq_args[0] = (unsigned int)(_zzq_request); \ - _zzq_args[1] = (unsigned int)(_zzq_arg1); \ - _zzq_args[2] = (unsigned int)(_zzq_arg2); \ - _zzq_args[3] = (unsigned int)(_zzq_arg3); \ - _zzq_args[4] = (unsigned int)(_zzq_arg4); \ - _zzq_args[5] = (unsigned int)(_zzq_arg5); \ - _zzq_ptr = _zzq_args; \ - __asm__ volatile("mr 3,%1\n\t" /*default*/ \ - "mr 4,%2\n\t" /*ptr*/ \ - __SPECIAL_INSTRUCTION_PREAMBLE \ - /* %R3 = client_request ( %R4 ) */ \ - "or 1,1,1\n\t" \ - "mr %0,3" /*result*/ \ - : "=b" (_zzq_result) \ - : "b" (_zzq_default), "b" (_zzq_ptr) \ - : "cc", "memory", "r3", "r4"); \ - _zzq_rlval = _zzq_result; \ - } - -#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ - { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ - unsigned int __addr; \ - __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ - /* %R3 = guest_NRADDR */ \ - "or 2,2,2\n\t" \ - "mr %0,3" \ - : "=b" (__addr) \ - : \ - : "cc", "memory", "r3" \ - ); \ - _zzq_orig->nraddr = __addr; \ - } - -#define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - __SPECIAL_INSTRUCTION_PREAMBLE \ - /* branch-and-link-to-noredir *%R11 */ \ - "or 3,3,3\n\t" -#endif /* PLAT_ppc32_linux */ - -/* ------------------------ ppc64-linux ------------------------ */ - -#if defined(PLAT_ppc64_linux) - -typedef - struct { - unsigned long long int nraddr; /* where's the code? */ - unsigned long long int r2; /* what tocptr do we need? */ - } - OrigFn; - -#define __SPECIAL_INSTRUCTION_PREAMBLE \ - "rotldi 0,0,3 ; rotldi 0,0,13\n\t" \ - "rotldi 0,0,61 ; rotldi 0,0,51\n\t" - -#define VALGRIND_DO_CLIENT_REQUEST( \ - _zzq_rlval, _zzq_default, _zzq_request, \ - _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ - \ - { unsigned long long int _zzq_args[6]; \ - register unsigned long long int _zzq_result __asm__("r3"); \ - register unsigned long long int* _zzq_ptr __asm__("r4"); \ - _zzq_args[0] = (unsigned long long int)(_zzq_request); \ - _zzq_args[1] = (unsigned long long int)(_zzq_arg1); \ - _zzq_args[2] = (unsigned long long int)(_zzq_arg2); \ - _zzq_args[3] = (unsigned long long int)(_zzq_arg3); \ - _zzq_args[4] = (unsigned long long int)(_zzq_arg4); \ - _zzq_args[5] = (unsigned long long int)(_zzq_arg5); \ - _zzq_ptr = _zzq_args; \ - __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ - /* %R3 = client_request ( %R4 ) */ \ - "or 1,1,1" \ - : "=r" (_zzq_result) \ - : "0" (_zzq_default), "r" (_zzq_ptr) \ - : "cc", "memory"); \ - _zzq_rlval = _zzq_result; \ - } - -#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ - { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ - register unsigned long long int __addr __asm__("r3"); \ - __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ - /* %R3 = guest_NRADDR */ \ - "or 2,2,2" \ - : "=r" (__addr) \ - : \ - : "cc", "memory" \ - ); \ - _zzq_orig->nraddr = __addr; \ - __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ - /* %R3 = guest_NRADDR_GPR2 */ \ - "or 4,4,4" \ - : "=r" (__addr) \ - : \ - : "cc", "memory" \ - ); \ - _zzq_orig->r2 = __addr; \ - } - -#define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - __SPECIAL_INSTRUCTION_PREAMBLE \ - /* branch-and-link-to-noredir *%R11 */ \ - "or 3,3,3\n\t" - -#endif /* PLAT_ppc64_linux */ - -/* ------------------------ ppc32-aix5 ------------------------- */ - -#if defined(PLAT_ppc32_aix5) - -typedef - struct { - unsigned int nraddr; /* where's the code? */ - unsigned int r2; /* what tocptr do we need? */ - } - OrigFn; - -#define __SPECIAL_INSTRUCTION_PREAMBLE \ - "rlwinm 0,0,3,0,0 ; rlwinm 0,0,13,0,0\n\t" \ - "rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t" - -#define VALGRIND_DO_CLIENT_REQUEST( \ - _zzq_rlval, _zzq_default, _zzq_request, \ - _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ - \ - { unsigned int _zzq_args[7]; \ - register unsigned int _zzq_result; \ - register unsigned int* _zzq_ptr; \ - _zzq_args[0] = (unsigned int)(_zzq_request); \ - _zzq_args[1] = (unsigned int)(_zzq_arg1); \ - _zzq_args[2] = (unsigned int)(_zzq_arg2); \ - _zzq_args[3] = (unsigned int)(_zzq_arg3); \ - _zzq_args[4] = (unsigned int)(_zzq_arg4); \ - _zzq_args[5] = (unsigned int)(_zzq_arg5); \ - _zzq_args[6] = (unsigned int)(_zzq_default); \ - _zzq_ptr = _zzq_args; \ - __asm__ volatile("mr 4,%1\n\t" \ - "lwz 3, 24(4)\n\t" \ - __SPECIAL_INSTRUCTION_PREAMBLE \ - /* %R3 = client_request ( %R4 ) */ \ - "or 1,1,1\n\t" \ - "mr %0,3" \ - : "=b" (_zzq_result) \ - : "b" (_zzq_ptr) \ - : "r3", "r4", "cc", "memory"); \ - _zzq_rlval = _zzq_result; \ - } - -#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ - { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ - register unsigned int __addr; \ - __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ - /* %R3 = guest_NRADDR */ \ - "or 2,2,2\n\t" \ - "mr %0,3" \ - : "=b" (__addr) \ - : \ - : "r3", "cc", "memory" \ - ); \ - _zzq_orig->nraddr = __addr; \ - __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ - /* %R3 = guest_NRADDR_GPR2 */ \ - "or 4,4,4\n\t" \ - "mr %0,3" \ - : "=b" (__addr) \ - : \ - : "r3", "cc", "memory" \ - ); \ - _zzq_orig->r2 = __addr; \ - } - -#define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - __SPECIAL_INSTRUCTION_PREAMBLE \ - /* branch-and-link-to-noredir *%R11 */ \ - "or 3,3,3\n\t" - -#endif /* PLAT_ppc32_aix5 */ - -/* ------------------------ ppc64-aix5 ------------------------- */ - -#if defined(PLAT_ppc64_aix5) - -typedef - struct { - unsigned long long int nraddr; /* where's the code? */ - unsigned long long int r2; /* what tocptr do we need? */ - } - OrigFn; - -#define __SPECIAL_INSTRUCTION_PREAMBLE \ - "rotldi 0,0,3 ; rotldi 0,0,13\n\t" \ - "rotldi 0,0,61 ; rotldi 0,0,51\n\t" - -#define VALGRIND_DO_CLIENT_REQUEST( \ - _zzq_rlval, _zzq_default, _zzq_request, \ - _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ - \ - { unsigned long long int _zzq_args[7]; \ - register unsigned long long int _zzq_result; \ - register unsigned long long int* _zzq_ptr; \ - _zzq_args[0] = (unsigned int long long)(_zzq_request); \ - _zzq_args[1] = (unsigned int long long)(_zzq_arg1); \ - _zzq_args[2] = (unsigned int long long)(_zzq_arg2); \ - _zzq_args[3] = (unsigned int long long)(_zzq_arg3); \ - _zzq_args[4] = (unsigned int long long)(_zzq_arg4); \ - _zzq_args[5] = (unsigned int long long)(_zzq_arg5); \ - _zzq_args[6] = (unsigned int long long)(_zzq_default); \ - _zzq_ptr = _zzq_args; \ - __asm__ volatile("mr 4,%1\n\t" \ - "ld 3, 48(4)\n\t" \ - __SPECIAL_INSTRUCTION_PREAMBLE \ - /* %R3 = client_request ( %R4 ) */ \ - "or 1,1,1\n\t" \ - "mr %0,3" \ - : "=b" (_zzq_result) \ - : "b" (_zzq_ptr) \ - : "r3", "r4", "cc", "memory"); \ - _zzq_rlval = _zzq_result; \ - } - -#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ - { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ - register unsigned long long int __addr; \ - __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ - /* %R3 = guest_NRADDR */ \ - "or 2,2,2\n\t" \ - "mr %0,3" \ - : "=b" (__addr) \ - : \ - : "r3", "cc", "memory" \ - ); \ - _zzq_orig->nraddr = __addr; \ - __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ - /* %R3 = guest_NRADDR_GPR2 */ \ - "or 4,4,4\n\t" \ - "mr %0,3" \ - : "=b" (__addr) \ - : \ - : "r3", "cc", "memory" \ - ); \ - _zzq_orig->r2 = __addr; \ - } - -#define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - __SPECIAL_INSTRUCTION_PREAMBLE \ - /* branch-and-link-to-noredir *%R11 */ \ - "or 3,3,3\n\t" - -#endif /* PLAT_ppc64_aix5 */ - -/* Insert assembly code for other platforms here... */ - -#endif /* NVALGRIND */ - - -/* ------------------------------------------------------------------ */ -/* PLATFORM SPECIFICS for FUNCTION WRAPPING. This is all very */ -/* ugly. It's the least-worst tradeoff I can think of. */ -/* ------------------------------------------------------------------ */ - -/* This section defines magic (a.k.a appalling-hack) macros for doing - guaranteed-no-redirection macros, so as to get from function - wrappers to the functions they are wrapping. The whole point is to - construct standard call sequences, but to do the call itself with a - special no-redirect call pseudo-instruction that the JIT - understands and handles specially. This section is long and - repetitious, and I can't see a way to make it shorter. - - The naming scheme is as follows: - - CALL_FN_{W,v}_{v,W,WW,WWW,WWWW,5W,6W,7W,etc} - - 'W' stands for "word" and 'v' for "void". Hence there are - different macros for calling arity 0, 1, 2, 3, 4, etc, functions, - and for each, the possibility of returning a word-typed result, or - no result. -*/ - -/* Use these to write the name of your wrapper. NOTE: duplicates - VG_WRAP_FUNCTION_Z{U,Z} in pub_tool_redir.h. */ - -#define I_WRAP_SONAME_FNNAME_ZU(soname,fnname) \ - _vgwZU_##soname##_##fnname - -#define I_WRAP_SONAME_FNNAME_ZZ(soname,fnname) \ - _vgwZZ_##soname##_##fnname - -/* Use this macro from within a wrapper function to collect the - context (address and possibly other info) of the original function. - Once you have that you can then use it in one of the CALL_FN_ - macros. The type of the argument _lval is OrigFn. */ -#define VALGRIND_GET_ORIG_FN(_lval) VALGRIND_GET_NR_CONTEXT(_lval) - -/* Derivatives of the main macros below, for calling functions - returning void. */ - -#define CALL_FN_v_v(fnptr) \ - do { volatile unsigned long _junk; \ - CALL_FN_W_v(_junk,fnptr); } while (0) - -#define CALL_FN_v_W(fnptr, arg1) \ - do { volatile unsigned long _junk; \ - CALL_FN_W_W(_junk,fnptr,arg1); } while (0) - -#define CALL_FN_v_WW(fnptr, arg1,arg2) \ - do { volatile unsigned long _junk; \ - CALL_FN_W_WW(_junk,fnptr,arg1,arg2); } while (0) - -#define CALL_FN_v_WWW(fnptr, arg1,arg2,arg3) \ - do { volatile unsigned long _junk; \ - CALL_FN_W_WWW(_junk,fnptr,arg1,arg2,arg3); } while (0) - -/* ------------------------- x86-linux ------------------------- */ - -#if defined(PLAT_x86_linux) - -/* These regs are trashed by the hidden call. No need to mention eax - as gcc can already see that, plus causes gcc to bomb. */ -#define __CALLER_SAVED_REGS /*"eax"*/ "ecx", "edx" - -/* These CALL_FN_ macros assume that on x86-linux, sizeof(unsigned - long) == 4. */ - -#define CALL_FN_W_v(lval, orig) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[1]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - __asm__ volatile( \ - "movl (%%eax), %%eax\n\t" /* target->%eax */ \ - VALGRIND_CALL_NOREDIR_EAX \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_W(lval, orig, arg1) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[2]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - __asm__ volatile( \ - "pushl 4(%%eax)\n\t" \ - "movl (%%eax), %%eax\n\t" /* target->%eax */ \ - VALGRIND_CALL_NOREDIR_EAX \ - "addl $4, %%esp\n" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_WW(lval, orig, arg1,arg2) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - __asm__ volatile( \ - "pushl 8(%%eax)\n\t" \ - "pushl 4(%%eax)\n\t" \ - "movl (%%eax), %%eax\n\t" /* target->%eax */ \ - VALGRIND_CALL_NOREDIR_EAX \ - "addl $8, %%esp\n" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[4]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - __asm__ volatile( \ - "pushl 12(%%eax)\n\t" \ - "pushl 8(%%eax)\n\t" \ - "pushl 4(%%eax)\n\t" \ - "movl (%%eax), %%eax\n\t" /* target->%eax */ \ - VALGRIND_CALL_NOREDIR_EAX \ - "addl $12, %%esp\n" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[5]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - _argvec[4] = (unsigned long)(arg4); \ - __asm__ volatile( \ - "pushl 16(%%eax)\n\t" \ - "pushl 12(%%eax)\n\t" \ - "pushl 8(%%eax)\n\t" \ - "pushl 4(%%eax)\n\t" \ - "movl (%%eax), %%eax\n\t" /* target->%eax */ \ - VALGRIND_CALL_NOREDIR_EAX \ - "addl $16, %%esp\n" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[6]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - _argvec[4] = (unsigned long)(arg4); \ - _argvec[5] = (unsigned long)(arg5); \ - __asm__ volatile( \ - "pushl 20(%%eax)\n\t" \ - "pushl 16(%%eax)\n\t" \ - "pushl 12(%%eax)\n\t" \ - "pushl 8(%%eax)\n\t" \ - "pushl 4(%%eax)\n\t" \ - "movl (%%eax), %%eax\n\t" /* target->%eax */ \ - VALGRIND_CALL_NOREDIR_EAX \ - "addl $20, %%esp\n" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[7]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - _argvec[4] = (unsigned long)(arg4); \ - _argvec[5] = (unsigned long)(arg5); \ - _argvec[6] = (unsigned long)(arg6); \ - __asm__ volatile( \ - "pushl 24(%%eax)\n\t" \ - "pushl 20(%%eax)\n\t" \ - "pushl 16(%%eax)\n\t" \ - "pushl 12(%%eax)\n\t" \ - "pushl 8(%%eax)\n\t" \ - "pushl 4(%%eax)\n\t" \ - "movl (%%eax), %%eax\n\t" /* target->%eax */ \ - VALGRIND_CALL_NOREDIR_EAX \ - "addl $24, %%esp\n" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[8]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - _argvec[4] = (unsigned long)(arg4); \ - _argvec[5] = (unsigned long)(arg5); \ - _argvec[6] = (unsigned long)(arg6); \ - _argvec[7] = (unsigned long)(arg7); \ - __asm__ volatile( \ - "pushl 28(%%eax)\n\t" \ - "pushl 24(%%eax)\n\t" \ - "pushl 20(%%eax)\n\t" \ - "pushl 16(%%eax)\n\t" \ - "pushl 12(%%eax)\n\t" \ - "pushl 8(%%eax)\n\t" \ - "pushl 4(%%eax)\n\t" \ - "movl (%%eax), %%eax\n\t" /* target->%eax */ \ - VALGRIND_CALL_NOREDIR_EAX \ - "addl $28, %%esp\n" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[9]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - _argvec[4] = (unsigned long)(arg4); \ - _argvec[5] = (unsigned long)(arg5); \ - _argvec[6] = (unsigned long)(arg6); \ - _argvec[7] = (unsigned long)(arg7); \ - _argvec[8] = (unsigned long)(arg8); \ - __asm__ volatile( \ - "pushl 32(%%eax)\n\t" \ - "pushl 28(%%eax)\n\t" \ - "pushl 24(%%eax)\n\t" \ - "pushl 20(%%eax)\n\t" \ - "pushl 16(%%eax)\n\t" \ - "pushl 12(%%eax)\n\t" \ - "pushl 8(%%eax)\n\t" \ - "pushl 4(%%eax)\n\t" \ - "movl (%%eax), %%eax\n\t" /* target->%eax */ \ - VALGRIND_CALL_NOREDIR_EAX \ - "addl $32, %%esp\n" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[10]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - _argvec[4] = (unsigned long)(arg4); \ - _argvec[5] = (unsigned long)(arg5); \ - _argvec[6] = (unsigned long)(arg6); \ - _argvec[7] = (unsigned long)(arg7); \ - _argvec[8] = (unsigned long)(arg8); \ - _argvec[9] = (unsigned long)(arg9); \ - __asm__ volatile( \ - "pushl 36(%%eax)\n\t" \ - "pushl 32(%%eax)\n\t" \ - "pushl 28(%%eax)\n\t" \ - "pushl 24(%%eax)\n\t" \ - "pushl 20(%%eax)\n\t" \ - "pushl 16(%%eax)\n\t" \ - "pushl 12(%%eax)\n\t" \ - "pushl 8(%%eax)\n\t" \ - "pushl 4(%%eax)\n\t" \ - "movl (%%eax), %%eax\n\t" /* target->%eax */ \ - VALGRIND_CALL_NOREDIR_EAX \ - "addl $36, %%esp\n" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9,arg10) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[11]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - _argvec[4] = (unsigned long)(arg4); \ - _argvec[5] = (unsigned long)(arg5); \ - _argvec[6] = (unsigned long)(arg6); \ - _argvec[7] = (unsigned long)(arg7); \ - _argvec[8] = (unsigned long)(arg8); \ - _argvec[9] = (unsigned long)(arg9); \ - _argvec[10] = (unsigned long)(arg10); \ - __asm__ volatile( \ - "pushl 40(%%eax)\n\t" \ - "pushl 36(%%eax)\n\t" \ - "pushl 32(%%eax)\n\t" \ - "pushl 28(%%eax)\n\t" \ - "pushl 24(%%eax)\n\t" \ - "pushl 20(%%eax)\n\t" \ - "pushl 16(%%eax)\n\t" \ - "pushl 12(%%eax)\n\t" \ - "pushl 8(%%eax)\n\t" \ - "pushl 4(%%eax)\n\t" \ - "movl (%%eax), %%eax\n\t" /* target->%eax */ \ - VALGRIND_CALL_NOREDIR_EAX \ - "addl $40, %%esp\n" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ - arg6,arg7,arg8,arg9,arg10, \ - arg11) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[12]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - _argvec[4] = (unsigned long)(arg4); \ - _argvec[5] = (unsigned long)(arg5); \ - _argvec[6] = (unsigned long)(arg6); \ - _argvec[7] = (unsigned long)(arg7); \ - _argvec[8] = (unsigned long)(arg8); \ - _argvec[9] = (unsigned long)(arg9); \ - _argvec[10] = (unsigned long)(arg10); \ - _argvec[11] = (unsigned long)(arg11); \ - __asm__ volatile( \ - "pushl 44(%%eax)\n\t" \ - "pushl 40(%%eax)\n\t" \ - "pushl 36(%%eax)\n\t" \ - "pushl 32(%%eax)\n\t" \ - "pushl 28(%%eax)\n\t" \ - "pushl 24(%%eax)\n\t" \ - "pushl 20(%%eax)\n\t" \ - "pushl 16(%%eax)\n\t" \ - "pushl 12(%%eax)\n\t" \ - "pushl 8(%%eax)\n\t" \ - "pushl 4(%%eax)\n\t" \ - "movl (%%eax), %%eax\n\t" /* target->%eax */ \ - VALGRIND_CALL_NOREDIR_EAX \ - "addl $44, %%esp\n" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ - arg6,arg7,arg8,arg9,arg10, \ - arg11,arg12) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[13]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - _argvec[4] = (unsigned long)(arg4); \ - _argvec[5] = (unsigned long)(arg5); \ - _argvec[6] = (unsigned long)(arg6); \ - _argvec[7] = (unsigned long)(arg7); \ - _argvec[8] = (unsigned long)(arg8); \ - _argvec[9] = (unsigned long)(arg9); \ - _argvec[10] = (unsigned long)(arg10); \ - _argvec[11] = (unsigned long)(arg11); \ - _argvec[12] = (unsigned long)(arg12); \ - __asm__ volatile( \ - "pushl 48(%%eax)\n\t" \ - "pushl 44(%%eax)\n\t" \ - "pushl 40(%%eax)\n\t" \ - "pushl 36(%%eax)\n\t" \ - "pushl 32(%%eax)\n\t" \ - "pushl 28(%%eax)\n\t" \ - "pushl 24(%%eax)\n\t" \ - "pushl 20(%%eax)\n\t" \ - "pushl 16(%%eax)\n\t" \ - "pushl 12(%%eax)\n\t" \ - "pushl 8(%%eax)\n\t" \ - "pushl 4(%%eax)\n\t" \ - "movl (%%eax), %%eax\n\t" /* target->%eax */ \ - VALGRIND_CALL_NOREDIR_EAX \ - "addl $48, %%esp\n" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#endif /* PLAT_x86_linux */ - -/* ------------------------ amd64-linux ------------------------ */ - -#if defined(PLAT_amd64_linux) - -/* ARGREGS: rdi rsi rdx rcx r8 r9 (the rest on stack in R-to-L order) */ - -/* These regs are trashed by the hidden call. */ -#define __CALLER_SAVED_REGS /*"rax",*/ "rcx", "rdx", "rsi", \ - "rdi", "r8", "r9", "r10", "r11" - -/* These CALL_FN_ macros assume that on amd64-linux, sizeof(unsigned - long) == 8. */ - -/* NB 9 Sept 07. There is a nasty kludge here in all these CALL_FN_ - macros. In order not to trash the stack redzone, we need to drop - %rsp by 128 before the hidden call, and restore afterwards. The - nastyness is that it is only by luck that the stack still appears - to be unwindable during the hidden call - since then the behaviour - of any routine using this macro does not match what the CFI data - says. Sigh. - - Why is this important? Imagine that a wrapper has a stack - allocated local, and passes to the hidden call, a pointer to it. - Because gcc does not know about the hidden call, it may allocate - that local in the redzone. Unfortunately the hidden call may then - trash it before it comes to use it. So we must step clear of the - redzone, for the duration of the hidden call, to make it safe. - - Probably the same problem afflicts the other redzone-style ABIs too - (ppc64-linux, ppc32-aix5, ppc64-aix5); but for those, the stack is - self describing (none of this CFI nonsense) so at least messing - with the stack pointer doesn't give a danger of non-unwindable - stack. */ - -#define CALL_FN_W_v(lval, orig) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[1]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - __asm__ volatile( \ - "subq $128,%%rsp\n\t" \ - "movq (%%rax), %%rax\n\t" /* target->%rax */ \ - VALGRIND_CALL_NOREDIR_RAX \ - "addq $128,%%rsp\n\t" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_W(lval, orig, arg1) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[2]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - __asm__ volatile( \ - "subq $128,%%rsp\n\t" \ - "movq 8(%%rax), %%rdi\n\t" \ - "movq (%%rax), %%rax\n\t" /* target->%rax */ \ - VALGRIND_CALL_NOREDIR_RAX \ - "addq $128,%%rsp\n\t" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_WW(lval, orig, arg1,arg2) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - __asm__ volatile( \ - "subq $128,%%rsp\n\t" \ - "movq 16(%%rax), %%rsi\n\t" \ - "movq 8(%%rax), %%rdi\n\t" \ - "movq (%%rax), %%rax\n\t" /* target->%rax */ \ - VALGRIND_CALL_NOREDIR_RAX \ - "addq $128,%%rsp\n\t" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[4]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - __asm__ volatile( \ - "subq $128,%%rsp\n\t" \ - "movq 24(%%rax), %%rdx\n\t" \ - "movq 16(%%rax), %%rsi\n\t" \ - "movq 8(%%rax), %%rdi\n\t" \ - "movq (%%rax), %%rax\n\t" /* target->%rax */ \ - VALGRIND_CALL_NOREDIR_RAX \ - "addq $128,%%rsp\n\t" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[5]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - _argvec[4] = (unsigned long)(arg4); \ - __asm__ volatile( \ - "subq $128,%%rsp\n\t" \ - "movq 32(%%rax), %%rcx\n\t" \ - "movq 24(%%rax), %%rdx\n\t" \ - "movq 16(%%rax), %%rsi\n\t" \ - "movq 8(%%rax), %%rdi\n\t" \ - "movq (%%rax), %%rax\n\t" /* target->%rax */ \ - VALGRIND_CALL_NOREDIR_RAX \ - "addq $128,%%rsp\n\t" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[6]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - _argvec[4] = (unsigned long)(arg4); \ - _argvec[5] = (unsigned long)(arg5); \ - __asm__ volatile( \ - "subq $128,%%rsp\n\t" \ - "movq 40(%%rax), %%r8\n\t" \ - "movq 32(%%rax), %%rcx\n\t" \ - "movq 24(%%rax), %%rdx\n\t" \ - "movq 16(%%rax), %%rsi\n\t" \ - "movq 8(%%rax), %%rdi\n\t" \ - "movq (%%rax), %%rax\n\t" /* target->%rax */ \ - VALGRIND_CALL_NOREDIR_RAX \ - "addq $128,%%rsp\n\t" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[7]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - _argvec[4] = (unsigned long)(arg4); \ - _argvec[5] = (unsigned long)(arg5); \ - _argvec[6] = (unsigned long)(arg6); \ - __asm__ volatile( \ - "subq $128,%%rsp\n\t" \ - "movq 48(%%rax), %%r9\n\t" \ - "movq 40(%%rax), %%r8\n\t" \ - "movq 32(%%rax), %%rcx\n\t" \ - "movq 24(%%rax), %%rdx\n\t" \ - "movq 16(%%rax), %%rsi\n\t" \ - "movq 8(%%rax), %%rdi\n\t" \ - "movq (%%rax), %%rax\n\t" /* target->%rax */ \ - "addq $128,%%rsp\n\t" \ - VALGRIND_CALL_NOREDIR_RAX \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[8]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - _argvec[4] = (unsigned long)(arg4); \ - _argvec[5] = (unsigned long)(arg5); \ - _argvec[6] = (unsigned long)(arg6); \ - _argvec[7] = (unsigned long)(arg7); \ - __asm__ volatile( \ - "subq $128,%%rsp\n\t" \ - "pushq 56(%%rax)\n\t" \ - "movq 48(%%rax), %%r9\n\t" \ - "movq 40(%%rax), %%r8\n\t" \ - "movq 32(%%rax), %%rcx\n\t" \ - "movq 24(%%rax), %%rdx\n\t" \ - "movq 16(%%rax), %%rsi\n\t" \ - "movq 8(%%rax), %%rdi\n\t" \ - "movq (%%rax), %%rax\n\t" /* target->%rax */ \ - VALGRIND_CALL_NOREDIR_RAX \ - "addq $8, %%rsp\n" \ - "addq $128,%%rsp\n\t" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[9]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - _argvec[4] = (unsigned long)(arg4); \ - _argvec[5] = (unsigned long)(arg5); \ - _argvec[6] = (unsigned long)(arg6); \ - _argvec[7] = (unsigned long)(arg7); \ - _argvec[8] = (unsigned long)(arg8); \ - __asm__ volatile( \ - "subq $128,%%rsp\n\t" \ - "pushq 64(%%rax)\n\t" \ - "pushq 56(%%rax)\n\t" \ - "movq 48(%%rax), %%r9\n\t" \ - "movq 40(%%rax), %%r8\n\t" \ - "movq 32(%%rax), %%rcx\n\t" \ - "movq 24(%%rax), %%rdx\n\t" \ - "movq 16(%%rax), %%rsi\n\t" \ - "movq 8(%%rax), %%rdi\n\t" \ - "movq (%%rax), %%rax\n\t" /* target->%rax */ \ - VALGRIND_CALL_NOREDIR_RAX \ - "addq $16, %%rsp\n" \ - "addq $128,%%rsp\n\t" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[10]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - _argvec[4] = (unsigned long)(arg4); \ - _argvec[5] = (unsigned long)(arg5); \ - _argvec[6] = (unsigned long)(arg6); \ - _argvec[7] = (unsigned long)(arg7); \ - _argvec[8] = (unsigned long)(arg8); \ - _argvec[9] = (unsigned long)(arg9); \ - __asm__ volatile( \ - "subq $128,%%rsp\n\t" \ - "pushq 72(%%rax)\n\t" \ - "pushq 64(%%rax)\n\t" \ - "pushq 56(%%rax)\n\t" \ - "movq 48(%%rax), %%r9\n\t" \ - "movq 40(%%rax), %%r8\n\t" \ - "movq 32(%%rax), %%rcx\n\t" \ - "movq 24(%%rax), %%rdx\n\t" \ - "movq 16(%%rax), %%rsi\n\t" \ - "movq 8(%%rax), %%rdi\n\t" \ - "movq (%%rax), %%rax\n\t" /* target->%rax */ \ - VALGRIND_CALL_NOREDIR_RAX \ - "addq $24, %%rsp\n" \ - "addq $128,%%rsp\n\t" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9,arg10) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[11]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - _argvec[4] = (unsigned long)(arg4); \ - _argvec[5] = (unsigned long)(arg5); \ - _argvec[6] = (unsigned long)(arg6); \ - _argvec[7] = (unsigned long)(arg7); \ - _argvec[8] = (unsigned long)(arg8); \ - _argvec[9] = (unsigned long)(arg9); \ - _argvec[10] = (unsigned long)(arg10); \ - __asm__ volatile( \ - "subq $128,%%rsp\n\t" \ - "pushq 80(%%rax)\n\t" \ - "pushq 72(%%rax)\n\t" \ - "pushq 64(%%rax)\n\t" \ - "pushq 56(%%rax)\n\t" \ - "movq 48(%%rax), %%r9\n\t" \ - "movq 40(%%rax), %%r8\n\t" \ - "movq 32(%%rax), %%rcx\n\t" \ - "movq 24(%%rax), %%rdx\n\t" \ - "movq 16(%%rax), %%rsi\n\t" \ - "movq 8(%%rax), %%rdi\n\t" \ - "movq (%%rax), %%rax\n\t" /* target->%rax */ \ - VALGRIND_CALL_NOREDIR_RAX \ - "addq $32, %%rsp\n" \ - "addq $128,%%rsp\n\t" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9,arg10,arg11) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[12]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - _argvec[4] = (unsigned long)(arg4); \ - _argvec[5] = (unsigned long)(arg5); \ - _argvec[6] = (unsigned long)(arg6); \ - _argvec[7] = (unsigned long)(arg7); \ - _argvec[8] = (unsigned long)(arg8); \ - _argvec[9] = (unsigned long)(arg9); \ - _argvec[10] = (unsigned long)(arg10); \ - _argvec[11] = (unsigned long)(arg11); \ - __asm__ volatile( \ - "subq $128,%%rsp\n\t" \ - "pushq 88(%%rax)\n\t" \ - "pushq 80(%%rax)\n\t" \ - "pushq 72(%%rax)\n\t" \ - "pushq 64(%%rax)\n\t" \ - "pushq 56(%%rax)\n\t" \ - "movq 48(%%rax), %%r9\n\t" \ - "movq 40(%%rax), %%r8\n\t" \ - "movq 32(%%rax), %%rcx\n\t" \ - "movq 24(%%rax), %%rdx\n\t" \ - "movq 16(%%rax), %%rsi\n\t" \ - "movq 8(%%rax), %%rdi\n\t" \ - "movq (%%rax), %%rax\n\t" /* target->%rax */ \ - VALGRIND_CALL_NOREDIR_RAX \ - "addq $40, %%rsp\n" \ - "addq $128,%%rsp\n\t" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9,arg10,arg11,arg12) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[13]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)(arg1); \ - _argvec[2] = (unsigned long)(arg2); \ - _argvec[3] = (unsigned long)(arg3); \ - _argvec[4] = (unsigned long)(arg4); \ - _argvec[5] = (unsigned long)(arg5); \ - _argvec[6] = (unsigned long)(arg6); \ - _argvec[7] = (unsigned long)(arg7); \ - _argvec[8] = (unsigned long)(arg8); \ - _argvec[9] = (unsigned long)(arg9); \ - _argvec[10] = (unsigned long)(arg10); \ - _argvec[11] = (unsigned long)(arg11); \ - _argvec[12] = (unsigned long)(arg12); \ - __asm__ volatile( \ - "subq $128,%%rsp\n\t" \ - "pushq 96(%%rax)\n\t" \ - "pushq 88(%%rax)\n\t" \ - "pushq 80(%%rax)\n\t" \ - "pushq 72(%%rax)\n\t" \ - "pushq 64(%%rax)\n\t" \ - "pushq 56(%%rax)\n\t" \ - "movq 48(%%rax), %%r9\n\t" \ - "movq 40(%%rax), %%r8\n\t" \ - "movq 32(%%rax), %%rcx\n\t" \ - "movq 24(%%rax), %%rdx\n\t" \ - "movq 16(%%rax), %%rsi\n\t" \ - "movq 8(%%rax), %%rdi\n\t" \ - "movq (%%rax), %%rax\n\t" /* target->%rax */ \ - VALGRIND_CALL_NOREDIR_RAX \ - "addq $48, %%rsp\n" \ - "addq $128,%%rsp\n\t" \ - : /*out*/ "=a" (_res) \ - : /*in*/ "a" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#endif /* PLAT_amd64_linux */ - -/* ------------------------ ppc32-linux ------------------------ */ - -#if defined(PLAT_ppc32_linux) - -/* This is useful for finding out about the on-stack stuff: - - extern int f9 ( int,int,int,int,int,int,int,int,int ); - extern int f10 ( int,int,int,int,int,int,int,int,int,int ); - extern int f11 ( int,int,int,int,int,int,int,int,int,int,int ); - extern int f12 ( int,int,int,int,int,int,int,int,int,int,int,int ); - - int g9 ( void ) { - return f9(11,22,33,44,55,66,77,88,99); - } - int g10 ( void ) { - return f10(11,22,33,44,55,66,77,88,99,110); - } - int g11 ( void ) { - return f11(11,22,33,44,55,66,77,88,99,110,121); - } - int g12 ( void ) { - return f12(11,22,33,44,55,66,77,88,99,110,121,132); - } -*/ - -/* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */ - -/* These regs are trashed by the hidden call. */ -#define __CALLER_SAVED_REGS \ - "lr", "ctr", "xer", \ - "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7", \ - "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", \ - "r11", "r12", "r13" - -/* These CALL_FN_ macros assume that on ppc32-linux, - sizeof(unsigned long) == 4. */ - -#define CALL_FN_W_v(lval, orig) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[1]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "lwz 11,0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr %0,3" \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_W(lval, orig, arg1) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[2]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)arg1; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "lwz 3,4(11)\n\t" /* arg1->r3 */ \ - "lwz 11,0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr %0,3" \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_WW(lval, orig, arg1,arg2) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)arg1; \ - _argvec[2] = (unsigned long)arg2; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "lwz 3,4(11)\n\t" /* arg1->r3 */ \ - "lwz 4,8(11)\n\t" \ - "lwz 11,0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr %0,3" \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[4]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)arg1; \ - _argvec[2] = (unsigned long)arg2; \ - _argvec[3] = (unsigned long)arg3; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "lwz 3,4(11)\n\t" /* arg1->r3 */ \ - "lwz 4,8(11)\n\t" \ - "lwz 5,12(11)\n\t" \ - "lwz 11,0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr %0,3" \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[5]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)arg1; \ - _argvec[2] = (unsigned long)arg2; \ - _argvec[3] = (unsigned long)arg3; \ - _argvec[4] = (unsigned long)arg4; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "lwz 3,4(11)\n\t" /* arg1->r3 */ \ - "lwz 4,8(11)\n\t" \ - "lwz 5,12(11)\n\t" \ - "lwz 6,16(11)\n\t" /* arg4->r6 */ \ - "lwz 11,0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr %0,3" \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[6]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)arg1; \ - _argvec[2] = (unsigned long)arg2; \ - _argvec[3] = (unsigned long)arg3; \ - _argvec[4] = (unsigned long)arg4; \ - _argvec[5] = (unsigned long)arg5; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "lwz 3,4(11)\n\t" /* arg1->r3 */ \ - "lwz 4,8(11)\n\t" \ - "lwz 5,12(11)\n\t" \ - "lwz 6,16(11)\n\t" /* arg4->r6 */ \ - "lwz 7,20(11)\n\t" \ - "lwz 11,0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr %0,3" \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[7]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)arg1; \ - _argvec[2] = (unsigned long)arg2; \ - _argvec[3] = (unsigned long)arg3; \ - _argvec[4] = (unsigned long)arg4; \ - _argvec[5] = (unsigned long)arg5; \ - _argvec[6] = (unsigned long)arg6; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "lwz 3,4(11)\n\t" /* arg1->r3 */ \ - "lwz 4,8(11)\n\t" \ - "lwz 5,12(11)\n\t" \ - "lwz 6,16(11)\n\t" /* arg4->r6 */ \ - "lwz 7,20(11)\n\t" \ - "lwz 8,24(11)\n\t" \ - "lwz 11,0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr %0,3" \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[8]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)arg1; \ - _argvec[2] = (unsigned long)arg2; \ - _argvec[3] = (unsigned long)arg3; \ - _argvec[4] = (unsigned long)arg4; \ - _argvec[5] = (unsigned long)arg5; \ - _argvec[6] = (unsigned long)arg6; \ - _argvec[7] = (unsigned long)arg7; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "lwz 3,4(11)\n\t" /* arg1->r3 */ \ - "lwz 4,8(11)\n\t" \ - "lwz 5,12(11)\n\t" \ - "lwz 6,16(11)\n\t" /* arg4->r6 */ \ - "lwz 7,20(11)\n\t" \ - "lwz 8,24(11)\n\t" \ - "lwz 9,28(11)\n\t" \ - "lwz 11,0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr %0,3" \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[9]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)arg1; \ - _argvec[2] = (unsigned long)arg2; \ - _argvec[3] = (unsigned long)arg3; \ - _argvec[4] = (unsigned long)arg4; \ - _argvec[5] = (unsigned long)arg5; \ - _argvec[6] = (unsigned long)arg6; \ - _argvec[7] = (unsigned long)arg7; \ - _argvec[8] = (unsigned long)arg8; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "lwz 3,4(11)\n\t" /* arg1->r3 */ \ - "lwz 4,8(11)\n\t" \ - "lwz 5,12(11)\n\t" \ - "lwz 6,16(11)\n\t" /* arg4->r6 */ \ - "lwz 7,20(11)\n\t" \ - "lwz 8,24(11)\n\t" \ - "lwz 9,28(11)\n\t" \ - "lwz 10,32(11)\n\t" /* arg8->r10 */ \ - "lwz 11,0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr %0,3" \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[10]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)arg1; \ - _argvec[2] = (unsigned long)arg2; \ - _argvec[3] = (unsigned long)arg3; \ - _argvec[4] = (unsigned long)arg4; \ - _argvec[5] = (unsigned long)arg5; \ - _argvec[6] = (unsigned long)arg6; \ - _argvec[7] = (unsigned long)arg7; \ - _argvec[8] = (unsigned long)arg8; \ - _argvec[9] = (unsigned long)arg9; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "addi 1,1,-16\n\t" \ - /* arg9 */ \ - "lwz 3,36(11)\n\t" \ - "stw 3,8(1)\n\t" \ - /* args1-8 */ \ - "lwz 3,4(11)\n\t" /* arg1->r3 */ \ - "lwz 4,8(11)\n\t" \ - "lwz 5,12(11)\n\t" \ - "lwz 6,16(11)\n\t" /* arg4->r6 */ \ - "lwz 7,20(11)\n\t" \ - "lwz 8,24(11)\n\t" \ - "lwz 9,28(11)\n\t" \ - "lwz 10,32(11)\n\t" /* arg8->r10 */ \ - "lwz 11,0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "addi 1,1,16\n\t" \ - "mr %0,3" \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9,arg10) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[11]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)arg1; \ - _argvec[2] = (unsigned long)arg2; \ - _argvec[3] = (unsigned long)arg3; \ - _argvec[4] = (unsigned long)arg4; \ - _argvec[5] = (unsigned long)arg5; \ - _argvec[6] = (unsigned long)arg6; \ - _argvec[7] = (unsigned long)arg7; \ - _argvec[8] = (unsigned long)arg8; \ - _argvec[9] = (unsigned long)arg9; \ - _argvec[10] = (unsigned long)arg10; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "addi 1,1,-16\n\t" \ - /* arg10 */ \ - "lwz 3,40(11)\n\t" \ - "stw 3,12(1)\n\t" \ - /* arg9 */ \ - "lwz 3,36(11)\n\t" \ - "stw 3,8(1)\n\t" \ - /* args1-8 */ \ - "lwz 3,4(11)\n\t" /* arg1->r3 */ \ - "lwz 4,8(11)\n\t" \ - "lwz 5,12(11)\n\t" \ - "lwz 6,16(11)\n\t" /* arg4->r6 */ \ - "lwz 7,20(11)\n\t" \ - "lwz 8,24(11)\n\t" \ - "lwz 9,28(11)\n\t" \ - "lwz 10,32(11)\n\t" /* arg8->r10 */ \ - "lwz 11,0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "addi 1,1,16\n\t" \ - "mr %0,3" \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9,arg10,arg11) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[12]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)arg1; \ - _argvec[2] = (unsigned long)arg2; \ - _argvec[3] = (unsigned long)arg3; \ - _argvec[4] = (unsigned long)arg4; \ - _argvec[5] = (unsigned long)arg5; \ - _argvec[6] = (unsigned long)arg6; \ - _argvec[7] = (unsigned long)arg7; \ - _argvec[8] = (unsigned long)arg8; \ - _argvec[9] = (unsigned long)arg9; \ - _argvec[10] = (unsigned long)arg10; \ - _argvec[11] = (unsigned long)arg11; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "addi 1,1,-32\n\t" \ - /* arg11 */ \ - "lwz 3,44(11)\n\t" \ - "stw 3,16(1)\n\t" \ - /* arg10 */ \ - "lwz 3,40(11)\n\t" \ - "stw 3,12(1)\n\t" \ - /* arg9 */ \ - "lwz 3,36(11)\n\t" \ - "stw 3,8(1)\n\t" \ - /* args1-8 */ \ - "lwz 3,4(11)\n\t" /* arg1->r3 */ \ - "lwz 4,8(11)\n\t" \ - "lwz 5,12(11)\n\t" \ - "lwz 6,16(11)\n\t" /* arg4->r6 */ \ - "lwz 7,20(11)\n\t" \ - "lwz 8,24(11)\n\t" \ - "lwz 9,28(11)\n\t" \ - "lwz 10,32(11)\n\t" /* arg8->r10 */ \ - "lwz 11,0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "addi 1,1,32\n\t" \ - "mr %0,3" \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9,arg10,arg11,arg12) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[13]; \ - volatile unsigned long _res; \ - _argvec[0] = (unsigned long)_orig.nraddr; \ - _argvec[1] = (unsigned long)arg1; \ - _argvec[2] = (unsigned long)arg2; \ - _argvec[3] = (unsigned long)arg3; \ - _argvec[4] = (unsigned long)arg4; \ - _argvec[5] = (unsigned long)arg5; \ - _argvec[6] = (unsigned long)arg6; \ - _argvec[7] = (unsigned long)arg7; \ - _argvec[8] = (unsigned long)arg8; \ - _argvec[9] = (unsigned long)arg9; \ - _argvec[10] = (unsigned long)arg10; \ - _argvec[11] = (unsigned long)arg11; \ - _argvec[12] = (unsigned long)arg12; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "addi 1,1,-32\n\t" \ - /* arg12 */ \ - "lwz 3,48(11)\n\t" \ - "stw 3,20(1)\n\t" \ - /* arg11 */ \ - "lwz 3,44(11)\n\t" \ - "stw 3,16(1)\n\t" \ - /* arg10 */ \ - "lwz 3,40(11)\n\t" \ - "stw 3,12(1)\n\t" \ - /* arg9 */ \ - "lwz 3,36(11)\n\t" \ - "stw 3,8(1)\n\t" \ - /* args1-8 */ \ - "lwz 3,4(11)\n\t" /* arg1->r3 */ \ - "lwz 4,8(11)\n\t" \ - "lwz 5,12(11)\n\t" \ - "lwz 6,16(11)\n\t" /* arg4->r6 */ \ - "lwz 7,20(11)\n\t" \ - "lwz 8,24(11)\n\t" \ - "lwz 9,28(11)\n\t" \ - "lwz 10,32(11)\n\t" /* arg8->r10 */ \ - "lwz 11,0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "addi 1,1,32\n\t" \ - "mr %0,3" \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[0]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#endif /* PLAT_ppc32_linux */ - -/* ------------------------ ppc64-linux ------------------------ */ - -#if defined(PLAT_ppc64_linux) - -/* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */ - -/* These regs are trashed by the hidden call. */ -#define __CALLER_SAVED_REGS \ - "lr", "ctr", "xer", \ - "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7", \ - "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", \ - "r11", "r12", "r13" - -/* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned - long) == 8. */ - -#define CALL_FN_W_v(lval, orig) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+0]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)" /* restore tocptr */ \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_W(lval, orig, arg1) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+1]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)" /* restore tocptr */ \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_WW(lval, orig, arg1,arg2) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+2]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)" /* restore tocptr */ \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+3]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)" /* restore tocptr */ \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+4]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 6, 32(11)\n\t" /* arg4->r6 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)" /* restore tocptr */ \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+5]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 6, 32(11)\n\t" /* arg4->r6 */ \ - "ld 7, 40(11)\n\t" /* arg5->r7 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)" /* restore tocptr */ \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+6]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 6, 32(11)\n\t" /* arg4->r6 */ \ - "ld 7, 40(11)\n\t" /* arg5->r7 */ \ - "ld 8, 48(11)\n\t" /* arg6->r8 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)" /* restore tocptr */ \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+7]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - _argvec[2+7] = (unsigned long)arg7; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 6, 32(11)\n\t" /* arg4->r6 */ \ - "ld 7, 40(11)\n\t" /* arg5->r7 */ \ - "ld 8, 48(11)\n\t" /* arg6->r8 */ \ - "ld 9, 56(11)\n\t" /* arg7->r9 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)" /* restore tocptr */ \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+8]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - _argvec[2+7] = (unsigned long)arg7; \ - _argvec[2+8] = (unsigned long)arg8; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 6, 32(11)\n\t" /* arg4->r6 */ \ - "ld 7, 40(11)\n\t" /* arg5->r7 */ \ - "ld 8, 48(11)\n\t" /* arg6->r8 */ \ - "ld 9, 56(11)\n\t" /* arg7->r9 */ \ - "ld 10, 64(11)\n\t" /* arg8->r10 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)" /* restore tocptr */ \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+9]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - _argvec[2+7] = (unsigned long)arg7; \ - _argvec[2+8] = (unsigned long)arg8; \ - _argvec[2+9] = (unsigned long)arg9; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "addi 1,1,-128\n\t" /* expand stack frame */ \ - /* arg9 */ \ - "ld 3,72(11)\n\t" \ - "std 3,112(1)\n\t" \ - /* args1-8 */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 6, 32(11)\n\t" /* arg4->r6 */ \ - "ld 7, 40(11)\n\t" /* arg5->r7 */ \ - "ld 8, 48(11)\n\t" /* arg6->r8 */ \ - "ld 9, 56(11)\n\t" /* arg7->r9 */ \ - "ld 10, 64(11)\n\t" /* arg8->r10 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)\n\t" /* restore tocptr */ \ - "addi 1,1,128" /* restore frame */ \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9,arg10) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+10]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - _argvec[2+7] = (unsigned long)arg7; \ - _argvec[2+8] = (unsigned long)arg8; \ - _argvec[2+9] = (unsigned long)arg9; \ - _argvec[2+10] = (unsigned long)arg10; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "addi 1,1,-128\n\t" /* expand stack frame */ \ - /* arg10 */ \ - "ld 3,80(11)\n\t" \ - "std 3,120(1)\n\t" \ - /* arg9 */ \ - "ld 3,72(11)\n\t" \ - "std 3,112(1)\n\t" \ - /* args1-8 */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 6, 32(11)\n\t" /* arg4->r6 */ \ - "ld 7, 40(11)\n\t" /* arg5->r7 */ \ - "ld 8, 48(11)\n\t" /* arg6->r8 */ \ - "ld 9, 56(11)\n\t" /* arg7->r9 */ \ - "ld 10, 64(11)\n\t" /* arg8->r10 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)\n\t" /* restore tocptr */ \ - "addi 1,1,128" /* restore frame */ \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9,arg10,arg11) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+11]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - _argvec[2+7] = (unsigned long)arg7; \ - _argvec[2+8] = (unsigned long)arg8; \ - _argvec[2+9] = (unsigned long)arg9; \ - _argvec[2+10] = (unsigned long)arg10; \ - _argvec[2+11] = (unsigned long)arg11; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "addi 1,1,-144\n\t" /* expand stack frame */ \ - /* arg11 */ \ - "ld 3,88(11)\n\t" \ - "std 3,128(1)\n\t" \ - /* arg10 */ \ - "ld 3,80(11)\n\t" \ - "std 3,120(1)\n\t" \ - /* arg9 */ \ - "ld 3,72(11)\n\t" \ - "std 3,112(1)\n\t" \ - /* args1-8 */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 6, 32(11)\n\t" /* arg4->r6 */ \ - "ld 7, 40(11)\n\t" /* arg5->r7 */ \ - "ld 8, 48(11)\n\t" /* arg6->r8 */ \ - "ld 9, 56(11)\n\t" /* arg7->r9 */ \ - "ld 10, 64(11)\n\t" /* arg8->r10 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)\n\t" /* restore tocptr */ \ - "addi 1,1,144" /* restore frame */ \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9,arg10,arg11,arg12) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+12]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - _argvec[2+7] = (unsigned long)arg7; \ - _argvec[2+8] = (unsigned long)arg8; \ - _argvec[2+9] = (unsigned long)arg9; \ - _argvec[2+10] = (unsigned long)arg10; \ - _argvec[2+11] = (unsigned long)arg11; \ - _argvec[2+12] = (unsigned long)arg12; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "addi 1,1,-144\n\t" /* expand stack frame */ \ - /* arg12 */ \ - "ld 3,96(11)\n\t" \ - "std 3,136(1)\n\t" \ - /* arg11 */ \ - "ld 3,88(11)\n\t" \ - "std 3,128(1)\n\t" \ - /* arg10 */ \ - "ld 3,80(11)\n\t" \ - "std 3,120(1)\n\t" \ - /* arg9 */ \ - "ld 3,72(11)\n\t" \ - "std 3,112(1)\n\t" \ - /* args1-8 */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 6, 32(11)\n\t" /* arg4->r6 */ \ - "ld 7, 40(11)\n\t" /* arg5->r7 */ \ - "ld 8, 48(11)\n\t" /* arg6->r8 */ \ - "ld 9, 56(11)\n\t" /* arg7->r9 */ \ - "ld 10, 64(11)\n\t" /* arg8->r10 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)\n\t" /* restore tocptr */ \ - "addi 1,1,144" /* restore frame */ \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#endif /* PLAT_ppc64_linux */ - -/* ------------------------ ppc32-aix5 ------------------------- */ - -#if defined(PLAT_ppc32_aix5) - -/* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */ - -/* These regs are trashed by the hidden call. */ -#define __CALLER_SAVED_REGS \ - "lr", "ctr", "xer", \ - "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7", \ - "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", \ - "r11", "r12", "r13" - -/* Expand the stack frame, copying enough info that unwinding - still works. Trashes r3. */ - -#define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr) \ - "addi 1,1,-" #_n_fr "\n\t" \ - "lwz 3," #_n_fr "(1)\n\t" \ - "stw 3,0(1)\n\t" - -#define VG_CONTRACT_FRAME_BY(_n_fr) \ - "addi 1,1," #_n_fr "\n\t" - -/* These CALL_FN_ macros assume that on ppc32-aix5, sizeof(unsigned - long) == 4. */ - -#define CALL_FN_W_v(lval, orig) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+0]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "stw 2,-8(11)\n\t" /* save tocptr */ \ - "lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \ - "lwz 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "lwz 2,-8(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_W(lval, orig, arg1) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+1]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "stw 2,-8(11)\n\t" /* save tocptr */ \ - "lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \ - "lwz 3, 4(11)\n\t" /* arg1->r3 */ \ - "lwz 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "lwz 2,-8(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_WW(lval, orig, arg1,arg2) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+2]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "stw 2,-8(11)\n\t" /* save tocptr */ \ - "lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \ - "lwz 3, 4(11)\n\t" /* arg1->r3 */ \ - "lwz 4, 8(11)\n\t" /* arg2->r4 */ \ - "lwz 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "lwz 2,-8(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+3]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "stw 2,-8(11)\n\t" /* save tocptr */ \ - "lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \ - "lwz 3, 4(11)\n\t" /* arg1->r3 */ \ - "lwz 4, 8(11)\n\t" /* arg2->r4 */ \ - "lwz 5, 12(11)\n\t" /* arg3->r5 */ \ - "lwz 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "lwz 2,-8(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+4]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "stw 2,-8(11)\n\t" /* save tocptr */ \ - "lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \ - "lwz 3, 4(11)\n\t" /* arg1->r3 */ \ - "lwz 4, 8(11)\n\t" /* arg2->r4 */ \ - "lwz 5, 12(11)\n\t" /* arg3->r5 */ \ - "lwz 6, 16(11)\n\t" /* arg4->r6 */ \ - "lwz 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "lwz 2,-8(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+5]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "stw 2,-8(11)\n\t" /* save tocptr */ \ - "lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \ - "lwz 3, 4(11)\n\t" /* arg1->r3 */ \ - "lwz 4, 8(11)\n\t" /* arg2->r4 */ \ - "lwz 5, 12(11)\n\t" /* arg3->r5 */ \ - "lwz 6, 16(11)\n\t" /* arg4->r6 */ \ - "lwz 7, 20(11)\n\t" /* arg5->r7 */ \ - "lwz 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "lwz 2,-8(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+6]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "stw 2,-8(11)\n\t" /* save tocptr */ \ - "lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \ - "lwz 3, 4(11)\n\t" /* arg1->r3 */ \ - "lwz 4, 8(11)\n\t" /* arg2->r4 */ \ - "lwz 5, 12(11)\n\t" /* arg3->r5 */ \ - "lwz 6, 16(11)\n\t" /* arg4->r6 */ \ - "lwz 7, 20(11)\n\t" /* arg5->r7 */ \ - "lwz 8, 24(11)\n\t" /* arg6->r8 */ \ - "lwz 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "lwz 2,-8(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+7]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - _argvec[2+7] = (unsigned long)arg7; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "stw 2,-8(11)\n\t" /* save tocptr */ \ - "lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \ - "lwz 3, 4(11)\n\t" /* arg1->r3 */ \ - "lwz 4, 8(11)\n\t" /* arg2->r4 */ \ - "lwz 5, 12(11)\n\t" /* arg3->r5 */ \ - "lwz 6, 16(11)\n\t" /* arg4->r6 */ \ - "lwz 7, 20(11)\n\t" /* arg5->r7 */ \ - "lwz 8, 24(11)\n\t" /* arg6->r8 */ \ - "lwz 9, 28(11)\n\t" /* arg7->r9 */ \ - "lwz 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "lwz 2,-8(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+8]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - _argvec[2+7] = (unsigned long)arg7; \ - _argvec[2+8] = (unsigned long)arg8; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "stw 2,-8(11)\n\t" /* save tocptr */ \ - "lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \ - "lwz 3, 4(11)\n\t" /* arg1->r3 */ \ - "lwz 4, 8(11)\n\t" /* arg2->r4 */ \ - "lwz 5, 12(11)\n\t" /* arg3->r5 */ \ - "lwz 6, 16(11)\n\t" /* arg4->r6 */ \ - "lwz 7, 20(11)\n\t" /* arg5->r7 */ \ - "lwz 8, 24(11)\n\t" /* arg6->r8 */ \ - "lwz 9, 28(11)\n\t" /* arg7->r9 */ \ - "lwz 10, 32(11)\n\t" /* arg8->r10 */ \ - "lwz 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "lwz 2,-8(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+9]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - _argvec[2+7] = (unsigned long)arg7; \ - _argvec[2+8] = (unsigned long)arg8; \ - _argvec[2+9] = (unsigned long)arg9; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "stw 2,-8(11)\n\t" /* save tocptr */ \ - "lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \ - VG_EXPAND_FRAME_BY_trashes_r3(64) \ - /* arg9 */ \ - "lwz 3,36(11)\n\t" \ - "stw 3,56(1)\n\t" \ - /* args1-8 */ \ - "lwz 3, 4(11)\n\t" /* arg1->r3 */ \ - "lwz 4, 8(11)\n\t" /* arg2->r4 */ \ - "lwz 5, 12(11)\n\t" /* arg3->r5 */ \ - "lwz 6, 16(11)\n\t" /* arg4->r6 */ \ - "lwz 7, 20(11)\n\t" /* arg5->r7 */ \ - "lwz 8, 24(11)\n\t" /* arg6->r8 */ \ - "lwz 9, 28(11)\n\t" /* arg7->r9 */ \ - "lwz 10, 32(11)\n\t" /* arg8->r10 */ \ - "lwz 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "lwz 2,-8(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(64) \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9,arg10) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+10]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - _argvec[2+7] = (unsigned long)arg7; \ - _argvec[2+8] = (unsigned long)arg8; \ - _argvec[2+9] = (unsigned long)arg9; \ - _argvec[2+10] = (unsigned long)arg10; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "stw 2,-8(11)\n\t" /* save tocptr */ \ - "lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \ - VG_EXPAND_FRAME_BY_trashes_r3(64) \ - /* arg10 */ \ - "lwz 3,40(11)\n\t" \ - "stw 3,60(1)\n\t" \ - /* arg9 */ \ - "lwz 3,36(11)\n\t" \ - "stw 3,56(1)\n\t" \ - /* args1-8 */ \ - "lwz 3, 4(11)\n\t" /* arg1->r3 */ \ - "lwz 4, 8(11)\n\t" /* arg2->r4 */ \ - "lwz 5, 12(11)\n\t" /* arg3->r5 */ \ - "lwz 6, 16(11)\n\t" /* arg4->r6 */ \ - "lwz 7, 20(11)\n\t" /* arg5->r7 */ \ - "lwz 8, 24(11)\n\t" /* arg6->r8 */ \ - "lwz 9, 28(11)\n\t" /* arg7->r9 */ \ - "lwz 10, 32(11)\n\t" /* arg8->r10 */ \ - "lwz 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "lwz 2,-8(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(64) \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9,arg10,arg11) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+11]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - _argvec[2+7] = (unsigned long)arg7; \ - _argvec[2+8] = (unsigned long)arg8; \ - _argvec[2+9] = (unsigned long)arg9; \ - _argvec[2+10] = (unsigned long)arg10; \ - _argvec[2+11] = (unsigned long)arg11; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "stw 2,-8(11)\n\t" /* save tocptr */ \ - "lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \ - VG_EXPAND_FRAME_BY_trashes_r3(72) \ - /* arg11 */ \ - "lwz 3,44(11)\n\t" \ - "stw 3,64(1)\n\t" \ - /* arg10 */ \ - "lwz 3,40(11)\n\t" \ - "stw 3,60(1)\n\t" \ - /* arg9 */ \ - "lwz 3,36(11)\n\t" \ - "stw 3,56(1)\n\t" \ - /* args1-8 */ \ - "lwz 3, 4(11)\n\t" /* arg1->r3 */ \ - "lwz 4, 8(11)\n\t" /* arg2->r4 */ \ - "lwz 5, 12(11)\n\t" /* arg3->r5 */ \ - "lwz 6, 16(11)\n\t" /* arg4->r6 */ \ - "lwz 7, 20(11)\n\t" /* arg5->r7 */ \ - "lwz 8, 24(11)\n\t" /* arg6->r8 */ \ - "lwz 9, 28(11)\n\t" /* arg7->r9 */ \ - "lwz 10, 32(11)\n\t" /* arg8->r10 */ \ - "lwz 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "lwz 2,-8(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(72) \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9,arg10,arg11,arg12) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+12]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - _argvec[2+7] = (unsigned long)arg7; \ - _argvec[2+8] = (unsigned long)arg8; \ - _argvec[2+9] = (unsigned long)arg9; \ - _argvec[2+10] = (unsigned long)arg10; \ - _argvec[2+11] = (unsigned long)arg11; \ - _argvec[2+12] = (unsigned long)arg12; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "stw 2,-8(11)\n\t" /* save tocptr */ \ - "lwz 2,-4(11)\n\t" /* use nraddr's tocptr */ \ - VG_EXPAND_FRAME_BY_trashes_r3(72) \ - /* arg12 */ \ - "lwz 3,48(11)\n\t" \ - "stw 3,68(1)\n\t" \ - /* arg11 */ \ - "lwz 3,44(11)\n\t" \ - "stw 3,64(1)\n\t" \ - /* arg10 */ \ - "lwz 3,40(11)\n\t" \ - "stw 3,60(1)\n\t" \ - /* arg9 */ \ - "lwz 3,36(11)\n\t" \ - "stw 3,56(1)\n\t" \ - /* args1-8 */ \ - "lwz 3, 4(11)\n\t" /* arg1->r3 */ \ - "lwz 4, 8(11)\n\t" /* arg2->r4 */ \ - "lwz 5, 12(11)\n\t" /* arg3->r5 */ \ - "lwz 6, 16(11)\n\t" /* arg4->r6 */ \ - "lwz 7, 20(11)\n\t" /* arg5->r7 */ \ - "lwz 8, 24(11)\n\t" /* arg6->r8 */ \ - "lwz 9, 28(11)\n\t" /* arg7->r9 */ \ - "lwz 10, 32(11)\n\t" /* arg8->r10 */ \ - "lwz 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "lwz 2,-8(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(72) \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#endif /* PLAT_ppc32_aix5 */ - -/* ------------------------ ppc64-aix5 ------------------------- */ - -#if defined(PLAT_ppc64_aix5) - -/* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */ - -/* These regs are trashed by the hidden call. */ -#define __CALLER_SAVED_REGS \ - "lr", "ctr", "xer", \ - "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7", \ - "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", \ - "r11", "r12", "r13" - -/* Expand the stack frame, copying enough info that unwinding - still works. Trashes r3. */ - -#define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr) \ - "addi 1,1,-" #_n_fr "\n\t" \ - "ld 3," #_n_fr "(1)\n\t" \ - "std 3,0(1)\n\t" - -#define VG_CONTRACT_FRAME_BY(_n_fr) \ - "addi 1,1," #_n_fr "\n\t" - -/* These CALL_FN_ macros assume that on ppc64-aix5, sizeof(unsigned - long) == 8. */ - -#define CALL_FN_W_v(lval, orig) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+0]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_W(lval, orig, arg1) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+1]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_WW(lval, orig, arg1,arg2) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+2]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+3]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+4]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 6, 32(11)\n\t" /* arg4->r6 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+5]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 6, 32(11)\n\t" /* arg4->r6 */ \ - "ld 7, 40(11)\n\t" /* arg5->r7 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+6]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 6, 32(11)\n\t" /* arg4->r6 */ \ - "ld 7, 40(11)\n\t" /* arg5->r7 */ \ - "ld 8, 48(11)\n\t" /* arg6->r8 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+7]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - _argvec[2+7] = (unsigned long)arg7; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 6, 32(11)\n\t" /* arg4->r6 */ \ - "ld 7, 40(11)\n\t" /* arg5->r7 */ \ - "ld 8, 48(11)\n\t" /* arg6->r8 */ \ - "ld 9, 56(11)\n\t" /* arg7->r9 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+8]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - _argvec[2+7] = (unsigned long)arg7; \ - _argvec[2+8] = (unsigned long)arg8; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 6, 32(11)\n\t" /* arg4->r6 */ \ - "ld 7, 40(11)\n\t" /* arg5->r7 */ \ - "ld 8, 48(11)\n\t" /* arg6->r8 */ \ - "ld 9, 56(11)\n\t" /* arg7->r9 */ \ - "ld 10, 64(11)\n\t" /* arg8->r10 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+9]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - _argvec[2+7] = (unsigned long)arg7; \ - _argvec[2+8] = (unsigned long)arg8; \ - _argvec[2+9] = (unsigned long)arg9; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - VG_EXPAND_FRAME_BY_trashes_r3(128) \ - /* arg9 */ \ - "ld 3,72(11)\n\t" \ - "std 3,112(1)\n\t" \ - /* args1-8 */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 6, 32(11)\n\t" /* arg4->r6 */ \ - "ld 7, 40(11)\n\t" /* arg5->r7 */ \ - "ld 8, 48(11)\n\t" /* arg6->r8 */ \ - "ld 9, 56(11)\n\t" /* arg7->r9 */ \ - "ld 10, 64(11)\n\t" /* arg8->r10 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(128) \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9,arg10) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+10]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - _argvec[2+7] = (unsigned long)arg7; \ - _argvec[2+8] = (unsigned long)arg8; \ - _argvec[2+9] = (unsigned long)arg9; \ - _argvec[2+10] = (unsigned long)arg10; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - VG_EXPAND_FRAME_BY_trashes_r3(128) \ - /* arg10 */ \ - "ld 3,80(11)\n\t" \ - "std 3,120(1)\n\t" \ - /* arg9 */ \ - "ld 3,72(11)\n\t" \ - "std 3,112(1)\n\t" \ - /* args1-8 */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 6, 32(11)\n\t" /* arg4->r6 */ \ - "ld 7, 40(11)\n\t" /* arg5->r7 */ \ - "ld 8, 48(11)\n\t" /* arg6->r8 */ \ - "ld 9, 56(11)\n\t" /* arg7->r9 */ \ - "ld 10, 64(11)\n\t" /* arg8->r10 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(128) \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9,arg10,arg11) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+11]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - _argvec[2+7] = (unsigned long)arg7; \ - _argvec[2+8] = (unsigned long)arg8; \ - _argvec[2+9] = (unsigned long)arg9; \ - _argvec[2+10] = (unsigned long)arg10; \ - _argvec[2+11] = (unsigned long)arg11; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - VG_EXPAND_FRAME_BY_trashes_r3(144) \ - /* arg11 */ \ - "ld 3,88(11)\n\t" \ - "std 3,128(1)\n\t" \ - /* arg10 */ \ - "ld 3,80(11)\n\t" \ - "std 3,120(1)\n\t" \ - /* arg9 */ \ - "ld 3,72(11)\n\t" \ - "std 3,112(1)\n\t" \ - /* args1-8 */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 6, 32(11)\n\t" /* arg4->r6 */ \ - "ld 7, 40(11)\n\t" /* arg5->r7 */ \ - "ld 8, 48(11)\n\t" /* arg6->r8 */ \ - "ld 9, 56(11)\n\t" /* arg7->r9 */ \ - "ld 10, 64(11)\n\t" /* arg8->r10 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(144) \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ - arg7,arg8,arg9,arg10,arg11,arg12) \ - do { \ - volatile OrigFn _orig = (orig); \ - volatile unsigned long _argvec[3+12]; \ - volatile unsigned long _res; \ - /* _argvec[0] holds current r2 across the call */ \ - _argvec[1] = (unsigned long)_orig.r2; \ - _argvec[2] = (unsigned long)_orig.nraddr; \ - _argvec[2+1] = (unsigned long)arg1; \ - _argvec[2+2] = (unsigned long)arg2; \ - _argvec[2+3] = (unsigned long)arg3; \ - _argvec[2+4] = (unsigned long)arg4; \ - _argvec[2+5] = (unsigned long)arg5; \ - _argvec[2+6] = (unsigned long)arg6; \ - _argvec[2+7] = (unsigned long)arg7; \ - _argvec[2+8] = (unsigned long)arg8; \ - _argvec[2+9] = (unsigned long)arg9; \ - _argvec[2+10] = (unsigned long)arg10; \ - _argvec[2+11] = (unsigned long)arg11; \ - _argvec[2+12] = (unsigned long)arg12; \ - __asm__ volatile( \ - "mr 11,%1\n\t" \ - VG_EXPAND_FRAME_BY_trashes_r3(512) \ - "std 2,-16(11)\n\t" /* save tocptr */ \ - "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ - VG_EXPAND_FRAME_BY_trashes_r3(144) \ - /* arg12 */ \ - "ld 3,96(11)\n\t" \ - "std 3,136(1)\n\t" \ - /* arg11 */ \ - "ld 3,88(11)\n\t" \ - "std 3,128(1)\n\t" \ - /* arg10 */ \ - "ld 3,80(11)\n\t" \ - "std 3,120(1)\n\t" \ - /* arg9 */ \ - "ld 3,72(11)\n\t" \ - "std 3,112(1)\n\t" \ - /* args1-8 */ \ - "ld 3, 8(11)\n\t" /* arg1->r3 */ \ - "ld 4, 16(11)\n\t" /* arg2->r4 */ \ - "ld 5, 24(11)\n\t" /* arg3->r5 */ \ - "ld 6, 32(11)\n\t" /* arg4->r6 */ \ - "ld 7, 40(11)\n\t" /* arg5->r7 */ \ - "ld 8, 48(11)\n\t" /* arg6->r8 */ \ - "ld 9, 56(11)\n\t" /* arg7->r9 */ \ - "ld 10, 64(11)\n\t" /* arg8->r10 */ \ - "ld 11, 0(11)\n\t" /* target->r11 */ \ - VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ - "mr 11,%1\n\t" \ - "mr %0,3\n\t" \ - "ld 2,-16(11)\n\t" /* restore tocptr */ \ - VG_CONTRACT_FRAME_BY(144) \ - VG_CONTRACT_FRAME_BY(512) \ - : /*out*/ "=r" (_res) \ - : /*in*/ "r" (&_argvec[2]) \ - : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS \ - ); \ - lval = (__typeof__(lval)) _res; \ - } while (0) - -#endif /* PLAT_ppc64_aix5 */ - - -/* ------------------------------------------------------------------ */ -/* ARCHITECTURE INDEPENDENT MACROS for CLIENT REQUESTS. */ -/* */ -/* ------------------------------------------------------------------ */ - -/* Some request codes. There are many more of these, but most are not - exposed to end-user view. These are the public ones, all of the - form 0x1000 + small_number. - - Core ones are in the range 0x00000000--0x0000ffff. The non-public - ones start at 0x2000. -*/ - -/* These macros are used by tools -- they must be public, but don't - embed them into other programs. */ -#define VG_USERREQ_TOOL_BASE(a,b) \ - ((unsigned int)(((a)&0xff) << 24 | ((b)&0xff) << 16)) -#define VG_IS_TOOL_USERREQ(a, b, v) \ - (VG_USERREQ_TOOL_BASE(a,b) == ((v) & 0xffff0000)) - -/* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !! - This enum comprises an ABI exported by Valgrind to programs - which use client requests. DO NOT CHANGE THE ORDER OF THESE - ENTRIES, NOR DELETE ANY -- add new ones at the end. */ -typedef - enum { VG_USERREQ__RUNNING_ON_VALGRIND = 0x1001, - VG_USERREQ__DISCARD_TRANSLATIONS = 0x1002, - - /* These allow any function to be called from the simulated - CPU but run on the real CPU. Nb: the first arg passed to - the function is always the ThreadId of the running - thread! So CLIENT_CALL0 actually requires a 1 arg - function, etc. */ - VG_USERREQ__CLIENT_CALL0 = 0x1101, - VG_USERREQ__CLIENT_CALL1 = 0x1102, - VG_USERREQ__CLIENT_CALL2 = 0x1103, - VG_USERREQ__CLIENT_CALL3 = 0x1104, - - /* Can be useful in regression testing suites -- eg. can - send Valgrind's output to /dev/null and still count - errors. */ - VG_USERREQ__COUNT_ERRORS = 0x1201, - - /* These are useful and can be interpreted by any tool that - tracks malloc() et al, by using vg_replace_malloc.c. */ - VG_USERREQ__MALLOCLIKE_BLOCK = 0x1301, - VG_USERREQ__FREELIKE_BLOCK = 0x1302, - /* Memory pool support. */ - VG_USERREQ__CREATE_MEMPOOL = 0x1303, - VG_USERREQ__DESTROY_MEMPOOL = 0x1304, - VG_USERREQ__MEMPOOL_ALLOC = 0x1305, - VG_USERREQ__MEMPOOL_FREE = 0x1306, - VG_USERREQ__MEMPOOL_TRIM = 0x1307, - VG_USERREQ__MOVE_MEMPOOL = 0x1308, - VG_USERREQ__MEMPOOL_CHANGE = 0x1309, - VG_USERREQ__MEMPOOL_EXISTS = 0x130a, - - /* Allow printfs to valgrind log. */ - VG_USERREQ__PRINTF = 0x1401, - VG_USERREQ__PRINTF_BACKTRACE = 0x1402, - - /* Stack support. */ - VG_USERREQ__STACK_REGISTER = 0x1501, - VG_USERREQ__STACK_DEREGISTER = 0x1502, - VG_USERREQ__STACK_CHANGE = 0x1503 - } Vg_ClientRequest; - -#if !defined(__GNUC__) -# define __extension__ /* */ -#endif - -/* Returns the number of Valgrinds this code is running under. That - is, 0 if running natively, 1 if running under Valgrind, 2 if - running under Valgrind which is running under another Valgrind, - etc. */ -#define RUNNING_ON_VALGRIND __extension__ \ - ({unsigned int _qzz_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0 /* if not */, \ - VG_USERREQ__RUNNING_ON_VALGRIND, \ - 0, 0, 0, 0, 0); \ - _qzz_res; \ - }) - - -/* Discard translation of code in the range [_qzz_addr .. _qzz_addr + - _qzz_len - 1]. Useful if you are debugging a JITter or some such, - since it provides a way to make sure valgrind will retranslate the - invalidated area. Returns no value. */ -#define VALGRIND_DISCARD_TRANSLATIONS(_qzz_addr,_qzz_len) \ - {unsigned int _qzz_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \ - VG_USERREQ__DISCARD_TRANSLATIONS, \ - _qzz_addr, _qzz_len, 0, 0, 0); \ - } - - -/* These requests are for getting Valgrind itself to print something. - Possibly with a backtrace. This is a really ugly hack. */ - -#if defined(NVALGRIND) - -# define VALGRIND_PRINTF(...) -# define VALGRIND_PRINTF_BACKTRACE(...) - -#else /* NVALGRIND */ - -/* Modern GCC will optimize the static routine out if unused, - and unused attribute will shut down warnings about it. */ -static int VALGRIND_PRINTF(const char *format, ...) - __attribute__((format(__printf__, 1, 2), __unused__)); -static int -VALGRIND_PRINTF(const char *format, ...) -{ - unsigned long _qzz_res; - va_list vargs; - va_start(vargs, format); - VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, VG_USERREQ__PRINTF, - (unsigned long)format, (unsigned long)vargs, - 0, 0, 0); - va_end(vargs); - return (int)_qzz_res; -} - -static int VALGRIND_PRINTF_BACKTRACE(const char *format, ...) - __attribute__((format(__printf__, 1, 2), __unused__)); -static int -VALGRIND_PRINTF_BACKTRACE(const char *format, ...) -{ - unsigned long _qzz_res; - va_list vargs; - va_start(vargs, format); - VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, VG_USERREQ__PRINTF_BACKTRACE, - (unsigned long)format, (unsigned long)vargs, - 0, 0, 0); - va_end(vargs); - return (int)_qzz_res; -} - -#endif /* NVALGRIND */ - - -/* These requests allow control to move from the simulated CPU to the - real CPU, calling an arbitary function. - - Note that the current ThreadId is inserted as the first argument. - So this call: - - VALGRIND_NON_SIMD_CALL2(f, arg1, arg2) - - requires f to have this signature: - - Word f(Word tid, Word arg1, Word arg2) - - where "Word" is a word-sized type. - - Note that these client requests are not entirely reliable. For example, - if you call a function with them that subsequently calls printf(), - there's a high chance Valgrind will crash. Generally, your prospects of - these working are made higher if the called function does not refer to - any global variables, and does not refer to any libc or other functions - (printf et al). Any kind of entanglement with libc or dynamic linking is - likely to have a bad outcome, for tricky reasons which we've grappled - with a lot in the past. -*/ -#define VALGRIND_NON_SIMD_CALL0(_qyy_fn) \ - __extension__ \ - ({unsigned long _qyy_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */, \ - VG_USERREQ__CLIENT_CALL0, \ - _qyy_fn, \ - 0, 0, 0, 0); \ - _qyy_res; \ - }) - -#define VALGRIND_NON_SIMD_CALL1(_qyy_fn, _qyy_arg1) \ - __extension__ \ - ({unsigned long _qyy_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */, \ - VG_USERREQ__CLIENT_CALL1, \ - _qyy_fn, \ - _qyy_arg1, 0, 0, 0); \ - _qyy_res; \ - }) - -#define VALGRIND_NON_SIMD_CALL2(_qyy_fn, _qyy_arg1, _qyy_arg2) \ - __extension__ \ - ({unsigned long _qyy_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */, \ - VG_USERREQ__CLIENT_CALL2, \ - _qyy_fn, \ - _qyy_arg1, _qyy_arg2, 0, 0); \ - _qyy_res; \ - }) - -#define VALGRIND_NON_SIMD_CALL3(_qyy_fn, _qyy_arg1, _qyy_arg2, _qyy_arg3) \ - __extension__ \ - ({unsigned long _qyy_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */, \ - VG_USERREQ__CLIENT_CALL3, \ - _qyy_fn, \ - _qyy_arg1, _qyy_arg2, \ - _qyy_arg3, 0); \ - _qyy_res; \ - }) - - -/* Counts the number of errors that have been recorded by a tool. Nb: - the tool must record the errors with VG_(maybe_record_error)() or - VG_(unique_error)() for them to be counted. */ -#define VALGRIND_COUNT_ERRORS \ - __extension__ \ - ({unsigned int _qyy_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */, \ - VG_USERREQ__COUNT_ERRORS, \ - 0, 0, 0, 0, 0); \ - _qyy_res; \ - }) - -/* Mark a block of memory as having been allocated by a malloc()-like - function. `addr' is the start of the usable block (ie. after any - redzone) `rzB' is redzone size if the allocator can apply redzones; - use '0' if not. Adding redzones makes it more likely Valgrind will spot - block overruns. `is_zeroed' indicates if the memory is zeroed, as it is - for calloc(). Put it immediately after the point where a block is - allocated. - - If you're using Memcheck: If you're allocating memory via superblocks, - and then handing out small chunks of each superblock, if you don't have - redzones on your small blocks, it's worth marking the superblock with - VALGRIND_MAKE_MEM_NOACCESS when it's created, so that block overruns are - detected. But if you can put redzones on, it's probably better to not do - this, so that messages for small overruns are described in terms of the - small block rather than the superblock (but if you have a big overrun - that skips over a redzone, you could miss an error this way). See - memcheck/tests/custom_alloc.c for an example. - - WARNING: if your allocator uses malloc() or 'new' to allocate - superblocks, rather than mmap() or brk(), this will not work properly -- - you'll likely get assertion failures during leak detection. This is - because Valgrind doesn't like seeing overlapping heap blocks. Sorry. - - Nb: block must be freed via a free()-like function specified - with VALGRIND_FREELIKE_BLOCK or mismatch errors will occur. */ -#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed) \ - {unsigned int _qzz_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \ - VG_USERREQ__MALLOCLIKE_BLOCK, \ - addr, sizeB, rzB, is_zeroed, 0); \ - } - -/* Mark a block of memory as having been freed by a free()-like function. - `rzB' is redzone size; it must match that given to - VALGRIND_MALLOCLIKE_BLOCK. Memory not freed will be detected by the leak - checker. Put it immediately after the point where the block is freed. */ -#define VALGRIND_FREELIKE_BLOCK(addr, rzB) \ - {unsigned int _qzz_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \ - VG_USERREQ__FREELIKE_BLOCK, \ - addr, rzB, 0, 0, 0); \ - } - -/* Create a memory pool. */ -#define VALGRIND_CREATE_MEMPOOL(pool, rzB, is_zeroed) \ - {unsigned int _qzz_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \ - VG_USERREQ__CREATE_MEMPOOL, \ - pool, rzB, is_zeroed, 0, 0); \ - } - -/* Destroy a memory pool. */ -#define VALGRIND_DESTROY_MEMPOOL(pool) \ - {unsigned int _qzz_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \ - VG_USERREQ__DESTROY_MEMPOOL, \ - pool, 0, 0, 0, 0); \ - } - -/* Associate a piece of memory with a memory pool. */ -#define VALGRIND_MEMPOOL_ALLOC(pool, addr, size) \ - {unsigned int _qzz_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \ - VG_USERREQ__MEMPOOL_ALLOC, \ - pool, addr, size, 0, 0); \ - } - -/* Disassociate a piece of memory from a memory pool. */ -#define VALGRIND_MEMPOOL_FREE(pool, addr) \ - {unsigned int _qzz_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \ - VG_USERREQ__MEMPOOL_FREE, \ - pool, addr, 0, 0, 0); \ - } - -/* Disassociate any pieces outside a particular range. */ -#define VALGRIND_MEMPOOL_TRIM(pool, addr, size) \ - {unsigned int _qzz_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \ - VG_USERREQ__MEMPOOL_TRIM, \ - pool, addr, size, 0, 0); \ - } - -/* Resize and/or move a piece associated with a memory pool. */ -#define VALGRIND_MOVE_MEMPOOL(poolA, poolB) \ - {unsigned int _qzz_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \ - VG_USERREQ__MOVE_MEMPOOL, \ - poolA, poolB, 0, 0, 0); \ - } - -/* Resize and/or move a piece associated with a memory pool. */ -#define VALGRIND_MEMPOOL_CHANGE(pool, addrA, addrB, size) \ - {unsigned int _qzz_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \ - VG_USERREQ__MEMPOOL_CHANGE, \ - pool, addrA, addrB, size, 0); \ - } - -/* Return 1 if a mempool exists, else 0. */ -#define VALGRIND_MEMPOOL_EXISTS(pool) \ - ({unsigned int _qzz_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \ - VG_USERREQ__MEMPOOL_EXISTS, \ - pool, 0, 0, 0, 0); \ - _qzz_res; \ - }) - -/* Mark a piece of memory as being a stack. Returns a stack id. */ -#define VALGRIND_STACK_REGISTER(start, end) \ - ({unsigned int _qzz_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \ - VG_USERREQ__STACK_REGISTER, \ - start, end, 0, 0, 0); \ - _qzz_res; \ - }) - -/* Unmark the piece of memory associated with a stack id as being a - stack. */ -#define VALGRIND_STACK_DEREGISTER(id) \ - {unsigned int _qzz_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \ - VG_USERREQ__STACK_DEREGISTER, \ - id, 0, 0, 0, 0); \ - } - -/* Change the start and end address of the stack id. */ -#define VALGRIND_STACK_CHANGE(id, start, end) \ - {unsigned int _qzz_res; \ - VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, \ - VG_USERREQ__STACK_CHANGE, \ - id, start, end, 0, 0); \ - } - - -#undef PLAT_x86_linux -#undef PLAT_amd64_linux -#undef PLAT_ppc32_linux -#undef PLAT_ppc64_linux -#undef PLAT_ppc32_aix5 -#undef PLAT_ppc64_aix5 - -#endif /* __VALGRIND_H */ diff --git a/contrib/libtcmalloc/src/thread_cache.cc b/contrib/libtcmalloc/src/thread_cache.cc deleted file mode 100644 index 81b3694d563..00000000000 --- a/contrib/libtcmalloc/src/thread_cache.cc +++ /dev/null @@ -1,479 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Ken Ashcraft - -#include "config.h" -#include "thread_cache.h" -#include -#include // for memcpy -#include // for max, min -#include "base/commandlineflags.h" // for SpinLockHolder -#include "base/spinlock.h" // for SpinLockHolder -#include "getenv_safe.h" // for TCMallocGetenvSafe -#include "central_freelist.h" // for CentralFreeListPadded -#include "maybe_threads.h" - -using std::min; -using std::max; - -// Note: this is initialized manually in InitModule to ensure that -// it's configured at right time -// -// DEFINE_int64(tcmalloc_max_total_thread_cache_bytes, -// EnvToInt64("TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES", -// kDefaultOverallThreadCacheSize), -// "Bound on the total amount of bytes allocated to " -// "thread caches. This bound is not strict, so it is possible " -// "for the cache to go over this bound in certain circumstances. " -// "Maximum value of this flag is capped to 1 GB."); - - -namespace tcmalloc { - -static bool phinited = false; - -volatile size_t ThreadCache::per_thread_cache_size_ = kMaxThreadCacheSize; -size_t ThreadCache::overall_thread_cache_size_ = kDefaultOverallThreadCacheSize; -ssize_t ThreadCache::unclaimed_cache_space_ = kDefaultOverallThreadCacheSize; -PageHeapAllocator threadcache_allocator; -ThreadCache* ThreadCache::thread_heaps_ = NULL; -int ThreadCache::thread_heap_count_ = 0; -ThreadCache* ThreadCache::next_memory_steal_ = NULL; -#ifdef HAVE_TLS -__thread ThreadCache::ThreadLocalData ThreadCache::threadlocal_data_ - ATTR_INITIAL_EXEC - = {0, 0}; -#endif -bool ThreadCache::tsd_inited_ = false; -pthread_key_t ThreadCache::heap_key_; - -void ThreadCache::Init(pthread_t tid) { - size_ = 0; - - max_size_ = 0; - IncreaseCacheLimitLocked(); - if (max_size_ == 0) { - // There isn't enough memory to go around. Just give the minimum to - // this thread. - max_size_ = kMinThreadCacheSize; - - // Take unclaimed_cache_space_ negative. - unclaimed_cache_space_ -= kMinThreadCacheSize; - ASSERT(unclaimed_cache_space_ < 0); - } - - next_ = NULL; - prev_ = NULL; - tid_ = tid; - in_setspecific_ = false; - for (size_t cl = 0; cl < kNumClasses; ++cl) { - list_[cl].Init(); - } - - uint32_t sampler_seed; - memcpy(&sampler_seed, &tid, sizeof(sampler_seed)); - sampler_.Init(sampler_seed); -} - -void ThreadCache::Cleanup() { - // Put unused memory back into central cache - for (int cl = 0; cl < kNumClasses; ++cl) { - if (list_[cl].length() > 0) { - ReleaseToCentralCache(&list_[cl], cl, list_[cl].length()); - } - } -} - -// Remove some objects of class "cl" from central cache and add to thread heap. -// On success, return the first object for immediate use; otherwise return NULL. -void* ThreadCache::FetchFromCentralCache(size_t cl, size_t byte_size) { - FreeList* list = &list_[cl]; - ASSERT(list->empty()); - const int batch_size = Static::sizemap()->num_objects_to_move(cl); - - const int num_to_move = min(list->max_length(), batch_size); - void *start, *end; - int fetch_count = Static::central_cache()[cl].RemoveRange( - &start, &end, num_to_move); - - ASSERT((start == NULL) == (fetch_count == 0)); - if (--fetch_count >= 0) { - size_ += byte_size * fetch_count; - list->PushRange(fetch_count, SLL_Next(start), end); - } - - // Increase max length slowly up to batch_size. After that, - // increase by batch_size in one shot so that the length is a - // multiple of batch_size. - if (list->max_length() < batch_size) { - list->set_max_length(list->max_length() + 1); - } else { - // Don't let the list get too long. In 32 bit builds, the length - // is represented by a 16 bit int, so we need to watch out for - // integer overflow. - int new_length = min(list->max_length() + batch_size, - kMaxDynamicFreeListLength); - // The list's max_length must always be a multiple of batch_size, - // and kMaxDynamicFreeListLength is not necessarily a multiple - // of batch_size. - new_length -= new_length % batch_size; - ASSERT(new_length % batch_size == 0); - list->set_max_length(new_length); - } - return start; -} - -void ThreadCache::ListTooLong(FreeList* list, size_t cl) { - const int batch_size = Static::sizemap()->num_objects_to_move(cl); - ReleaseToCentralCache(list, cl, batch_size); - - // If the list is too long, we need to transfer some number of - // objects to the central cache. Ideally, we would transfer - // num_objects_to_move, so the code below tries to make max_length - // converge on num_objects_to_move. - - if (list->max_length() < batch_size) { - // Slow start the max_length so we don't overreserve. - list->set_max_length(list->max_length() + 1); - } else if (list->max_length() > batch_size) { - // If we consistently go over max_length, shrink max_length. If we don't - // shrink it, some amount of memory will always stay in this freelist. - list->set_length_overages(list->length_overages() + 1); - if (list->length_overages() > kMaxOverages) { - ASSERT(list->max_length() > batch_size); - list->set_max_length(list->max_length() - batch_size); - list->set_length_overages(0); - } - } -} - -// Remove some objects of class "cl" from thread heap and add to central cache -void ThreadCache::ReleaseToCentralCache(FreeList* src, size_t cl, int N) { - ASSERT(src == &list_[cl]); - if (N > src->length()) N = src->length(); - size_t delta_bytes = N * Static::sizemap()->ByteSizeForClass(cl); - - // We return prepackaged chains of the correct size to the central cache. - // TODO: Use the same format internally in the thread caches? - int batch_size = Static::sizemap()->num_objects_to_move(cl); - while (N > batch_size) { - void *tail, *head; - src->PopRange(batch_size, &head, &tail); - Static::central_cache()[cl].InsertRange(head, tail, batch_size); - N -= batch_size; - } - void *tail, *head; - src->PopRange(N, &head, &tail); - Static::central_cache()[cl].InsertRange(head, tail, N); - size_ -= delta_bytes; -} - -// Release idle memory to the central cache -void ThreadCache::Scavenge() { - // If the low-water mark for the free list is L, it means we would - // not have had to allocate anything from the central cache even if - // we had reduced the free list size by L. We aim to get closer to - // that situation by dropping L/2 nodes from the free list. This - // may not release much memory, but if so we will call scavenge again - // pretty soon and the low-water marks will be high on that call. - for (int cl = 0; cl < kNumClasses; cl++) { - FreeList* list = &list_[cl]; - const int lowmark = list->lowwatermark(); - if (lowmark > 0) { - const int drop = (lowmark > 1) ? lowmark/2 : 1; - ReleaseToCentralCache(list, cl, drop); - - // Shrink the max length if it isn't used. Only shrink down to - // batch_size -- if the thread was active enough to get the max_length - // above batch_size, it will likely be that active again. If - // max_length shinks below batch_size, the thread will have to - // go through the slow-start behavior again. The slow-start is useful - // mainly for threads that stay relatively idle for their entire - // lifetime. - const int batch_size = Static::sizemap()->num_objects_to_move(cl); - if (list->max_length() > batch_size) { - list->set_max_length( - max(list->max_length() - batch_size, batch_size)); - } - } - list->clear_lowwatermark(); - } - - IncreaseCacheLimit(); -} - -void ThreadCache::IncreaseCacheLimit() { - SpinLockHolder h(Static::pageheap_lock()); - IncreaseCacheLimitLocked(); -} - -void ThreadCache::IncreaseCacheLimitLocked() { - if (unclaimed_cache_space_ > 0) { - // Possibly make unclaimed_cache_space_ negative. - unclaimed_cache_space_ -= kStealAmount; - max_size_ += kStealAmount; - return; - } - // Don't hold pageheap_lock too long. Try to steal from 10 other - // threads before giving up. The i < 10 condition also prevents an - // infinite loop in case none of the existing thread heaps are - // suitable places to steal from. - for (int i = 0; i < 10; - ++i, next_memory_steal_ = next_memory_steal_->next_) { - // Reached the end of the linked list. Start at the beginning. - if (next_memory_steal_ == NULL) { - ASSERT(thread_heaps_ != NULL); - next_memory_steal_ = thread_heaps_; - } - if (next_memory_steal_ == this || - next_memory_steal_->max_size_ <= kMinThreadCacheSize) { - continue; - } - next_memory_steal_->max_size_ -= kStealAmount; - max_size_ += kStealAmount; - - next_memory_steal_ = next_memory_steal_->next_; - return; - } -} - -int ThreadCache::GetSamplePeriod() { - return sampler_.GetSamplePeriod(); -} - -void ThreadCache::InitModule() { - SpinLockHolder h(Static::pageheap_lock()); - if (!phinited) { - const char *tcb = TCMallocGetenvSafe("TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES"); - if (tcb) { - set_overall_thread_cache_size(strtoll(tcb, NULL, 10)); - } - Static::InitStaticVars(); - threadcache_allocator.Init(); - phinited = 1; - } -} - -void ThreadCache::InitTSD() { - ASSERT(!tsd_inited_); - perftools_pthread_key_create(&heap_key_, DestroyThreadCache); - tsd_inited_ = true; - -#ifdef PTHREADS_CRASHES_IF_RUN_TOO_EARLY - // We may have used a fake pthread_t for the main thread. Fix it. - pthread_t zero; - memset(&zero, 0, sizeof(zero)); - SpinLockHolder h(Static::pageheap_lock()); - for (ThreadCache* h = thread_heaps_; h != NULL; h = h->next_) { - if (h->tid_ == zero) { - h->tid_ = pthread_self(); - } - } -#endif -} - -ThreadCache* ThreadCache::CreateCacheIfNecessary() { - // Initialize per-thread data if necessary - ThreadCache* heap = NULL; - { - SpinLockHolder h(Static::pageheap_lock()); - // On some old glibc's, and on freebsd's libc (as of freebsd 8.1), - // calling pthread routines (even pthread_self) too early could - // cause a segfault. Since we can call pthreads quite early, we - // have to protect against that in such situations by making a - // 'fake' pthread. This is not ideal since it doesn't work well - // when linking tcmalloc statically with apps that create threads - // before main, so we only do it if we have to. -#ifdef PTHREADS_CRASHES_IF_RUN_TOO_EARLY - pthread_t me; - if (!tsd_inited_) { - memset(&me, 0, sizeof(me)); - } else { - me = pthread_self(); - } -#else - const pthread_t me = pthread_self(); -#endif - - // This may be a recursive malloc call from pthread_setspecific() - // In that case, the heap for this thread has already been created - // and added to the linked list. So we search for that first. - for (ThreadCache* h = thread_heaps_; h != NULL; h = h->next_) { - if (h->tid_ == me) { - heap = h; - break; - } - } - - if (heap == NULL) heap = NewHeap(me); - } - - // We call pthread_setspecific() outside the lock because it may - // call malloc() recursively. We check for the recursive call using - // the "in_setspecific_" flag so that we can avoid calling - // pthread_setspecific() if we are already inside pthread_setspecific(). - if (!heap->in_setspecific_ && tsd_inited_) { - heap->in_setspecific_ = true; - perftools_pthread_setspecific(heap_key_, heap); -#ifdef HAVE_TLS - // Also keep a copy in __thread for faster retrieval - threadlocal_data_.heap = heap; - SetMinSizeForSlowPath(kMaxSize + 1); -#endif - heap->in_setspecific_ = false; - } - return heap; -} - -ThreadCache* ThreadCache::NewHeap(pthread_t tid) { - // Create the heap and add it to the linked list - ThreadCache *heap = threadcache_allocator.New(); - heap->Init(tid); - heap->next_ = thread_heaps_; - heap->prev_ = NULL; - if (thread_heaps_ != NULL) { - thread_heaps_->prev_ = heap; - } else { - // This is the only thread heap at the momment. - ASSERT(next_memory_steal_ == NULL); - next_memory_steal_ = heap; - } - thread_heaps_ = heap; - thread_heap_count_++; - return heap; -} - -void ThreadCache::BecomeIdle() { - if (!tsd_inited_) return; // No caches yet - ThreadCache* heap = GetThreadHeap(); - if (heap == NULL) return; // No thread cache to remove - if (heap->in_setspecific_) return; // Do not disturb the active caller - - heap->in_setspecific_ = true; - perftools_pthread_setspecific(heap_key_, NULL); -#ifdef HAVE_TLS - // Also update the copy in __thread - threadlocal_data_.heap = NULL; - SetMinSizeForSlowPath(0); -#endif - heap->in_setspecific_ = false; - if (GetThreadHeap() == heap) { - // Somehow heap got reinstated by a recursive call to malloc - // from pthread_setspecific. We give up in this case. - return; - } - - // We can now get rid of the heap - DeleteCache(heap); -} - -void ThreadCache::BecomeTemporarilyIdle() { - ThreadCache* heap = GetCacheIfPresent(); - if (heap) - heap->Cleanup(); -} - -void ThreadCache::DestroyThreadCache(void* ptr) { - // Note that "ptr" cannot be NULL since pthread promises not - // to invoke the destructor on NULL values, but for safety, - // we check anyway. - if (ptr == NULL) return; -#ifdef HAVE_TLS - // Prevent fast path of GetThreadHeap() from returning heap. - threadlocal_data_.heap = NULL; - SetMinSizeForSlowPath(0); -#endif - DeleteCache(reinterpret_cast(ptr)); -} - -void ThreadCache::DeleteCache(ThreadCache* heap) { - // Remove all memory from heap - heap->Cleanup(); - - // Remove from linked list - SpinLockHolder h(Static::pageheap_lock()); - if (heap->next_ != NULL) heap->next_->prev_ = heap->prev_; - if (heap->prev_ != NULL) heap->prev_->next_ = heap->next_; - if (thread_heaps_ == heap) thread_heaps_ = heap->next_; - thread_heap_count_--; - - if (next_memory_steal_ == heap) next_memory_steal_ = heap->next_; - if (next_memory_steal_ == NULL) next_memory_steal_ = thread_heaps_; - unclaimed_cache_space_ += heap->max_size_; - - threadcache_allocator.Delete(heap); -} - -void ThreadCache::RecomputePerThreadCacheSize() { - // Divide available space across threads - int n = thread_heap_count_ > 0 ? thread_heap_count_ : 1; - size_t space = overall_thread_cache_size_ / n; - - // Limit to allowed range - if (space < kMinThreadCacheSize) space = kMinThreadCacheSize; - if (space > kMaxThreadCacheSize) space = kMaxThreadCacheSize; - - double ratio = space / max(1, per_thread_cache_size_); - size_t claimed = 0; - for (ThreadCache* h = thread_heaps_; h != NULL; h = h->next_) { - // Increasing the total cache size should not circumvent the - // slow-start growth of max_size_. - if (ratio < 1.0) { - h->max_size_ = static_cast(h->max_size_ * ratio); - } - claimed += h->max_size_; - } - unclaimed_cache_space_ = overall_thread_cache_size_ - claimed; - per_thread_cache_size_ = space; -} - -void ThreadCache::GetThreadStats(uint64_t* total_bytes, uint64_t* class_count) { - for (ThreadCache* h = thread_heaps_; h != NULL; h = h->next_) { - *total_bytes += h->Size(); - if (class_count) { - for (int cl = 0; cl < kNumClasses; ++cl) { - class_count[cl] += h->freelist_length(cl); - } - } - } -} - -void ThreadCache::set_overall_thread_cache_size(size_t new_size) { - // Clip the value to a reasonable range - if (new_size < kMinThreadCacheSize) new_size = kMinThreadCacheSize; - if (new_size > (1<<30)) new_size = (1<<30); // Limit to 1GB - overall_thread_cache_size_ = new_size; - - RecomputePerThreadCacheSize(); -} - -} // namespace tcmalloc diff --git a/contrib/libtcmalloc/src/thread_cache.h b/contrib/libtcmalloc/src/thread_cache.h deleted file mode 100644 index ff7ab1ae77d..00000000000 --- a/contrib/libtcmalloc/src/thread_cache.h +++ /dev/null @@ -1,474 +0,0 @@ -// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- -// Copyright (c) 2008, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Sanjay Ghemawat - -#ifndef TCMALLOC_THREAD_CACHE_H_ -#define TCMALLOC_THREAD_CACHE_H_ - -#include "config.h" -#ifdef HAVE_PTHREAD -#include // for pthread_t, pthread_key_t -#endif -#include // for size_t, NULL -#ifdef HAVE_STDINT_H -#include // for uint32_t, uint64_t -#endif -#include // for ssize_t -#include "base/commandlineflags.h" -#include "common.h" -#include "linked_list.h" -#include "maybe_threads.h" -#include "page_heap_allocator.h" -#include "sampler.h" -#include "static_vars.h" - -#include "common.h" // for SizeMap, kMaxSize, etc -#include "internal_logging.h" // for ASSERT, etc -#include "linked_list.h" // for SLL_Pop, SLL_PopRange, etc -#include "page_heap_allocator.h" // for PageHeapAllocator -#include "sampler.h" // for Sampler -#include "static_vars.h" // for Static - -DECLARE_int64(tcmalloc_sample_parameter); - -namespace tcmalloc { - -//------------------------------------------------------------------- -// Data kept per thread -//------------------------------------------------------------------- - -class ThreadCache { - public: -#ifdef HAVE_TLS - enum { have_tls = true }; -#else - enum { have_tls = false }; -#endif - - // All ThreadCache objects are kept in a linked list (for stats collection) - ThreadCache* next_; - ThreadCache* prev_; - - void Init(pthread_t tid); - void Cleanup(); - - // Accessors (mostly just for printing stats) - int freelist_length(size_t cl) const { return list_[cl].length(); } - - // Total byte size in cache - size_t Size() const { return size_; } - - // Allocate an object of the given size and class. The size given - // must be the same as the size of the class in the size map. - void* Allocate(size_t size, size_t cl); - void Deallocate(void* ptr, size_t size_class); - - void Scavenge(); - - int GetSamplePeriod(); - - // Record allocation of "k" bytes. Return true iff allocation - // should be sampled - bool SampleAllocation(size_t k); - - static void InitModule(); - static void InitTSD(); - static ThreadCache* GetThreadHeap(); - static ThreadCache* GetCache(); - static ThreadCache* GetCacheIfPresent(); - static ThreadCache* GetCacheWhichMustBePresent(); - static ThreadCache* CreateCacheIfNecessary(); - static void BecomeIdle(); - static void BecomeTemporarilyIdle(); - static size_t MinSizeForSlowPath(); - static void SetMinSizeForSlowPath(size_t size); - static void SetUseEmergencyMalloc(); - static void ResetUseEmergencyMalloc(); - static bool IsUseEmergencyMalloc(); - - static bool IsFastPathAllowed() { return MinSizeForSlowPath() != 0; } - - // Return the number of thread heaps in use. - static inline int HeapsInUse(); - - // Adds to *total_bytes the total number of bytes used by all thread heaps. - // Also, if class_count is not NULL, it must be an array of size kNumClasses, - // and this function will increment each element of class_count by the number - // of items in all thread-local freelists of the corresponding size class. - // REQUIRES: Static::pageheap_lock is held. - static void GetThreadStats(uint64_t* total_bytes, uint64_t* class_count); - - // Sets the total thread cache size to new_size, recomputing the - // individual thread cache sizes as necessary. - // REQUIRES: Static::pageheap lock is held. - static void set_overall_thread_cache_size(size_t new_size); - static size_t overall_thread_cache_size() { - return overall_thread_cache_size_; - } - - private: - class FreeList { - private: - void* list_; // Linked list of nodes - -#ifdef _LP64 - // On 64-bit hardware, manipulating 16-bit values may be slightly slow. - uint32_t length_; // Current length. - uint32_t lowater_; // Low water mark for list length. - uint32_t max_length_; // Dynamic max list length based on usage. - // Tracks the number of times a deallocation has caused - // length_ > max_length_. After the kMaxOverages'th time, max_length_ - // shrinks and length_overages_ is reset to zero. - uint32_t length_overages_; -#else - // If we aren't using 64-bit pointers then pack these into less space. - uint16_t length_; - uint16_t lowater_; - uint16_t max_length_; - uint16_t length_overages_; -#endif - - public: - void Init() { - list_ = NULL; - length_ = 0; - lowater_ = 0; - max_length_ = 1; - length_overages_ = 0; - } - - // Return current length of list - size_t length() const { - return length_; - } - - // Return the maximum length of the list. - size_t max_length() const { - return max_length_; - } - - // Set the maximum length of the list. If 'new_max' > length(), the - // client is responsible for removing objects from the list. - void set_max_length(size_t new_max) { - max_length_ = new_max; - } - - // Return the number of times that length() has gone over max_length(). - size_t length_overages() const { - return length_overages_; - } - - void set_length_overages(size_t new_count) { - length_overages_ = new_count; - } - - // Is list empty? - bool empty() const { - return list_ == NULL; - } - - // Low-water mark management - int lowwatermark() const { return lowater_; } - void clear_lowwatermark() { lowater_ = length_; } - - void Push(void* ptr) { - SLL_Push(&list_, ptr); - length_++; - } - - void* Pop() { - ASSERT(list_ != NULL); - length_--; - if (length_ < lowater_) lowater_ = length_; - return SLL_Pop(&list_); - } - - void* Next() { - return SLL_Next(&list_); - } - - void PushRange(int N, void *start, void *end) { - SLL_PushRange(&list_, start, end); - length_ += N; - } - - void PopRange(int N, void **start, void **end) { - SLL_PopRange(&list_, N, start, end); - ASSERT(length_ >= N); - length_ -= N; - if (length_ < lowater_) lowater_ = length_; - } - }; - - // Gets and returns an object from the central cache, and, if possible, - // also adds some objects of that size class to this thread cache. - void* FetchFromCentralCache(size_t cl, size_t byte_size); - - // Releases some number of items from src. Adjusts the list's max_length - // to eventually converge on num_objects_to_move(cl). - void ListTooLong(FreeList* src, size_t cl); - - // Releases N items from this thread cache. - void ReleaseToCentralCache(FreeList* src, size_t cl, int N); - - // Increase max_size_ by reducing unclaimed_cache_space_ or by - // reducing the max_size_ of some other thread. In both cases, - // the delta is kStealAmount. - void IncreaseCacheLimit(); - // Same as above but requires Static::pageheap_lock() is held. - void IncreaseCacheLimitLocked(); - - // If TLS is available, we also store a copy of the per-thread object - // in a __thread variable since __thread variables are faster to read - // than pthread_getspecific(). We still need pthread_setspecific() - // because __thread variables provide no way to run cleanup code when - // a thread is destroyed. - // We also give a hint to the compiler to use the "initial exec" TLS - // model. This is faster than the default TLS model, at the cost that - // you cannot dlopen this library. (To see the difference, look at - // the CPU use of __tls_get_addr with and without this attribute.) - // Since we don't really use dlopen in google code -- and using dlopen - // on a malloc replacement is asking for trouble in any case -- that's - // a good tradeoff for us. -#ifdef HAVE_TLS - struct ThreadLocalData { - ThreadCache* heap; - // min_size_for_slow_path is 0 if heap is NULL or kMaxSize + 1 otherwise. - // The latter is the common case and allows allocation to be faster - // than it would be otherwise: typically a single branch will - // determine that the requested allocation is no more than kMaxSize - // and we can then proceed, knowing that global and thread-local tcmalloc - // state is initialized. - size_t min_size_for_slow_path; - - bool use_emergency_malloc; - size_t old_min_size_for_slow_path; - }; - static __thread ThreadLocalData threadlocal_data_ ATTR_INITIAL_EXEC; -#endif - - // Thread-specific key. Initialization here is somewhat tricky - // because some Linux startup code invokes malloc() before it - // is in a good enough state to handle pthread_keycreate(). - // Therefore, we use TSD keys only after tsd_inited is set to true. - // Until then, we use a slow path to get the heap object. - static bool tsd_inited_; - static pthread_key_t heap_key_; - - // Linked list of heap objects. Protected by Static::pageheap_lock. - static ThreadCache* thread_heaps_; - static int thread_heap_count_; - - // A pointer to one of the objects in thread_heaps_. Represents - // the next ThreadCache from which a thread over its max_size_ should - // steal memory limit. Round-robin through all of the objects in - // thread_heaps_. Protected by Static::pageheap_lock. - static ThreadCache* next_memory_steal_; - - // Overall thread cache size. Protected by Static::pageheap_lock. - static size_t overall_thread_cache_size_; - - // Global per-thread cache size. Writes are protected by - // Static::pageheap_lock. Reads are done without any locking, which should be - // fine as long as size_t can be written atomically and we don't place - // invariants between this variable and other pieces of state. - static volatile size_t per_thread_cache_size_; - - // Represents overall_thread_cache_size_ minus the sum of max_size_ - // across all ThreadCaches. Protected by Static::pageheap_lock. - static ssize_t unclaimed_cache_space_; - - // This class is laid out with the most frequently used fields - // first so that hot elements are placed on the same cache line. - - size_t size_; // Combined size of data - size_t max_size_; // size_ > max_size_ --> Scavenge() - - // We sample allocations, biased by the size of the allocation - Sampler sampler_; // A sampler - - FreeList list_[kNumClasses]; // Array indexed by size-class - - pthread_t tid_; // Which thread owns it - bool in_setspecific_; // In call to pthread_setspecific? - - // Allocate a new heap. REQUIRES: Static::pageheap_lock is held. - static ThreadCache* NewHeap(pthread_t tid); - - // Use only as pthread thread-specific destructor function. - static void DestroyThreadCache(void* ptr); - - static void DeleteCache(ThreadCache* heap); - static void RecomputePerThreadCacheSize(); - - // Ensure that this class is cacheline-aligned. This is critical for - // performance, as false sharing would negate many of the benefits - // of a per-thread cache. -} CACHELINE_ALIGNED; - -// Allocator for thread heaps -// This is logically part of the ThreadCache class, but MSVC, at -// least, does not like using ThreadCache as a template argument -// before the class is fully defined. So we put it outside the class. -extern PageHeapAllocator threadcache_allocator; - -inline int ThreadCache::HeapsInUse() { - return threadcache_allocator.inuse(); -} - -inline bool ThreadCache::SampleAllocation(size_t k) { -#ifndef NO_TCMALLOC_SAMPLES - return UNLIKELY(FLAGS_tcmalloc_sample_parameter > 0) && sampler_.SampleAllocation(k); -#else - return false; -#endif -} - -inline void* ThreadCache::Allocate(size_t size, size_t cl) { - ASSERT(size <= kMaxSize); - ASSERT(size == Static::sizemap()->ByteSizeForClass(cl)); - - FreeList* list = &list_[cl]; - if (UNLIKELY(list->empty())) { - return FetchFromCentralCache(cl, size); - } - size_ -= size; - return list->Pop(); -} - -inline void ThreadCache::Deallocate(void* ptr, size_t cl) { - FreeList* list = &list_[cl]; - size_ += Static::sizemap()->ByteSizeForClass(cl); - ssize_t size_headroom = max_size_ - size_ - 1; - - // This catches back-to-back frees of allocs in the same size - // class. A more comprehensive (and expensive) test would be to walk - // the entire freelist. But this might be enough to find some bugs. - ASSERT(ptr != list->Next()); - - list->Push(ptr); - ssize_t list_headroom = - static_cast(list->max_length()) - list->length(); - - // There are two relatively uncommon things that require further work. - // In the common case we're done, and in that case we need a single branch - // because of the bitwise-or trick that follows. - if (UNLIKELY((list_headroom | size_headroom) < 0)) { - if (list_headroom < 0) { - ListTooLong(list, cl); - } - if (size_ >= max_size_) Scavenge(); - } -} - -inline ThreadCache* ThreadCache::GetThreadHeap() { -#ifdef HAVE_TLS - return threadlocal_data_.heap; -#else - return reinterpret_cast( - perftools_pthread_getspecific(heap_key_)); -#endif -} - -inline ThreadCache* ThreadCache::GetCacheWhichMustBePresent() { -#ifdef HAVE_TLS - ASSERT(threadlocal_data_.heap); - return threadlocal_data_.heap; -#else - ASSERT(perftools_pthread_getspecific(heap_key_)); - return reinterpret_cast( - perftools_pthread_getspecific(heap_key_)); -#endif -} - -inline ThreadCache* ThreadCache::GetCache() { - ThreadCache* ptr = NULL; - if (!tsd_inited_) { - InitModule(); - } else { - ptr = GetThreadHeap(); - } - if (ptr == NULL) ptr = CreateCacheIfNecessary(); - return ptr; -} - -// In deletion paths, we do not try to create a thread-cache. This is -// because we may be in the thread destruction code and may have -// already cleaned up the cache for this thread. -inline ThreadCache* ThreadCache::GetCacheIfPresent() { -#ifndef HAVE_TLS - if (!tsd_inited_) return NULL; -#endif - return GetThreadHeap(); -} - -inline size_t ThreadCache::MinSizeForSlowPath() { -#ifdef HAVE_TLS - return threadlocal_data_.min_size_for_slow_path; -#else - return 0; -#endif -} - -inline void ThreadCache::SetMinSizeForSlowPath(size_t size) { -#ifdef HAVE_TLS - threadlocal_data_.min_size_for_slow_path = size; -#endif -} - -inline void ThreadCache::SetUseEmergencyMalloc() { -#ifdef HAVE_TLS - threadlocal_data_.old_min_size_for_slow_path = threadlocal_data_.min_size_for_slow_path; - threadlocal_data_.min_size_for_slow_path = 0; - threadlocal_data_.use_emergency_malloc = true; -#endif -} - -inline void ThreadCache::ResetUseEmergencyMalloc() { -#ifdef HAVE_TLS - threadlocal_data_.min_size_for_slow_path = threadlocal_data_.old_min_size_for_slow_path; - threadlocal_data_.use_emergency_malloc = false; -#endif -} - -inline bool ThreadCache::IsUseEmergencyMalloc() { -#if defined(HAVE_TLS) && defined(ENABLE_EMERGENCY_MALLOC) - return UNLIKELY(threadlocal_data_.use_emergency_malloc); -#else - return false; -#endif -} - - -} // namespace tcmalloc - -#endif // TCMALLOC_THREAD_CACHE_H_ diff --git a/contrib/libunwind b/contrib/libunwind index 5afe6d87ae9..68cffcbbd18 160000 --- a/contrib/libunwind +++ b/contrib/libunwind @@ -1 +1 @@ -Subproject commit 5afe6d87ae9e66485c7fcb106d2f7c2c0359c8f6 +Subproject commit 68cffcbbd1840e14664a5f7f19c5e43f65c525b5 diff --git a/contrib/libunwind-cmake/CMakeLists.txt b/contrib/libunwind-cmake/CMakeLists.txt index f09d0979692..73bb0fc3beb 100644 --- a/contrib/libunwind-cmake/CMakeLists.txt +++ b/contrib/libunwind-cmake/CMakeLists.txt @@ -1,3 +1,6 @@ +include(CheckCCompilerFlag) +include(CheckCXXCompilerFlag) + set(LIBUNWIND_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libunwind) set(LIBUNWIND_CXX_SOURCES @@ -11,7 +14,9 @@ endif () set(LIBUNWIND_C_SOURCES ${LIBUNWIND_SOURCE_DIR}/src/UnwindLevel1.c ${LIBUNWIND_SOURCE_DIR}/src/UnwindLevel1-gcc-ext.c - ${LIBUNWIND_SOURCE_DIR}/src/Unwind-sjlj.c) + ${LIBUNWIND_SOURCE_DIR}/src/Unwind-sjlj.c + # Use unw_backtrace to override libgcc's backtrace symbol for better ABI compatibility + unwind-override.c) set_source_files_properties(${LIBUNWIND_C_SOURCES} PROPERTIES COMPILE_FLAGS "-std=c99") set(LIBUNWIND_ASM_SOURCES @@ -28,7 +33,22 @@ add_library(unwind ${LIBUNWIND_SOURCES}) target_include_directories(unwind SYSTEM BEFORE PUBLIC $) target_compile_definitions(unwind PRIVATE -D_LIBUNWIND_NO_HEAP=1 -D_DEBUG -D_LIBUNWIND_IS_NATIVE_ONLY) -target_compile_options(unwind PRIVATE -fno-exceptions -funwind-tables -fno-sanitize=all -nostdinc++ -fno-rtti) +target_compile_options(unwind PRIVATE -fno-exceptions -funwind-tables -fno-sanitize=all $<$:-nostdinc++ -fno-rtti>) + +check_c_compiler_flag(-Wunused-but-set-variable HAVE_WARNING_UNUSED_BUT_SET_VARIABLE) +if (HAVE_WARNING_UNUSED_BUT_SET_VARIABLE) + target_compile_options(unwind PRIVATE -Wno-unused-but-set-variable) +endif () + +check_cxx_compiler_flag(-Wmissing-attributes HAVE_WARNING_MISSING_ATTRIBUTES) +if (HAVE_WARNING_MISSING_ATTRIBUTES) + target_compile_options(unwind PRIVATE -Wno-missing-attributes) +endif () + +check_cxx_compiler_flag(-Wmaybe-uninitialized HAVE_WARNING_MAYBE_UNINITIALIZED) +if (HAVE_WARNING_MAYBE_UNINITIALIZED) + target_compile_options(unwind PRIVATE -Wno-maybe-uninitialized) +endif () install( TARGETS unwind diff --git a/contrib/libunwind-cmake/unwind-override.c b/contrib/libunwind-cmake/unwind-override.c new file mode 100644 index 00000000000..616bab6ae4b --- /dev/null +++ b/contrib/libunwind-cmake/unwind-override.c @@ -0,0 +1,6 @@ +#include + +int backtrace(void ** buffer, int size) +{ + return unw_backtrace(buffer, size); +} diff --git a/contrib/llvm b/contrib/llvm index 163def21781..778c297395b 160000 --- a/contrib/llvm +++ b/contrib/llvm @@ -1 +1 @@ -Subproject commit 163def217817c90fb982a6daf384744d8472b92b +Subproject commit 778c297395b4a2dfd60e13969a0f9488bf2c16cf diff --git a/contrib/lz4 b/contrib/lz4 index 7a4e3b1fac5..3d67671559b 160000 --- a/contrib/lz4 +++ b/contrib/lz4 @@ -1 +1 @@ -Subproject commit 7a4e3b1fac5cd9d4ec7c8d0091329ba107ec2131 +Subproject commit 3d67671559be723b0912bbee2fcd2eb14783a721 diff --git a/contrib/openssl b/contrib/openssl new file mode 160000 index 00000000000..c74e7895eb0 --- /dev/null +++ b/contrib/openssl @@ -0,0 +1 @@ +Subproject commit c74e7895eb0d219d4007775eec134dd2bcd9d1ae diff --git a/contrib/openssl-cmake/CMakeLists.txt b/contrib/openssl-cmake/CMakeLists.txt new file mode 100644 index 00000000000..c2e74dc0023 --- /dev/null +++ b/contrib/openssl-cmake/CMakeLists.txt @@ -0,0 +1,1019 @@ +set(OPENSSL_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/openssl) +set(OPENSSL_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/openssl) + +#file(READ ${CMAKE_CURRENT_SOURCE_DIR}/${OPENSSL_SOURCE_DIR}/ssl/VERSION SSL_VERSION) +#string(STRIP ${SSL_VERSION} SSL_VERSION) +#string(REPLACE ":" "." SSL_VERSION ${SSL_VERSION}) +#string(REGEX REPLACE "\\..*" "" SSL_MAJOR_VERSION ${SSL_VERSION}) + +#file(READ ${CMAKE_CURRENT_SOURCE_DIR}/${OPENSSL_SOURCE_DIR}/crypto/VERSION CRYPTO_VERSION) +#string(STRIP ${CRYPTO_VERSION} CRYPTO_VERSION) +#string(REPLACE ":" "." CRYPTO_VERSION ${CRYPTO_VERSION}) +#string(REGEX REPLACE "\\..*" "" CRYPTO_MAJOR_VERSION ${CRYPTO_VERSION}) + +set(OPENSSLDIR "/etc/ssl" CACHE PATH "Set the default openssl directory") +set(OPENSSL_ENGINESDIR "/usr/lib/engines-3" CACHE PATH "Set the default openssl directory for engines") +set(OPENSSL_MODULESDIR "/usr/local/lib/ossl-modules" CACHE PATH "Set the default openssl directory for modules") + +# How these lists were generated? +# I compiled the original OpenSSL with it's original build system and copied the list of source files from build commands. + +add_definitions(-DOPENSSLDIR="${OPENSSLDIR}" -DENGINESDIR="${OPENSSL_ENGINESDIR}" -DMODULESDIR="${OPENSSL_MODULESDIR}") + +if(ARCH_AMD64) + add_definitions(-DAES_ASM -DBSAES_ASM -DECP_NISTZ256_ASM -DGHASH_ASM -DKECCAK1600_ASM -DMD5_ASM -DOPENSSL_BN_ASM_GF2m -DOPENSSL_BN_ASM_MONT -DOPENSSL_BN_ASM_MONT5 -DOPENSSL_CPUID_OBJ -DOPENSSL_IA32_SSE2 -DPOLY1305_ASM -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DVPAES_ASM -DWHIRLPOOL_ASM -DX25519_ASM -DOPENSSL_USE_NODELETE -DL_ENDIAN) +elseif(ARCH_AARCH64) + add_definitions(-DECP_NISTZ256_ASM -DKECCAK1600_ASM -DOPENSSL_BN_ASM_MONT -DOPENSSL_CPUID_OBJ -DPOLY1305_ASM -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DVPAES_ASM -DOPENSSL_USE_NODELETE -DL_ENDIAN) +endif() + +enable_language(ASM) +if (COMPILER_CLANG) + add_definitions(-Wno-unused-command-line-argument) +endif () + +if (ARCH_AMD64) + macro(perl_generate_asm FILE_IN FILE_OUT) + add_custom_command(OUTPUT ${FILE_OUT} + COMMAND /usr/bin/env perl ${FILE_IN} ${FILE_OUT} + # ASM code has broken unwind tables (CFI), strip them. + # Otherwise asynchronous unwind (that we use for query profiler) + # will lead to segfault while trying to interpret wrong "CFA expression". + COMMAND sed -i -e '/^\.cfi_/d' ${FILE_OUT}) + endmacro() + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/aes/asm/aes-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/aes/aes-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/aes/asm/aesni-mb-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/aes/aesni-mb-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/aes/asm/aesni-sha1-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/aes/aesni-sha1-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/aes/asm/aesni-sha256-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/aes/aesni-sha256-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/aes/asm/aesni-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/aes/aesni-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/aes/asm/bsaes-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/aes/bsaes-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/aes/asm/vpaes-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/aes/vpaes-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/bn/asm/rsaz-avx2.pl ${OPENSSL_BINARY_DIR}/crypto/bn/rsaz-avx2.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/bn/asm/rsaz-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/bn/rsaz-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/bn/asm/x86_64-gf2m.pl ${OPENSSL_BINARY_DIR}/crypto/bn/x86_64-gf2m.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/bn/asm/x86_64-mont.pl ${OPENSSL_BINARY_DIR}/crypto/bn/x86_64-mont.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/bn/asm/x86_64-mont5.pl ${OPENSSL_BINARY_DIR}/crypto/bn/x86_64-mont5.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/camellia/asm/cmll-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/camellia/cmll-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/chacha/asm/chacha-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/chacha/chacha-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/ec/asm/ecp_nistz256-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/ec/ecp_nistz256-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/ec/asm/x25519-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/ec/x25519-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/x86_64cpuid.pl ${OPENSSL_BINARY_DIR}/crypto/x86_64cpuid.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/md5/asm/md5-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/md5/md5-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/modes/asm/aesni-gcm-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/modes/aesni-gcm-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/modes/asm/ghash-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/modes/ghash-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/poly1305/asm/poly1305-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/poly1305/poly1305-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/rc4/asm/rc4-md5-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/rc4/rc4-md5-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/rc4/asm/rc4-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/rc4/rc4-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/sha/asm/keccak1600-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/sha/keccak1600-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/sha/asm/sha1-mb-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/sha/sha1-mb-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/sha/asm/sha1-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/sha/sha1-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/sha/asm/sha256-mb-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/sha/sha256-mb-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/sha/asm/sha512-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/sha/sha256-x86_64.s) # This is not a mistake + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/sha/asm/sha512-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/sha/sha512-x86_64.s) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/whrlpool/asm/wp-x86_64.pl ${OPENSSL_BINARY_DIR}/crypto/whrlpool/wp-x86_64.s) +elseif (ARCH_AARCH64) + macro(perl_generate_asm FILE_IN FILE_OUT) + add_custom_command(OUTPUT ${FILE_OUT} + COMMAND /usr/bin/env perl ${FILE_IN} "linux64" ${FILE_OUT}) + # Hope that the ASM code for AArch64 doesn't have broken CFI. Otherwise, add the same sed as for x86_64. + endmacro() + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/aes/asm/aesv8-armx.pl ${OPENSSL_BINARY_DIR}/crypto/aes/aesv8-armx.S) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/aes/asm/vpaes-armv8.pl ${OPENSSL_BINARY_DIR}/crypto/aes/vpaes-armv8.S) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/bn/asm/armv8-mont.pl ${OPENSSL_BINARY_DIR}/crypto/bn/armv8-mont.S) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/chacha/asm/chacha-armv8.pl ${OPENSSL_BINARY_DIR}/crypto/chacha/chacha-armv8.S) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/ec/asm/ecp_nistz256-armv8.pl ${OPENSSL_BINARY_DIR}/crypto/ec/ecp_nistz256-armv8.S) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/arm64cpuid.pl ${OPENSSL_BINARY_DIR}/crypto/arm64cpuid.S) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/modes/asm/ghashv8-armx.pl ${OPENSSL_BINARY_DIR}/crypto/modes/ghashv8-armx.S) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/poly1305/asm/poly1305-armv8.pl ${OPENSSL_BINARY_DIR}/crypto/poly1305/poly1305-armv8.S) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/sha/asm/keccak1600-armv8.pl ${OPENSSL_BINARY_DIR}/crypto/sha/keccak1600-armv8.S) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/sha/asm/sha1-armv8.pl ${OPENSSL_BINARY_DIR}/crypto/sha/sha1-armv8.S) + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/sha/asm/sha512-armv8.pl ${OPENSSL_BINARY_DIR}/crypto/sha/sha256-armv8.S) # This is not a mistake + perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/sha/asm/sha512-armv8.pl ${OPENSSL_BINARY_DIR}/crypto/sha/sha512-armv8.S) +endif () + +set(CRYPTO_SRCS +${OPENSSL_SOURCE_DIR}/crypto/aes/aes_cfb.c +${OPENSSL_SOURCE_DIR}/crypto/aes/aes_ecb.c +${OPENSSL_SOURCE_DIR}/crypto/aes/aes_ige.c +${OPENSSL_SOURCE_DIR}/crypto/aes/aes_misc.c +${OPENSSL_SOURCE_DIR}/crypto/aes/aes_ofb.c +${OPENSSL_SOURCE_DIR}/crypto/aes/aes_wrap.c +${OPENSSL_SOURCE_DIR}/crypto/aria/aria.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/a_bitstr.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/a_d2i_fp.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/a_digest.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/a_dup.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/a_gentm.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/a_i2d_fp.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/a_int.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/a_mbstr.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/a_object.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/a_octet.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/a_print.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/a_sign.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/a_strex.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/a_strnid.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/a_time.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/a_type.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/a_utctm.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/a_utf8.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/a_verify.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/ameth_lib.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/asn1_err.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/asn1_gen.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/asn1_item_list.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/asn1_lib.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/asn1_par.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/asn_mime.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/asn_moid.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/asn_mstbl.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/asn_pack.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/bio_asn1.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/bio_ndef.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/d2i_param.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/d2i_pr.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/d2i_pu.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/evp_asn1.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/f_int.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/f_string.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/i2d_param.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/i2d_pr.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/i2d_pu.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/n_pkey.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/nsseq.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/p5_pbe.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/p5_pbev2.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/p5_scrypt.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/p8_pkey.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/t_bitst.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/t_pkey.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/t_spki.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/tasn_dec.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/tasn_enc.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/tasn_fre.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/tasn_new.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/tasn_prn.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/tasn_scn.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/tasn_typ.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/tasn_utl.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/x_algor.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/x_bignum.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/x_info.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/x_int64.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/x_long.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/x_pkey.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/x_sig.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/x_spki.c +${OPENSSL_SOURCE_DIR}/crypto/asn1/x_val.c +${OPENSSL_SOURCE_DIR}/crypto/async/arch/async_null.c +${OPENSSL_SOURCE_DIR}/crypto/async/arch/async_posix.c +${OPENSSL_SOURCE_DIR}/crypto/async/arch/async_win.c +${OPENSSL_SOURCE_DIR}/crypto/async/async.c +${OPENSSL_SOURCE_DIR}/crypto/async/async_err.c +${OPENSSL_SOURCE_DIR}/crypto/async/async_wait.c +${OPENSSL_SOURCE_DIR}/crypto/bf/bf_cfb64.c +${OPENSSL_SOURCE_DIR}/crypto/bf/bf_ecb.c +${OPENSSL_SOURCE_DIR}/crypto/bf/bf_enc.c +${OPENSSL_SOURCE_DIR}/crypto/bf/bf_ofb64.c +${OPENSSL_SOURCE_DIR}/crypto/bf/bf_skey.c +${OPENSSL_SOURCE_DIR}/crypto/bio/b_addr.c +${OPENSSL_SOURCE_DIR}/crypto/bio/b_dump.c +${OPENSSL_SOURCE_DIR}/crypto/bio/b_print.c +${OPENSSL_SOURCE_DIR}/crypto/bio/b_sock.c +${OPENSSL_SOURCE_DIR}/crypto/bio/b_sock2.c +${OPENSSL_SOURCE_DIR}/crypto/bio/bf_buff.c +${OPENSSL_SOURCE_DIR}/crypto/bio/bf_lbuf.c +${OPENSSL_SOURCE_DIR}/crypto/bio/bf_nbio.c +${OPENSSL_SOURCE_DIR}/crypto/bio/bf_null.c +${OPENSSL_SOURCE_DIR}/crypto/bio/bio_cb.c +${OPENSSL_SOURCE_DIR}/crypto/bio/bio_err.c +${OPENSSL_SOURCE_DIR}/crypto/bio/bio_lib.c +${OPENSSL_SOURCE_DIR}/crypto/bio/bio_meth.c +${OPENSSL_SOURCE_DIR}/crypto/bio/bss_acpt.c +${OPENSSL_SOURCE_DIR}/crypto/bio/bss_bio.c +${OPENSSL_SOURCE_DIR}/crypto/bio/bss_conn.c +${OPENSSL_SOURCE_DIR}/crypto/bio/bss_dgram.c +${OPENSSL_SOURCE_DIR}/crypto/bio/bss_fd.c +${OPENSSL_SOURCE_DIR}/crypto/bio/bss_file.c +${OPENSSL_SOURCE_DIR}/crypto/bio/bss_log.c +${OPENSSL_SOURCE_DIR}/crypto/bio/bss_mem.c +${OPENSSL_SOURCE_DIR}/crypto/bio/bss_null.c +${OPENSSL_SOURCE_DIR}/crypto/bio/bss_sock.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_add.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_blind.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_const.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_conv.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_ctx.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_depr.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_dh.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_div.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_err.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_exp.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_exp2.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_gcd.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_gf2m.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_intern.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_kron.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_lib.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_mod.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_mont.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_mpi.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_mul.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_nist.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_prime.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_print.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_rand.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_recp.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_rsa_fips186_4.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_shift.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_sqr.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_sqrt.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_srp.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_word.c +${OPENSSL_SOURCE_DIR}/crypto/bn/bn_x931p.c +${OPENSSL_SOURCE_DIR}/crypto/buffer/buf_err.c +${OPENSSL_SOURCE_DIR}/crypto/buffer/buffer.c +${OPENSSL_SOURCE_DIR}/crypto/camellia/cmll_cfb.c +${OPENSSL_SOURCE_DIR}/crypto/camellia/cmll_ctr.c +${OPENSSL_SOURCE_DIR}/crypto/camellia/cmll_ecb.c +${OPENSSL_SOURCE_DIR}/crypto/camellia/cmll_misc.c +${OPENSSL_SOURCE_DIR}/crypto/camellia/cmll_ofb.c +${OPENSSL_SOURCE_DIR}/crypto/cast/c_cfb64.c +${OPENSSL_SOURCE_DIR}/crypto/cast/c_ecb.c +${OPENSSL_SOURCE_DIR}/crypto/cast/c_enc.c +${OPENSSL_SOURCE_DIR}/crypto/cast/c_ofb64.c +${OPENSSL_SOURCE_DIR}/crypto/cast/c_skey.c +${OPENSSL_SOURCE_DIR}/crypto/cmac/cm_ameth.c +${OPENSSL_SOURCE_DIR}/crypto/cmac/cmac.c +${OPENSSL_SOURCE_DIR}/crypto/cmp/cmp_asn.c +${OPENSSL_SOURCE_DIR}/crypto/cmp/cmp_ctx.c +${OPENSSL_SOURCE_DIR}/crypto/cmp/cmp_err.c +${OPENSSL_SOURCE_DIR}/crypto/cmp/cmp_hdr.c +${OPENSSL_SOURCE_DIR}/crypto/cmp/cmp_msg.c +${OPENSSL_SOURCE_DIR}/crypto/cmp/cmp_protect.c +${OPENSSL_SOURCE_DIR}/crypto/cmp/cmp_status.c +${OPENSSL_SOURCE_DIR}/crypto/cmp/cmp_util.c +${OPENSSL_SOURCE_DIR}/crypto/cms/cms_asn1.c +${OPENSSL_SOURCE_DIR}/crypto/cms/cms_att.c +${OPENSSL_SOURCE_DIR}/crypto/cms/cms_cd.c +${OPENSSL_SOURCE_DIR}/crypto/cms/cms_dd.c +${OPENSSL_SOURCE_DIR}/crypto/cms/cms_enc.c +${OPENSSL_SOURCE_DIR}/crypto/cms/cms_env.c +${OPENSSL_SOURCE_DIR}/crypto/cms/cms_err.c +${OPENSSL_SOURCE_DIR}/crypto/cms/cms_ess.c +${OPENSSL_SOURCE_DIR}/crypto/cms/cms_io.c +${OPENSSL_SOURCE_DIR}/crypto/cms/cms_kari.c +${OPENSSL_SOURCE_DIR}/crypto/cms/cms_lib.c +${OPENSSL_SOURCE_DIR}/crypto/cms/cms_pwri.c +${OPENSSL_SOURCE_DIR}/crypto/cms/cms_sd.c +${OPENSSL_SOURCE_DIR}/crypto/cms/cms_smime.c +${OPENSSL_SOURCE_DIR}/crypto/comp/c_zlib.c +${OPENSSL_SOURCE_DIR}/crypto/comp/comp_err.c +${OPENSSL_SOURCE_DIR}/crypto/comp/comp_lib.c +${OPENSSL_SOURCE_DIR}/crypto/conf/conf_api.c +${OPENSSL_SOURCE_DIR}/crypto/conf/conf_def.c +${OPENSSL_SOURCE_DIR}/crypto/conf/conf_err.c +${OPENSSL_SOURCE_DIR}/crypto/conf/conf_lib.c +${OPENSSL_SOURCE_DIR}/crypto/conf/conf_mall.c +${OPENSSL_SOURCE_DIR}/crypto/conf/conf_mod.c +${OPENSSL_SOURCE_DIR}/crypto/conf/conf_sap.c +${OPENSSL_SOURCE_DIR}/crypto/conf/conf_ssl.c +${OPENSSL_SOURCE_DIR}/crypto/crmf/crmf_asn.c +${OPENSSL_SOURCE_DIR}/crypto/crmf/crmf_err.c +${OPENSSL_SOURCE_DIR}/crypto/crmf/crmf_lib.c +${OPENSSL_SOURCE_DIR}/crypto/crmf/crmf_pbm.c +${OPENSSL_SOURCE_DIR}/crypto/ct/ct_b64.c +${OPENSSL_SOURCE_DIR}/crypto/ct/ct_err.c +${OPENSSL_SOURCE_DIR}/crypto/ct/ct_log.c +${OPENSSL_SOURCE_DIR}/crypto/ct/ct_oct.c +${OPENSSL_SOURCE_DIR}/crypto/ct/ct_policy.c +${OPENSSL_SOURCE_DIR}/crypto/ct/ct_prn.c +${OPENSSL_SOURCE_DIR}/crypto/ct/ct_sct.c +${OPENSSL_SOURCE_DIR}/crypto/ct/ct_sct_ctx.c +${OPENSSL_SOURCE_DIR}/crypto/ct/ct_vfy.c +${OPENSSL_SOURCE_DIR}/crypto/ct/ct_x509v3.c +${OPENSSL_SOURCE_DIR}/crypto/des/cbc_cksm.c +${OPENSSL_SOURCE_DIR}/crypto/des/cbc_enc.c +${OPENSSL_SOURCE_DIR}/crypto/des/cfb64ede.c +${OPENSSL_SOURCE_DIR}/crypto/des/cfb64enc.c +${OPENSSL_SOURCE_DIR}/crypto/des/cfb_enc.c +${OPENSSL_SOURCE_DIR}/crypto/des/des_enc.c +${OPENSSL_SOURCE_DIR}/crypto/des/ecb3_enc.c +${OPENSSL_SOURCE_DIR}/crypto/des/ecb_enc.c +${OPENSSL_SOURCE_DIR}/crypto/des/fcrypt.c +${OPENSSL_SOURCE_DIR}/crypto/des/fcrypt_b.c +${OPENSSL_SOURCE_DIR}/crypto/des/ofb64ede.c +${OPENSSL_SOURCE_DIR}/crypto/des/ofb64enc.c +${OPENSSL_SOURCE_DIR}/crypto/des/ofb_enc.c +${OPENSSL_SOURCE_DIR}/crypto/des/pcbc_enc.c +${OPENSSL_SOURCE_DIR}/crypto/des/qud_cksm.c +${OPENSSL_SOURCE_DIR}/crypto/des/rand_key.c +${OPENSSL_SOURCE_DIR}/crypto/des/set_key.c +${OPENSSL_SOURCE_DIR}/crypto/des/str2key.c +${OPENSSL_SOURCE_DIR}/crypto/des/xcbc_enc.c +${OPENSSL_SOURCE_DIR}/crypto/dh/dh_ameth.c +${OPENSSL_SOURCE_DIR}/crypto/dh/dh_asn1.c +${OPENSSL_SOURCE_DIR}/crypto/dh/dh_check.c +${OPENSSL_SOURCE_DIR}/crypto/dh/dh_depr.c +${OPENSSL_SOURCE_DIR}/crypto/dh/dh_err.c +${OPENSSL_SOURCE_DIR}/crypto/dh/dh_gen.c +${OPENSSL_SOURCE_DIR}/crypto/dh/dh_kdf.c +${OPENSSL_SOURCE_DIR}/crypto/dh/dh_key.c +${OPENSSL_SOURCE_DIR}/crypto/dh/dh_lib.c +${OPENSSL_SOURCE_DIR}/crypto/dh/dh_meth.c +${OPENSSL_SOURCE_DIR}/crypto/dh/dh_pmeth.c +${OPENSSL_SOURCE_DIR}/crypto/dh/dh_prn.c +${OPENSSL_SOURCE_DIR}/crypto/dh/dh_rfc5114.c +${OPENSSL_SOURCE_DIR}/crypto/dh/dh_rfc7919.c +${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_ameth.c +${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_asn1.c +${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_depr.c +${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_err.c +${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_gen.c +${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_key.c +${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_lib.c +${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_meth.c +${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_ossl.c +${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_pmeth.c +${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_prn.c +${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_sign.c +${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_vrf.c +${OPENSSL_SOURCE_DIR}/crypto/dso/dso_dl.c +${OPENSSL_SOURCE_DIR}/crypto/dso/dso_dlfcn.c +${OPENSSL_SOURCE_DIR}/crypto/dso/dso_err.c +${OPENSSL_SOURCE_DIR}/crypto/dso/dso_lib.c +${OPENSSL_SOURCE_DIR}/crypto/dso/dso_openssl.c +${OPENSSL_SOURCE_DIR}/crypto/dso/dso_vms.c +${OPENSSL_SOURCE_DIR}/crypto/dso/dso_win32.c +${OPENSSL_SOURCE_DIR}/crypto/ec/curve448/arch_32/f_impl.c +${OPENSSL_SOURCE_DIR}/crypto/ec/curve448/curve448.c +${OPENSSL_SOURCE_DIR}/crypto/ec/curve448/curve448_tables.c +${OPENSSL_SOURCE_DIR}/crypto/ec/curve448/eddsa.c +${OPENSSL_SOURCE_DIR}/crypto/ec/curve448/f_generic.c +${OPENSSL_SOURCE_DIR}/crypto/ec/curve448/scalar.c +${OPENSSL_SOURCE_DIR}/crypto/ec/curve25519.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ec2_oct.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ec2_smpl.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ec_ameth.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ec_asn1.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ec_check.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ec_curve.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ec_cvt.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ec_err.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ec_key.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ec_kmeth.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ec_lib.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ec_mult.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ec_oct.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ec_pmeth.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ec_print.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ecdh_kdf.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ecdh_ossl.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ecdsa_ossl.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ecdsa_sign.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ecdsa_vrf.c +${OPENSSL_SOURCE_DIR}/crypto/ec/eck_prn.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ecp_mont.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ecp_nist.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ecp_nistp224.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ecp_nistp256.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ecp_nistp521.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ecp_nistputil.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ecp_nistz256.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ecp_oct.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ecp_smpl.c +${OPENSSL_SOURCE_DIR}/crypto/ec/ecx_meth.c +${OPENSSL_SOURCE_DIR}/crypto/engine/eng_all.c +${OPENSSL_SOURCE_DIR}/crypto/engine/eng_cnf.c +${OPENSSL_SOURCE_DIR}/crypto/engine/eng_ctrl.c +${OPENSSL_SOURCE_DIR}/crypto/engine/eng_dyn.c +${OPENSSL_SOURCE_DIR}/crypto/engine/eng_err.c +${OPENSSL_SOURCE_DIR}/crypto/engine/eng_fat.c +${OPENSSL_SOURCE_DIR}/crypto/engine/eng_init.c +${OPENSSL_SOURCE_DIR}/crypto/engine/eng_lib.c +${OPENSSL_SOURCE_DIR}/crypto/engine/eng_list.c +${OPENSSL_SOURCE_DIR}/crypto/engine/eng_openssl.c +${OPENSSL_SOURCE_DIR}/crypto/engine/eng_pkey.c +${OPENSSL_SOURCE_DIR}/crypto/engine/eng_rdrand.c +${OPENSSL_SOURCE_DIR}/crypto/engine/eng_table.c +${OPENSSL_SOURCE_DIR}/crypto/engine/tb_asnmth.c +${OPENSSL_SOURCE_DIR}/crypto/engine/tb_cipher.c +${OPENSSL_SOURCE_DIR}/crypto/engine/tb_dh.c +${OPENSSL_SOURCE_DIR}/crypto/engine/tb_digest.c +${OPENSSL_SOURCE_DIR}/crypto/engine/tb_dsa.c +${OPENSSL_SOURCE_DIR}/crypto/engine/tb_eckey.c +${OPENSSL_SOURCE_DIR}/crypto/engine/tb_pkmeth.c +${OPENSSL_SOURCE_DIR}/crypto/engine/tb_rand.c +${OPENSSL_SOURCE_DIR}/crypto/engine/tb_rsa.c +${OPENSSL_SOURCE_DIR}/crypto/err/err.c +${OPENSSL_SOURCE_DIR}/crypto/err/err_all.c +${OPENSSL_SOURCE_DIR}/crypto/err/err_blocks.c +${OPENSSL_SOURCE_DIR}/crypto/err/err_prn.c +${OPENSSL_SOURCE_DIR}/crypto/ess/ess_asn1.c +${OPENSSL_SOURCE_DIR}/crypto/ess/ess_err.c +${OPENSSL_SOURCE_DIR}/crypto/ess/ess_lib.c +${OPENSSL_SOURCE_DIR}/crypto/evp/bio_b64.c +${OPENSSL_SOURCE_DIR}/crypto/evp/bio_enc.c +${OPENSSL_SOURCE_DIR}/crypto/evp/bio_md.c +${OPENSSL_SOURCE_DIR}/crypto/evp/bio_ok.c +${OPENSSL_SOURCE_DIR}/crypto/evp/c_allc.c +${OPENSSL_SOURCE_DIR}/crypto/evp/c_alld.c +${OPENSSL_SOURCE_DIR}/crypto/evp/cmeth_lib.c +${OPENSSL_SOURCE_DIR}/crypto/evp/digest.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_aes.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_aes_cbc_hmac_sha1.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_aes_cbc_hmac_sha256.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_aria.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_bf.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_camellia.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_cast.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_chacha20_poly1305.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_des.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_des3.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_idea.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_null.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_old.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_rc2.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_rc4.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_rc4_hmac_md5.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_rc5.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_seed.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_sm4.c +${OPENSSL_SOURCE_DIR}/crypto/evp/e_xcbc_d.c +${OPENSSL_SOURCE_DIR}/crypto/evp/encode.c +${OPENSSL_SOURCE_DIR}/crypto/evp/evp_cnf.c +${OPENSSL_SOURCE_DIR}/crypto/evp/evp_enc.c +${OPENSSL_SOURCE_DIR}/crypto/evp/evp_err.c +${OPENSSL_SOURCE_DIR}/crypto/evp/evp_fetch.c +${OPENSSL_SOURCE_DIR}/crypto/evp/evp_key.c +${OPENSSL_SOURCE_DIR}/crypto/evp/evp_lib.c +${OPENSSL_SOURCE_DIR}/crypto/evp/evp_pbe.c +${OPENSSL_SOURCE_DIR}/crypto/evp/evp_pkey.c +${OPENSSL_SOURCE_DIR}/crypto/evp/evp_utils.c +${OPENSSL_SOURCE_DIR}/crypto/evp/exchange.c +${OPENSSL_SOURCE_DIR}/crypto/evp/kdf_lib.c +${OPENSSL_SOURCE_DIR}/crypto/evp/kdf_meth.c +${OPENSSL_SOURCE_DIR}/crypto/evp/keymgmt_lib.c +${OPENSSL_SOURCE_DIR}/crypto/evp/keymgmt_meth.c +${OPENSSL_SOURCE_DIR}/crypto/evp/legacy_blake2.c +${OPENSSL_SOURCE_DIR}/crypto/evp/legacy_md4.c +${OPENSSL_SOURCE_DIR}/crypto/evp/legacy_md5.c +${OPENSSL_SOURCE_DIR}/crypto/evp/legacy_md5_sha1.c +${OPENSSL_SOURCE_DIR}/crypto/evp/legacy_mdc2.c +${OPENSSL_SOURCE_DIR}/crypto/evp/legacy_sha.c +${OPENSSL_SOURCE_DIR}/crypto/evp/m_null.c +${OPENSSL_SOURCE_DIR}/crypto/evp/m_ripemd.c +${OPENSSL_SOURCE_DIR}/crypto/evp/m_sigver.c +${OPENSSL_SOURCE_DIR}/crypto/evp/m_wp.c +${OPENSSL_SOURCE_DIR}/crypto/evp/mac_lib.c +${OPENSSL_SOURCE_DIR}/crypto/evp/mac_meth.c +${OPENSSL_SOURCE_DIR}/crypto/evp/names.c +${OPENSSL_SOURCE_DIR}/crypto/evp/p5_crpt.c +${OPENSSL_SOURCE_DIR}/crypto/evp/p5_crpt2.c +${OPENSSL_SOURCE_DIR}/crypto/evp/p_dec.c +${OPENSSL_SOURCE_DIR}/crypto/evp/p_enc.c +${OPENSSL_SOURCE_DIR}/crypto/evp/p_lib.c +${OPENSSL_SOURCE_DIR}/crypto/evp/p_open.c +${OPENSSL_SOURCE_DIR}/crypto/evp/p_seal.c +${OPENSSL_SOURCE_DIR}/crypto/evp/p_sign.c +${OPENSSL_SOURCE_DIR}/crypto/evp/p_verify.c +${OPENSSL_SOURCE_DIR}/crypto/evp/pbe_scrypt.c +${OPENSSL_SOURCE_DIR}/crypto/evp/pkey_kdf.c +${OPENSSL_SOURCE_DIR}/crypto/evp/pkey_mac.c +${OPENSSL_SOURCE_DIR}/crypto/evp/pmeth_fn.c +${OPENSSL_SOURCE_DIR}/crypto/evp/pmeth_gn.c +${OPENSSL_SOURCE_DIR}/crypto/evp/pmeth_lib.c +${OPENSSL_SOURCE_DIR}/crypto/hmac/hm_ameth.c +${OPENSSL_SOURCE_DIR}/crypto/hmac/hmac.c +${OPENSSL_SOURCE_DIR}/crypto/idea/i_cbc.c +${OPENSSL_SOURCE_DIR}/crypto/idea/i_cfb64.c +${OPENSSL_SOURCE_DIR}/crypto/idea/i_ecb.c +${OPENSSL_SOURCE_DIR}/crypto/idea/i_ofb64.c +${OPENSSL_SOURCE_DIR}/crypto/idea/i_skey.c +${OPENSSL_SOURCE_DIR}/crypto/kdf/kdf_err.c +${OPENSSL_SOURCE_DIR}/crypto/lhash/lh_stats.c +${OPENSSL_SOURCE_DIR}/crypto/lhash/lhash.c +${OPENSSL_SOURCE_DIR}/crypto/asn1_dsa.c +${OPENSSL_SOURCE_DIR}/crypto/bsearch.c +${OPENSSL_SOURCE_DIR}/crypto/context.c +${OPENSSL_SOURCE_DIR}/crypto/core_algorithm.c +${OPENSSL_SOURCE_DIR}/crypto/core_fetch.c +${OPENSSL_SOURCE_DIR}/crypto/core_namemap.c +${OPENSSL_SOURCE_DIR}/crypto/cpt_err.c +${OPENSSL_SOURCE_DIR}/crypto/cryptlib.c +${OPENSSL_SOURCE_DIR}/crypto/ctype.c +${OPENSSL_SOURCE_DIR}/crypto/cversion.c +${OPENSSL_SOURCE_DIR}/crypto/ebcdic.c +${OPENSSL_SOURCE_DIR}/crypto/ex_data.c +${OPENSSL_SOURCE_DIR}/crypto/getenv.c +${OPENSSL_SOURCE_DIR}/crypto/info.c +${OPENSSL_SOURCE_DIR}/crypto/init.c +${OPENSSL_SOURCE_DIR}/crypto/initthread.c +${OPENSSL_SOURCE_DIR}/crypto/mem.c +${OPENSSL_SOURCE_DIR}/crypto/mem_sec.c +${OPENSSL_SOURCE_DIR}/crypto/o_dir.c +${OPENSSL_SOURCE_DIR}/crypto/o_fips.c +${OPENSSL_SOURCE_DIR}/crypto/o_fopen.c +${OPENSSL_SOURCE_DIR}/crypto/o_init.c +${OPENSSL_SOURCE_DIR}/crypto/o_str.c +${OPENSSL_SOURCE_DIR}/crypto/o_time.c +${OPENSSL_SOURCE_DIR}/crypto/packet.c +${OPENSSL_SOURCE_DIR}/crypto/param_build.c +${OPENSSL_SOURCE_DIR}/crypto/params.c +${OPENSSL_SOURCE_DIR}/crypto/params_from_text.c +${OPENSSL_SOURCE_DIR}/crypto/provider.c +${OPENSSL_SOURCE_DIR}/crypto/provider_conf.c +${OPENSSL_SOURCE_DIR}/crypto/provider_core.c +${OPENSSL_SOURCE_DIR}/crypto/provider_predefined.c +${OPENSSL_SOURCE_DIR}/crypto/sparse_array.c +${OPENSSL_SOURCE_DIR}/crypto/threads_none.c +${OPENSSL_SOURCE_DIR}/crypto/threads_pthread.c +${OPENSSL_SOURCE_DIR}/crypto/threads_win.c +${OPENSSL_SOURCE_DIR}/crypto/trace.c +${OPENSSL_SOURCE_DIR}/crypto/uid.c +${OPENSSL_SOURCE_DIR}/crypto/md4/md4_dgst.c +${OPENSSL_SOURCE_DIR}/crypto/md4/md4_one.c +${OPENSSL_SOURCE_DIR}/crypto/md5/md5_dgst.c +${OPENSSL_SOURCE_DIR}/crypto/md5/md5_one.c +${OPENSSL_SOURCE_DIR}/crypto/md5/md5_sha1.c +${OPENSSL_SOURCE_DIR}/crypto/mdc2/mdc2_one.c +${OPENSSL_SOURCE_DIR}/crypto/mdc2/mdc2dgst.c +${OPENSSL_SOURCE_DIR}/crypto/modes/cbc128.c +${OPENSSL_SOURCE_DIR}/crypto/modes/ccm128.c +${OPENSSL_SOURCE_DIR}/crypto/modes/cfb128.c +${OPENSSL_SOURCE_DIR}/crypto/modes/ctr128.c +${OPENSSL_SOURCE_DIR}/crypto/modes/cts128.c +${OPENSSL_SOURCE_DIR}/crypto/modes/gcm128.c +${OPENSSL_SOURCE_DIR}/crypto/modes/ocb128.c +${OPENSSL_SOURCE_DIR}/crypto/modes/ofb128.c +${OPENSSL_SOURCE_DIR}/crypto/modes/siv128.c +${OPENSSL_SOURCE_DIR}/crypto/modes/wrap128.c +${OPENSSL_SOURCE_DIR}/crypto/modes/xts128.c +${OPENSSL_SOURCE_DIR}/crypto/objects/o_names.c +${OPENSSL_SOURCE_DIR}/crypto/objects/obj_dat.c +${OPENSSL_SOURCE_DIR}/crypto/objects/obj_err.c +${OPENSSL_SOURCE_DIR}/crypto/objects/obj_lib.c +${OPENSSL_SOURCE_DIR}/crypto/objects/obj_xref.c +${OPENSSL_SOURCE_DIR}/crypto/ocsp/ocsp_asn.c +${OPENSSL_SOURCE_DIR}/crypto/ocsp/ocsp_cl.c +${OPENSSL_SOURCE_DIR}/crypto/ocsp/ocsp_err.c +${OPENSSL_SOURCE_DIR}/crypto/ocsp/ocsp_ext.c +${OPENSSL_SOURCE_DIR}/crypto/ocsp/ocsp_ht.c +${OPENSSL_SOURCE_DIR}/crypto/ocsp/ocsp_lib.c +${OPENSSL_SOURCE_DIR}/crypto/ocsp/ocsp_prn.c +${OPENSSL_SOURCE_DIR}/crypto/ocsp/ocsp_srv.c +${OPENSSL_SOURCE_DIR}/crypto/ocsp/ocsp_vfy.c +${OPENSSL_SOURCE_DIR}/crypto/ocsp/v3_ocsp.c +${OPENSSL_SOURCE_DIR}/crypto/pem/pem_all.c +${OPENSSL_SOURCE_DIR}/crypto/pem/pem_err.c +${OPENSSL_SOURCE_DIR}/crypto/pem/pem_info.c +${OPENSSL_SOURCE_DIR}/crypto/pem/pem_lib.c +${OPENSSL_SOURCE_DIR}/crypto/pem/pem_oth.c +${OPENSSL_SOURCE_DIR}/crypto/pem/pem_pk8.c +${OPENSSL_SOURCE_DIR}/crypto/pem/pem_pkey.c +${OPENSSL_SOURCE_DIR}/crypto/pem/pem_sign.c +${OPENSSL_SOURCE_DIR}/crypto/pem/pem_x509.c +${OPENSSL_SOURCE_DIR}/crypto/pem/pem_xaux.c +${OPENSSL_SOURCE_DIR}/crypto/pem/pvkfmt.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs12/p12_add.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs12/p12_asn.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs12/p12_attr.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs12/p12_crpt.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs12/p12_crt.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs12/p12_decr.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs12/p12_init.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs12/p12_key.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs12/p12_kiss.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs12/p12_mutl.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs12/p12_npas.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs12/p12_p8d.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs12/p12_p8e.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs12/p12_sbag.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs12/p12_utl.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs12/pk12err.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs7/bio_pk7.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs7/pk7_asn1.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs7/pk7_attr.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs7/pk7_doit.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs7/pk7_lib.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs7/pk7_mime.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs7/pk7_smime.c +${OPENSSL_SOURCE_DIR}/crypto/pkcs7/pkcs7err.c +${OPENSSL_SOURCE_DIR}/crypto/poly1305/poly1305.c +${OPENSSL_SOURCE_DIR}/crypto/poly1305/poly1305_ameth.c +${OPENSSL_SOURCE_DIR}/crypto/property/defn_cache.c +${OPENSSL_SOURCE_DIR}/crypto/property/property.c +${OPENSSL_SOURCE_DIR}/crypto/property/property_err.c +${OPENSSL_SOURCE_DIR}/crypto/property/property_parse.c +${OPENSSL_SOURCE_DIR}/crypto/property/property_string.c +${OPENSSL_SOURCE_DIR}/crypto/rand/drbg_ctr.c +${OPENSSL_SOURCE_DIR}/crypto/rand/drbg_hash.c +${OPENSSL_SOURCE_DIR}/crypto/rand/drbg_hmac.c +${OPENSSL_SOURCE_DIR}/crypto/rand/drbg_lib.c +${OPENSSL_SOURCE_DIR}/crypto/rand/rand_crng_test.c +${OPENSSL_SOURCE_DIR}/crypto/rand/rand_egd.c +${OPENSSL_SOURCE_DIR}/crypto/rand/rand_err.c +${OPENSSL_SOURCE_DIR}/crypto/rand/rand_lib.c +${OPENSSL_SOURCE_DIR}/crypto/rand/rand_unix.c +${OPENSSL_SOURCE_DIR}/crypto/rand/rand_vms.c +${OPENSSL_SOURCE_DIR}/crypto/rand/rand_vxworks.c +${OPENSSL_SOURCE_DIR}/crypto/rand/rand_win.c +${OPENSSL_SOURCE_DIR}/crypto/rand/randfile.c +${OPENSSL_SOURCE_DIR}/crypto/rc2/rc2_cbc.c +${OPENSSL_SOURCE_DIR}/crypto/rc2/rc2_ecb.c +${OPENSSL_SOURCE_DIR}/crypto/rc2/rc2_skey.c +${OPENSSL_SOURCE_DIR}/crypto/rc2/rc2cfb64.c +${OPENSSL_SOURCE_DIR}/crypto/rc2/rc2ofb64.c +${OPENSSL_SOURCE_DIR}/crypto/ripemd/rmd_dgst.c +${OPENSSL_SOURCE_DIR}/crypto/ripemd/rmd_one.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_ameth.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_asn1.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_chk.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_crpt.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_depr.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_err.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_gen.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_lib.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_meth.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_mp.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_none.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_oaep.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_ossl.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_pk1.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_pmeth.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_prn.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_pss.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_saos.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_sign.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_sp800_56b_check.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_sp800_56b_gen.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_ssl.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_x931.c +${OPENSSL_SOURCE_DIR}/crypto/rsa/rsa_x931g.c +${OPENSSL_SOURCE_DIR}/crypto/seed/seed.c +${OPENSSL_SOURCE_DIR}/crypto/seed/seed_cbc.c +${OPENSSL_SOURCE_DIR}/crypto/seed/seed_cfb.c +${OPENSSL_SOURCE_DIR}/crypto/seed/seed_ecb.c +${OPENSSL_SOURCE_DIR}/crypto/seed/seed_ofb.c +${OPENSSL_SOURCE_DIR}/crypto/serializer/serializer_err.c +${OPENSSL_SOURCE_DIR}/crypto/serializer/serializer_lib.c +${OPENSSL_SOURCE_DIR}/crypto/serializer/serializer_meth.c +${OPENSSL_SOURCE_DIR}/crypto/serializer/serializer_pkey.c +${OPENSSL_SOURCE_DIR}/crypto/sha/sha1_one.c +${OPENSSL_SOURCE_DIR}/crypto/sha/sha1dgst.c +${OPENSSL_SOURCE_DIR}/crypto/sha/sha256.c +${OPENSSL_SOURCE_DIR}/crypto/sha/sha3.c +${OPENSSL_SOURCE_DIR}/crypto/sha/sha512.c +${OPENSSL_SOURCE_DIR}/crypto/siphash/siphash.c +${OPENSSL_SOURCE_DIR}/crypto/siphash/siphash_ameth.c +${OPENSSL_SOURCE_DIR}/crypto/sm2/sm2_crypt.c +${OPENSSL_SOURCE_DIR}/crypto/sm2/sm2_err.c +${OPENSSL_SOURCE_DIR}/crypto/sm2/sm2_pmeth.c +${OPENSSL_SOURCE_DIR}/crypto/sm2/sm2_sign.c +${OPENSSL_SOURCE_DIR}/crypto/sm3/m_sm3.c +${OPENSSL_SOURCE_DIR}/crypto/sm3/sm3.c +${OPENSSL_SOURCE_DIR}/crypto/sm4/sm4.c +${OPENSSL_SOURCE_DIR}/crypto/srp/srp_lib.c +${OPENSSL_SOURCE_DIR}/crypto/srp/srp_vfy.c +${OPENSSL_SOURCE_DIR}/crypto/stack/stack.c +${OPENSSL_SOURCE_DIR}/crypto/store/loader_file.c +${OPENSSL_SOURCE_DIR}/crypto/store/store_err.c +${OPENSSL_SOURCE_DIR}/crypto/store/store_init.c +${OPENSSL_SOURCE_DIR}/crypto/store/store_lib.c +${OPENSSL_SOURCE_DIR}/crypto/store/store_register.c +${OPENSSL_SOURCE_DIR}/crypto/store/store_strings.c +${OPENSSL_SOURCE_DIR}/crypto/ts/ts_asn1.c +${OPENSSL_SOURCE_DIR}/crypto/ts/ts_conf.c +${OPENSSL_SOURCE_DIR}/crypto/ts/ts_err.c +${OPENSSL_SOURCE_DIR}/crypto/ts/ts_lib.c +${OPENSSL_SOURCE_DIR}/crypto/ts/ts_req_print.c +${OPENSSL_SOURCE_DIR}/crypto/ts/ts_req_utils.c +${OPENSSL_SOURCE_DIR}/crypto/ts/ts_rsp_print.c +${OPENSSL_SOURCE_DIR}/crypto/ts/ts_rsp_sign.c +${OPENSSL_SOURCE_DIR}/crypto/ts/ts_rsp_utils.c +${OPENSSL_SOURCE_DIR}/crypto/ts/ts_rsp_verify.c +${OPENSSL_SOURCE_DIR}/crypto/ts/ts_verify_ctx.c +${OPENSSL_SOURCE_DIR}/crypto/txt_db/txt_db.c +${OPENSSL_SOURCE_DIR}/crypto/ui/ui_err.c +${OPENSSL_SOURCE_DIR}/crypto/ui/ui_lib.c +${OPENSSL_SOURCE_DIR}/crypto/ui/ui_null.c +${OPENSSL_SOURCE_DIR}/crypto/ui/ui_openssl.c +${OPENSSL_SOURCE_DIR}/crypto/ui/ui_util.c +${OPENSSL_SOURCE_DIR}/crypto/whrlpool/wp_dgst.c +${OPENSSL_SOURCE_DIR}/crypto/x509/by_dir.c +${OPENSSL_SOURCE_DIR}/crypto/x509/by_file.c +${OPENSSL_SOURCE_DIR}/crypto/x509/by_store.c +${OPENSSL_SOURCE_DIR}/crypto/x509/pcy_cache.c +${OPENSSL_SOURCE_DIR}/crypto/x509/pcy_data.c +${OPENSSL_SOURCE_DIR}/crypto/x509/pcy_lib.c +${OPENSSL_SOURCE_DIR}/crypto/x509/pcy_map.c +${OPENSSL_SOURCE_DIR}/crypto/x509/pcy_node.c +${OPENSSL_SOURCE_DIR}/crypto/x509/pcy_tree.c +${OPENSSL_SOURCE_DIR}/crypto/x509/t_crl.c +${OPENSSL_SOURCE_DIR}/crypto/x509/t_req.c +${OPENSSL_SOURCE_DIR}/crypto/x509/t_x509.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_addr.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_admis.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_akey.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_akeya.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_alt.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_asid.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_bcons.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_bitst.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_conf.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_cpols.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_crld.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_enum.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_extku.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_genn.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_ia5.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_info.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_int.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_lib.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_ncons.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_pci.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_pcia.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_pcons.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_pku.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_pmaps.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_prn.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_purp.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_skey.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_sxnet.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_tlsf.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3_utl.c +${OPENSSL_SOURCE_DIR}/crypto/x509/v3err.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509_att.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509_cmp.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509_d2.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509_def.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509_err.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509_ext.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509_lu.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509_meth.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509_obj.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509_r2x.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509_req.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509_set.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509_trs.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509_txt.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509_v3.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509_vfy.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509_vpm.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509cset.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509name.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509rset.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509spki.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x509type.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x_all.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x_attrib.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x_crl.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x_exten.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x_name.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x_pubkey.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x_req.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x_x509.c +${OPENSSL_SOURCE_DIR}/crypto/x509/x_x509a.c +${OPENSSL_SOURCE_DIR}/providers/implementations/asymciphers/rsa_enc.c +${OPENSSL_SOURCE_DIR}/providers/defltprov.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_ccm.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_ccm_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_gcm.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_gcm_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_ocb.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_ocb_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_siv.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_siv_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_wrp.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_xts.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_xts_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aria.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aria_ccm.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aria_ccm_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aria_gcm.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aria_gcm_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aria_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_blowfish.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_blowfish_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_camellia.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_camellia_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_cast5.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_cast5_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_chacha20.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_chacha20_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_chacha20_poly1305.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_chacha20_poly1305_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_des.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_des_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_desx.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_desx_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_idea.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_idea_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_rc2.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_rc2_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_rc4.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_rc4_hmac_md5.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_rc4_hmac_md5_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_rc4_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_seed.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_seed_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_sm4.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_sm4_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_tdes.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_tdes_default.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_tdes_default_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_tdes_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_tdes_wrap.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_tdes_wrap_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/digests/blake2_prov.c +${OPENSSL_SOURCE_DIR}/providers/implementations/digests/blake2b_prov.c +${OPENSSL_SOURCE_DIR}/providers/implementations/digests/blake2s_prov.c +${OPENSSL_SOURCE_DIR}/providers/implementations/digests/md5_prov.c +${OPENSSL_SOURCE_DIR}/providers/implementations/digests/md5_sha1_prov.c +${OPENSSL_SOURCE_DIR}/providers/implementations/digests/sha2_prov.c +${OPENSSL_SOURCE_DIR}/providers/implementations/digests/sha3_prov.c +${OPENSSL_SOURCE_DIR}/providers/implementations/digests/sm3_prov.c +${OPENSSL_SOURCE_DIR}/providers/implementations/exchange/dh_exch.c +${OPENSSL_SOURCE_DIR}/providers/implementations/kdfs/hkdf.c +${OPENSSL_SOURCE_DIR}/providers/implementations/kdfs/kbkdf.c +${OPENSSL_SOURCE_DIR}/providers/implementations/kdfs/krb5kdf.c +${OPENSSL_SOURCE_DIR}/providers/implementations/kdfs/pbkdf2.c +${OPENSSL_SOURCE_DIR}/providers/implementations/kdfs/scrypt.c +${OPENSSL_SOURCE_DIR}/providers/implementations/kdfs/sshkdf.c +${OPENSSL_SOURCE_DIR}/providers/implementations/kdfs/sskdf.c +${OPENSSL_SOURCE_DIR}/providers/implementations/kdfs/tls1_prf.c +${OPENSSL_SOURCE_DIR}/providers/implementations/kdfs/x942kdf.c +${OPENSSL_SOURCE_DIR}/providers/implementations/keymgmt/dh_kmgmt.c +${OPENSSL_SOURCE_DIR}/providers/implementations/keymgmt/dsa_kmgmt.c +${OPENSSL_SOURCE_DIR}/providers/implementations/keymgmt/rsa_kmgmt.c +${OPENSSL_SOURCE_DIR}/providers/implementations/macs/blake2b_mac.c +${OPENSSL_SOURCE_DIR}/providers/implementations/macs/blake2s_mac.c +${OPENSSL_SOURCE_DIR}/providers/implementations/macs/cmac_prov.c +${OPENSSL_SOURCE_DIR}/providers/implementations/macs/gmac_prov.c +${OPENSSL_SOURCE_DIR}/providers/implementations/macs/hmac_prov.c +${OPENSSL_SOURCE_DIR}/providers/implementations/macs/kmac_prov.c +${OPENSSL_SOURCE_DIR}/providers/implementations/macs/poly1305_prov.c +${OPENSSL_SOURCE_DIR}/providers/implementations/macs/siphash_prov.c +${OPENSSL_SOURCE_DIR}/providers/implementations/serializers/serializer_common.c +${OPENSSL_SOURCE_DIR}/providers/implementations/serializers/serializer_dh.c +${OPENSSL_SOURCE_DIR}/providers/implementations/serializers/serializer_dh_param.c +${OPENSSL_SOURCE_DIR}/providers/implementations/serializers/serializer_dh_priv.c +${OPENSSL_SOURCE_DIR}/providers/implementations/serializers/serializer_dh_pub.c +${OPENSSL_SOURCE_DIR}/providers/implementations/serializers/serializer_dsa.c +${OPENSSL_SOURCE_DIR}/providers/implementations/serializers/serializer_dsa_param.c +${OPENSSL_SOURCE_DIR}/providers/implementations/serializers/serializer_dsa_priv.c +${OPENSSL_SOURCE_DIR}/providers/implementations/serializers/serializer_dsa_pub.c +${OPENSSL_SOURCE_DIR}/providers/implementations/serializers/serializer_rsa.c +${OPENSSL_SOURCE_DIR}/providers/implementations/serializers/serializer_rsa_priv.c +${OPENSSL_SOURCE_DIR}/providers/implementations/serializers/serializer_rsa_pub.c +${OPENSSL_SOURCE_DIR}/providers/implementations/signature/dsa.c +${OPENSSL_SOURCE_DIR}/providers/common/bio_prov.c +${OPENSSL_SOURCE_DIR}/providers/common/provider_err.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/ciphercommon.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/ciphercommon_block.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/ciphercommon_ccm.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/ciphercommon_ccm_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/ciphercommon_gcm.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/ciphercommon_gcm_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/ciphercommon_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/digests/digestcommon.c +${OPENSSL_SOURCE_DIR}/providers/common/nid_to_name.c +${OPENSSL_SOURCE_DIR}/providers/common/provider_util.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_xts_fips.c +${OPENSSL_SOURCE_DIR}/providers/implementations/kdfs/pbkdf2_fips.c +) + +if (ARCH_AMD64) + set (CRYPTO_SRCS ${CRYPTO_SRCS} + ${OPENSSL_SOURCE_DIR}/crypto/bn/asm/x86_64-gcc.c + ${OPENSSL_SOURCE_DIR}/crypto/bn/rsaz_exp.c + ${OPENSSL_BINARY_DIR}/crypto/aes/aes-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/aes/aesni-mb-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/aes/aesni-sha1-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/aes/aesni-sha256-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/aes/aesni-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/aes/bsaes-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/aes/vpaes-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/bn/rsaz-avx2.s + ${OPENSSL_BINARY_DIR}/crypto/bn/rsaz-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/bn/x86_64-gf2m.s + ${OPENSSL_BINARY_DIR}/crypto/bn/x86_64-mont.s + ${OPENSSL_BINARY_DIR}/crypto/bn/x86_64-mont5.s + ${OPENSSL_BINARY_DIR}/crypto/camellia/cmll-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/chacha/chacha-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/ec/ecp_nistz256-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/ec/x25519-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/x86_64cpuid.s + ${OPENSSL_BINARY_DIR}/crypto/md5/md5-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/modes/aesni-gcm-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/modes/ghash-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/poly1305/poly1305-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/rc4/rc4-md5-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/rc4/rc4-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/sha/keccak1600-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/sha/sha1-mb-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/sha/sha1-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/sha/sha256-mb-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/sha/sha256-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/sha/sha512-x86_64.s + ${OPENSSL_BINARY_DIR}/crypto/whrlpool/wp-x86_64.s) +elseif (ARCH_AARCH64) + set (CRYPTO_SRCS ${CRYPTO_SRCS} + ${OPENSSL_SOURCE_DIR}/crypto/aes/aes_cbc.c + ${OPENSSL_SOURCE_DIR}/crypto/aes/aes_core.c + ${OPENSSL_SOURCE_DIR}/crypto/bn/bn_asm.c + ${OPENSSL_SOURCE_DIR}/crypto/camellia/camellia.c + ${OPENSSL_SOURCE_DIR}/crypto/camellia/cmll_cbc.c + ${OPENSSL_SOURCE_DIR}/crypto/armcap.c + ${OPENSSL_SOURCE_DIR}/crypto/rc4/rc4_enc.c + ${OPENSSL_SOURCE_DIR}/crypto/rc4/rc4_skey.c + ${OPENSSL_SOURCE_DIR}/crypto/whrlpool/wp_block.c + ${OPENSSL_BINARY_DIR}/crypto/aes/aesv8-armx.S + ${OPENSSL_BINARY_DIR}/crypto/aes/vpaes-armv8.S + ${OPENSSL_BINARY_DIR}/crypto/bn/armv8-mont.S + ${OPENSSL_BINARY_DIR}/crypto/chacha/chacha-armv8.S + ${OPENSSL_BINARY_DIR}/crypto/ec/ecp_nistz256-armv8.S + ${OPENSSL_BINARY_DIR}/crypto/arm64cpuid.S + ${OPENSSL_BINARY_DIR}/crypto/modes/ghashv8-armx.S + ${OPENSSL_BINARY_DIR}/crypto/poly1305/poly1305-armv8.S + ${OPENSSL_BINARY_DIR}/crypto/sha/keccak1600-armv8.S + ${OPENSSL_BINARY_DIR}/crypto/sha/sha1-armv8.S + ${OPENSSL_BINARY_DIR}/crypto/sha/sha256-armv8.S + ${OPENSSL_BINARY_DIR}/crypto/sha/sha512-armv8.S) +endif () + +set(SSL_SRCS +${OPENSSL_SOURCE_DIR}/crypto/packet.c +${OPENSSL_SOURCE_DIR}/ssl/bio_ssl.c +${OPENSSL_SOURCE_DIR}/ssl/d1_lib.c +${OPENSSL_SOURCE_DIR}/ssl/d1_msg.c +${OPENSSL_SOURCE_DIR}/ssl/d1_srtp.c +${OPENSSL_SOURCE_DIR}/ssl/methods.c +${OPENSSL_SOURCE_DIR}/ssl/pqueue.c +${OPENSSL_SOURCE_DIR}/ssl/s3_cbc.c +${OPENSSL_SOURCE_DIR}/ssl/s3_enc.c +${OPENSSL_SOURCE_DIR}/ssl/s3_lib.c +${OPENSSL_SOURCE_DIR}/ssl/s3_msg.c +${OPENSSL_SOURCE_DIR}/ssl/ssl_asn1.c +${OPENSSL_SOURCE_DIR}/ssl/ssl_cert.c +${OPENSSL_SOURCE_DIR}/ssl/ssl_ciph.c +${OPENSSL_SOURCE_DIR}/ssl/ssl_conf.c +${OPENSSL_SOURCE_DIR}/ssl/ssl_err.c +${OPENSSL_SOURCE_DIR}/ssl/ssl_init.c +${OPENSSL_SOURCE_DIR}/ssl/ssl_lib.c +${OPENSSL_SOURCE_DIR}/ssl/ssl_mcnf.c +${OPENSSL_SOURCE_DIR}/ssl/ssl_rsa.c +${OPENSSL_SOURCE_DIR}/ssl/ssl_sess.c +${OPENSSL_SOURCE_DIR}/ssl/ssl_stat.c +${OPENSSL_SOURCE_DIR}/ssl/ssl_txt.c +${OPENSSL_SOURCE_DIR}/ssl/ssl_utst.c +${OPENSSL_SOURCE_DIR}/ssl/t1_enc.c +${OPENSSL_SOURCE_DIR}/ssl/t1_lib.c +${OPENSSL_SOURCE_DIR}/ssl/t1_trce.c +${OPENSSL_SOURCE_DIR}/ssl/tls13_enc.c +${OPENSSL_SOURCE_DIR}/ssl/tls_srp.c +${OPENSSL_SOURCE_DIR}/ssl/record/dtls1_bitmap.c +${OPENSSL_SOURCE_DIR}/ssl/record/rec_layer_d1.c +${OPENSSL_SOURCE_DIR}/ssl/record/rec_layer_s3.c +${OPENSSL_SOURCE_DIR}/ssl/record/ssl3_buffer.c +${OPENSSL_SOURCE_DIR}/ssl/record/ssl3_record.c +${OPENSSL_SOURCE_DIR}/ssl/record/ssl3_record_tls13.c +${OPENSSL_SOURCE_DIR}/ssl/statem/extensions.c +${OPENSSL_SOURCE_DIR}/ssl/statem/extensions_clnt.c +${OPENSSL_SOURCE_DIR}/ssl/statem/extensions_cust.c +${OPENSSL_SOURCE_DIR}/ssl/statem/extensions_srvr.c +${OPENSSL_SOURCE_DIR}/ssl/statem/statem.c +${OPENSSL_SOURCE_DIR}/ssl/statem/statem_clnt.c +${OPENSSL_SOURCE_DIR}/ssl/statem/statem_dtls.c +${OPENSSL_SOURCE_DIR}/ssl/statem/statem_lib.c +${OPENSSL_SOURCE_DIR}/ssl/statem/statem_srvr.c +) + +add_library(crypto ${CRYPTO_SRCS}) +add_library(ssl ${SSL_SRCS}) + +if (ARCH_AMD64) + target_include_directories(crypto + SYSTEM PUBLIC linux_x86_64/include + PRIVATE linux_x86_64/include_private) +elseif (ARCH_AARCH64) + target_include_directories(crypto + SYSTEM PUBLIC linux_aarch64/include + PRIVATE linux_aarch64/include_private) +endif () + +target_include_directories(crypto +SYSTEM PUBLIC ${OPENSSL_SOURCE_DIR}/include +PRIVATE ${OPENSSL_SOURCE_DIR}/providers/common/include +PRIVATE ${OPENSSL_SOURCE_DIR}/providers/implementations/include +PRIVATE ${OPENSSL_SOURCE_DIR}/crypto +PRIVATE ${OPENSSL_SOURCE_DIR}/crypto/include +PRIVATE ${OPENSSL_SOURCE_DIR}/crypto/ec/curve448 +PRIVATE ${OPENSSL_SOURCE_DIR}/crypto/ec/curve448/arch_32 +PRIVATE ${OPENSSL_SOURCE_DIR}) + +target_include_directories(ssl +PRIVATE ${OPENSSL_SOURCE_DIR}) + +target_link_libraries(ssl crypto) diff --git a/contrib/openssl-cmake/linux_aarch64/include/crypto/bn_conf.h b/contrib/openssl-cmake/linux_aarch64/include/crypto/bn_conf.h new file mode 100644 index 00000000000..bb8d515212c --- /dev/null +++ b/contrib/openssl-cmake/linux_aarch64/include/crypto/bn_conf.h @@ -0,0 +1,28 @@ +/* WARNING: do not edit! */ +/* Generated by Makefile from /home/milovidov/work/ClickHouse/contrib/openssl/include/crypto/bn_conf.h.in */ +/* + * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#ifndef OSSL_CRYPTO_BN_CONF_H +# define OSSL_CRYPTO_BN_CONF_H + +/* + * The contents of this file are not used in the UEFI build, as + * both 32-bit and 64-bit builds are supported from a single run + * of the Configure script. + */ + +/* Should we define BN_DIV2W here? */ + +/* Only one for the following should be defined */ +#define SIXTY_FOUR_BIT_LONG +#undef SIXTY_FOUR_BIT +#undef THIRTY_TWO_BIT + +#endif diff --git a/contrib/openssl-cmake/linux_aarch64/include/crypto/dso_conf.h b/contrib/openssl-cmake/linux_aarch64/include/crypto/dso_conf.h new file mode 100644 index 00000000000..fe09bd26706 --- /dev/null +++ b/contrib/openssl-cmake/linux_aarch64/include/crypto/dso_conf.h @@ -0,0 +1,17 @@ +/* WARNING: do not edit! */ +/* Generated by Makefile from /home/milovidov/work/ClickHouse/contrib/openssl/include/crypto/dso_conf.h.in */ +/* + * Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#ifndef OSSL_CRYPTO_DSO_CONF_H +# define OSSL_CRYPTO_DSO_CONF_H +# define DSO_DLFCN +# define HAVE_DLFCN_H +# define DSO_EXTENSION ".so" +#endif diff --git a/contrib/openssl-cmake/linux_aarch64/include/openssl/configuration.h b/contrib/openssl-cmake/linux_aarch64/include/openssl/configuration.h new file mode 100644 index 00000000000..ecc580369f2 --- /dev/null +++ b/contrib/openssl-cmake/linux_aarch64/include/openssl/configuration.h @@ -0,0 +1,132 @@ +/* + * WARNING: do not edit! + * Generated by Makefile from include/openssl/configuration.h.in + * + * Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#ifndef OPENSSL_CONFIGURATION_H +# define OPENSSL_CONFIGURATION_H + +# ifdef __cplusplus +extern "C" { +# endif + +# ifdef OPENSSL_ALGORITHM_DEFINES +# error OPENSSL_ALGORITHM_DEFINES no longer supported +# endif + +/* + * OpenSSL was configured with the following options: + */ + +# define OPENSSL_CONFIGURED_API 30000 + +/// This fragment was edited to avoid dependency on "getrandom" function that is not available on old libc and old Linux kernels. +/// The DEVRANDOM method is also good. + +//# ifndef OPENSSL_RAND_SEED_OS +//# define OPENSSL_RAND_SEED_OS +//# endif +#define OPENSSL_RAND_SEED_DEVRANDOM + +# ifndef OPENSSL_THREADS +# define OPENSSL_THREADS +# endif +# ifndef OPENSSL_NO_ASAN +# define OPENSSL_NO_ASAN +# endif +# ifndef OPENSSL_NO_CRYPTO_MDEBUG +# define OPENSSL_NO_CRYPTO_MDEBUG +# endif +# ifndef OPENSSL_NO_CRYPTO_MDEBUG_BACKTRACE +# define OPENSSL_NO_CRYPTO_MDEBUG_BACKTRACE +# endif +# ifndef OPENSSL_NO_DEVCRYPTOENG +# define OPENSSL_NO_DEVCRYPTOENG +# endif +# ifndef OPENSSL_NO_EC_NISTP_64_GCC_128 +# define OPENSSL_NO_EC_NISTP_64_GCC_128 +# endif +# ifndef OPENSSL_NO_EGD +# define OPENSSL_NO_EGD +# endif +# ifndef OPENSSL_NO_EXTERNAL_TESTS +# define OPENSSL_NO_EXTERNAL_TESTS +# endif +# ifndef OPENSSL_NO_FUZZ_AFL +# define OPENSSL_NO_FUZZ_AFL +# endif +# ifndef OPENSSL_NO_FUZZ_LIBFUZZER +# define OPENSSL_NO_FUZZ_LIBFUZZER +# endif +# ifndef OPENSSL_NO_KTLS +# define OPENSSL_NO_KTLS +# endif +# ifndef OPENSSL_NO_MD2 +# define OPENSSL_NO_MD2 +# endif +# ifndef OPENSSL_NO_MSAN +# define OPENSSL_NO_MSAN +# endif +# ifndef OPENSSL_NO_RC5 +# define OPENSSL_NO_RC5 +# endif +# ifndef OPENSSL_NO_SCTP +# define OPENSSL_NO_SCTP +# endif +# ifndef OPENSSL_NO_SSL_TRACE +# define OPENSSL_NO_SSL_TRACE +# endif +# ifndef OPENSSL_NO_SSL3 +# define OPENSSL_NO_SSL3 +# endif +# ifndef OPENSSL_NO_SSL3_METHOD +# define OPENSSL_NO_SSL3_METHOD +# endif +# ifndef OPENSSL_NO_TRACE +# define OPENSSL_NO_TRACE +# endif +# ifndef OPENSSL_NO_UBSAN +# define OPENSSL_NO_UBSAN +# endif +# ifndef OPENSSL_NO_UNIT_TEST +# define OPENSSL_NO_UNIT_TEST +# endif +# ifndef OPENSSL_NO_UPLINK +# define OPENSSL_NO_UPLINK +# endif +# ifndef OPENSSL_NO_WEAK_SSL_CIPHERS +# define OPENSSL_NO_WEAK_SSL_CIPHERS +# endif +# ifndef OPENSSL_NO_STATIC_ENGINE +# define OPENSSL_NO_STATIC_ENGINE +# endif + + +/* Generate 80386 code? */ +# undef I386_ONLY + +/* + * The following are cipher-specific, but are part of the public API. + */ +# if !defined(OPENSSL_SYS_UEFI) +# undef BN_LLONG +/* Only one for the following should be defined */ +# define SIXTY_FOUR_BIT_LONG +# undef SIXTY_FOUR_BIT +# undef THIRTY_TWO_BIT +# endif + +# define RC4_INT unsigned char + +# ifdef __cplusplus +} +# endif + +#endif /* OPENSSL_CONFIGURATION_H */ diff --git a/contrib/openssl-cmake/linux_aarch64/include/openssl/opensslv.h b/contrib/openssl-cmake/linux_aarch64/include/openssl/opensslv.h new file mode 100644 index 00000000000..216bfbf924b --- /dev/null +++ b/contrib/openssl-cmake/linux_aarch64/include/openssl/opensslv.h @@ -0,0 +1,114 @@ +/* + * WARNING: do not edit! + * Generated by Makefile from /home/milovidov/work/ClickHouse/contrib/openssl/include/openssl/opensslv.h.in + * + * Copyright 1999-2019 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#ifndef OPENSSL_OPENSSLV_H +# define OPENSSL_OPENSSLV_H +# pragma once + +# ifdef __cplusplus +extern "C" { +# endif + +/* + * SECTION 1: VERSION DATA. These will change for each release + */ + +/* + * Base version macros + * + * These macros express version number MAJOR.MINOR.PATCH exactly + */ +# define OPENSSL_VERSION_MAJOR 3 +# define OPENSSL_VERSION_MINOR 0 +# define OPENSSL_VERSION_PATCH 0 + +/* + * Additional version information + * + * These are also part of the new version scheme, but aren't part + * of the version number itself. + */ + +/* Could be: #define OPENSSL_VERSION_PRE_RELEASE "-alpha.1" */ +# define OPENSSL_VERSION_PRE_RELEASE "-dev" +/* Could be: #define OPENSSL_VERSION_BUILD_METADATA "+fips" */ +/* Could be: #define OPENSSL_VERSION_BUILD_METADATA "+vendor.1" */ +# define OPENSSL_VERSION_BUILD_METADATA "" + +/* + * Note: The OpenSSL Project will never define OPENSSL_VERSION_BUILD_METADATA + * to be anything but the empty string. Its use is entirely reserved for + * others + */ + +/* + * Shared library version + * + * This is strictly to express ABI version, which may or may not + * be related to the API version expressed with the macros above. + * This is defined in free form. + */ +# define OPENSSL_SHLIB_VERSION 3 + +/* + * SECTION 2: USEFUL MACROS + */ + +/* For checking general API compatibility when preprocessing */ +# define OPENSSL_VERSION_PREREQ(maj,min) \ + ((OPENSSL_VERSION_MAJOR << 16) + OPENSSL_VERSION_MINOR >= ((maj) << 16) + (min)) + +/* + * Macros to get the version in easily digested string form, both the short + * "MAJOR.MINOR.PATCH" variant (where MAJOR, MINOR and PATCH are replaced + * with the values from the corresponding OPENSSL_VERSION_ macros) and the + * longer variant with OPENSSL_VERSION_PRE_RELEASE_STR and + * OPENSSL_VERSION_BUILD_METADATA_STR appended. + */ +# define OPENSSL_VERSION_STR "3.0.0" +# define OPENSSL_FULL_VERSION_STR "3.0.0-dev" + +/* + * SECTION 3: ADDITIONAL METADATA + * + * These strings are defined separately to allow them to be parsable. + */ +# define OPENSSL_RELEASE_DATE "xx XXX xxxx" + +/* + * SECTION 4: BACKWARD COMPATIBILITY + */ + +# define OPENSSL_VERSION_TEXT "OpenSSL 3.0.0-dev xx XXX xxxx" + +/* Synthesize OPENSSL_VERSION_NUMBER with the layout 0xMNN00PPSL */ +# ifdef OPENSSL_VERSION_PRE_RELEASE +# define _OPENSSL_VERSION_PRE_RELEASE 0x0 +# else +# define _OPENSSL_VERSION_PRE_RELEASE 0xf +# endif +# define OPENSSL_VERSION_NUMBER \ + ( (OPENSSL_VERSION_MAJOR<<28) \ + |(OPENSSL_VERSION_MINOR<<20) \ + |(OPENSSL_VERSION_PATCH<<4) \ + |_OPENSSL_VERSION_PRE_RELEASE ) + +# ifdef __cplusplus +} +# endif + +# include +# ifndef OPENSSL_NO_DEPRECATED_3_0 +# define HEADER_OPENSSLV_H +# endif + +#endif /* OPENSSL_OPENSSLV_H */ diff --git a/contrib/openssl-cmake/linux_aarch64/include_private/buildinf.h b/contrib/openssl-cmake/linux_aarch64/include_private/buildinf.h new file mode 100644 index 00000000000..e33d9778c21 --- /dev/null +++ b/contrib/openssl-cmake/linux_aarch64/include_private/buildinf.h @@ -0,0 +1,23 @@ +/* + * WARNING: do not edit! + * Generated by util/mkbuildinf.pl + * + * Copyright 2014-2017 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#define PLATFORM "platform: linux-aarch64" +#define DATE "built on: Sun Dec 15 01:54:14 2019 UTC" + +/* + * Generate compiler_flags as an array of individual characters. This is a + * workaround for the situation where CFLAGS gets too long for a C90 string + * literal + */ + +/// Edited because I don't care. +static const char compiler_flags[] = ""; diff --git a/contrib/openssl-cmake/linux_x86_64/include/crypto/bn_conf.h b/contrib/openssl-cmake/linux_x86_64/include/crypto/bn_conf.h new file mode 100644 index 00000000000..bb8d515212c --- /dev/null +++ b/contrib/openssl-cmake/linux_x86_64/include/crypto/bn_conf.h @@ -0,0 +1,28 @@ +/* WARNING: do not edit! */ +/* Generated by Makefile from /home/milovidov/work/ClickHouse/contrib/openssl/include/crypto/bn_conf.h.in */ +/* + * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#ifndef OSSL_CRYPTO_BN_CONF_H +# define OSSL_CRYPTO_BN_CONF_H + +/* + * The contents of this file are not used in the UEFI build, as + * both 32-bit and 64-bit builds are supported from a single run + * of the Configure script. + */ + +/* Should we define BN_DIV2W here? */ + +/* Only one for the following should be defined */ +#define SIXTY_FOUR_BIT_LONG +#undef SIXTY_FOUR_BIT +#undef THIRTY_TWO_BIT + +#endif diff --git a/contrib/openssl-cmake/linux_x86_64/include/crypto/dso_conf.h b/contrib/openssl-cmake/linux_x86_64/include/crypto/dso_conf.h new file mode 100644 index 00000000000..fe09bd26706 --- /dev/null +++ b/contrib/openssl-cmake/linux_x86_64/include/crypto/dso_conf.h @@ -0,0 +1,17 @@ +/* WARNING: do not edit! */ +/* Generated by Makefile from /home/milovidov/work/ClickHouse/contrib/openssl/include/crypto/dso_conf.h.in */ +/* + * Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#ifndef OSSL_CRYPTO_DSO_CONF_H +# define OSSL_CRYPTO_DSO_CONF_H +# define DSO_DLFCN +# define HAVE_DLFCN_H +# define DSO_EXTENSION ".so" +#endif diff --git a/contrib/openssl-cmake/linux_x86_64/include/openssl/configuration.h b/contrib/openssl-cmake/linux_x86_64/include/openssl/configuration.h new file mode 100644 index 00000000000..b46970bf2da --- /dev/null +++ b/contrib/openssl-cmake/linux_x86_64/include/openssl/configuration.h @@ -0,0 +1,132 @@ +/* + * WARNING: do not edit! + * Generated by Makefile from /home/milovidov/work/ClickHouse/contrib/openssl/include/openssl/configuration.h.in + * + * Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#ifndef OPENSSL_CONFIGURATION_H +# define OPENSSL_CONFIGURATION_H + +# ifdef __cplusplus +extern "C" { +# endif + +# ifdef OPENSSL_ALGORITHM_DEFINES +# error OPENSSL_ALGORITHM_DEFINES no longer supported +# endif + +/* + * OpenSSL was configured with the following options: + */ + +# define OPENSSL_CONFIGURED_API 30000 + +/// This fragment was edited to avoid dependency on "getrandom" function that is not available on old libc and old Linux kernels. +/// The DEVRANDOM method is also good. + +//# ifndef OPENSSL_RAND_SEED_OS +//# define OPENSSL_RAND_SEED_OS +//# endif +#define OPENSSL_RAND_SEED_DEVRANDOM + +# ifndef OPENSSL_THREADS +# define OPENSSL_THREADS +# endif +# ifndef OPENSSL_NO_ASAN +# define OPENSSL_NO_ASAN +# endif +# ifndef OPENSSL_NO_CRYPTO_MDEBUG +# define OPENSSL_NO_CRYPTO_MDEBUG +# endif +# ifndef OPENSSL_NO_CRYPTO_MDEBUG_BACKTRACE +# define OPENSSL_NO_CRYPTO_MDEBUG_BACKTRACE +# endif +# ifndef OPENSSL_NO_DEVCRYPTOENG +# define OPENSSL_NO_DEVCRYPTOENG +# endif +# ifndef OPENSSL_NO_EC_NISTP_64_GCC_128 +# define OPENSSL_NO_EC_NISTP_64_GCC_128 +# endif +# ifndef OPENSSL_NO_EGD +# define OPENSSL_NO_EGD +# endif +# ifndef OPENSSL_NO_EXTERNAL_TESTS +# define OPENSSL_NO_EXTERNAL_TESTS +# endif +# ifndef OPENSSL_NO_FUZZ_AFL +# define OPENSSL_NO_FUZZ_AFL +# endif +# ifndef OPENSSL_NO_FUZZ_LIBFUZZER +# define OPENSSL_NO_FUZZ_LIBFUZZER +# endif +# ifndef OPENSSL_NO_KTLS +# define OPENSSL_NO_KTLS +# endif +# ifndef OPENSSL_NO_MD2 +# define OPENSSL_NO_MD2 +# endif +# ifndef OPENSSL_NO_MSAN +# define OPENSSL_NO_MSAN +# endif +# ifndef OPENSSL_NO_RC5 +# define OPENSSL_NO_RC5 +# endif +# ifndef OPENSSL_NO_SCTP +# define OPENSSL_NO_SCTP +# endif +# ifndef OPENSSL_NO_SSL_TRACE +# define OPENSSL_NO_SSL_TRACE +# endif +# ifndef OPENSSL_NO_SSL3 +# define OPENSSL_NO_SSL3 +# endif +# ifndef OPENSSL_NO_SSL3_METHOD +# define OPENSSL_NO_SSL3_METHOD +# endif +# ifndef OPENSSL_NO_TRACE +# define OPENSSL_NO_TRACE +# endif +# ifndef OPENSSL_NO_UBSAN +# define OPENSSL_NO_UBSAN +# endif +# ifndef OPENSSL_NO_UNIT_TEST +# define OPENSSL_NO_UNIT_TEST +# endif +# ifndef OPENSSL_NO_UPLINK +# define OPENSSL_NO_UPLINK +# endif +# ifndef OPENSSL_NO_WEAK_SSL_CIPHERS +# define OPENSSL_NO_WEAK_SSL_CIPHERS +# endif +# ifndef OPENSSL_NO_STATIC_ENGINE +# define OPENSSL_NO_STATIC_ENGINE +# endif + + +/* Generate 80386 code? */ +# undef I386_ONLY + +/* + * The following are cipher-specific, but are part of the public API. + */ +# if !defined(OPENSSL_SYS_UEFI) +# undef BN_LLONG +/* Only one for the following should be defined */ +# define SIXTY_FOUR_BIT_LONG +# undef SIXTY_FOUR_BIT +# undef THIRTY_TWO_BIT +# endif + +# define RC4_INT unsigned int + +# ifdef __cplusplus +} +# endif + +#endif /* OPENSSL_CONFIGURATION_H */ diff --git a/contrib/openssl-cmake/linux_x86_64/include/openssl/opensslv.h b/contrib/openssl-cmake/linux_x86_64/include/openssl/opensslv.h new file mode 100644 index 00000000000..216bfbf924b --- /dev/null +++ b/contrib/openssl-cmake/linux_x86_64/include/openssl/opensslv.h @@ -0,0 +1,114 @@ +/* + * WARNING: do not edit! + * Generated by Makefile from /home/milovidov/work/ClickHouse/contrib/openssl/include/openssl/opensslv.h.in + * + * Copyright 1999-2019 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#ifndef OPENSSL_OPENSSLV_H +# define OPENSSL_OPENSSLV_H +# pragma once + +# ifdef __cplusplus +extern "C" { +# endif + +/* + * SECTION 1: VERSION DATA. These will change for each release + */ + +/* + * Base version macros + * + * These macros express version number MAJOR.MINOR.PATCH exactly + */ +# define OPENSSL_VERSION_MAJOR 3 +# define OPENSSL_VERSION_MINOR 0 +# define OPENSSL_VERSION_PATCH 0 + +/* + * Additional version information + * + * These are also part of the new version scheme, but aren't part + * of the version number itself. + */ + +/* Could be: #define OPENSSL_VERSION_PRE_RELEASE "-alpha.1" */ +# define OPENSSL_VERSION_PRE_RELEASE "-dev" +/* Could be: #define OPENSSL_VERSION_BUILD_METADATA "+fips" */ +/* Could be: #define OPENSSL_VERSION_BUILD_METADATA "+vendor.1" */ +# define OPENSSL_VERSION_BUILD_METADATA "" + +/* + * Note: The OpenSSL Project will never define OPENSSL_VERSION_BUILD_METADATA + * to be anything but the empty string. Its use is entirely reserved for + * others + */ + +/* + * Shared library version + * + * This is strictly to express ABI version, which may or may not + * be related to the API version expressed with the macros above. + * This is defined in free form. + */ +# define OPENSSL_SHLIB_VERSION 3 + +/* + * SECTION 2: USEFUL MACROS + */ + +/* For checking general API compatibility when preprocessing */ +# define OPENSSL_VERSION_PREREQ(maj,min) \ + ((OPENSSL_VERSION_MAJOR << 16) + OPENSSL_VERSION_MINOR >= ((maj) << 16) + (min)) + +/* + * Macros to get the version in easily digested string form, both the short + * "MAJOR.MINOR.PATCH" variant (where MAJOR, MINOR and PATCH are replaced + * with the values from the corresponding OPENSSL_VERSION_ macros) and the + * longer variant with OPENSSL_VERSION_PRE_RELEASE_STR and + * OPENSSL_VERSION_BUILD_METADATA_STR appended. + */ +# define OPENSSL_VERSION_STR "3.0.0" +# define OPENSSL_FULL_VERSION_STR "3.0.0-dev" + +/* + * SECTION 3: ADDITIONAL METADATA + * + * These strings are defined separately to allow them to be parsable. + */ +# define OPENSSL_RELEASE_DATE "xx XXX xxxx" + +/* + * SECTION 4: BACKWARD COMPATIBILITY + */ + +# define OPENSSL_VERSION_TEXT "OpenSSL 3.0.0-dev xx XXX xxxx" + +/* Synthesize OPENSSL_VERSION_NUMBER with the layout 0xMNN00PPSL */ +# ifdef OPENSSL_VERSION_PRE_RELEASE +# define _OPENSSL_VERSION_PRE_RELEASE 0x0 +# else +# define _OPENSSL_VERSION_PRE_RELEASE 0xf +# endif +# define OPENSSL_VERSION_NUMBER \ + ( (OPENSSL_VERSION_MAJOR<<28) \ + |(OPENSSL_VERSION_MINOR<<20) \ + |(OPENSSL_VERSION_PATCH<<4) \ + |_OPENSSL_VERSION_PRE_RELEASE ) + +# ifdef __cplusplus +} +# endif + +# include +# ifndef OPENSSL_NO_DEPRECATED_3_0 +# define HEADER_OPENSSLV_H +# endif + +#endif /* OPENSSL_OPENSSLV_H */ diff --git a/contrib/openssl-cmake/linux_x86_64/include_private/buildinf.h b/contrib/openssl-cmake/linux_x86_64/include_private/buildinf.h new file mode 100644 index 00000000000..2eb54e5618c --- /dev/null +++ b/contrib/openssl-cmake/linux_x86_64/include_private/buildinf.h @@ -0,0 +1,23 @@ +/* + * WARNING: do not edit! + * Generated by util/mkbuildinf.pl + * + * Copyright 2014-2017 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#define PLATFORM "platform: linux-x86_64" +#define DATE "built on: Sat Dec 14 00:28:08 2019 UTC" + +/* + * Generate compiler_flags as an array of individual characters. This is a + * workaround for the situation where CFLAGS gets too long for a C90 string + * literal + */ + +/// Edited because I don't care. +static const char compiler_flags[] = ""; diff --git a/contrib/orc-cmake/CMakeLists.txt b/contrib/orc-cmake/CMakeLists.txt deleted file mode 100644 index 066ba00aede..00000000000 --- a/contrib/orc-cmake/CMakeLists.txt +++ /dev/null @@ -1,229 +0,0 @@ -# modifyed copy of contrib/orc/c++/src/CMakeLists.txt -set(LIBRARY_INCLUDE ${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include) -set(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/orc/c++/src) - -set(PROTOBUF_INCLUDE_DIR ${Protobuf_INCLUDE_DIR}) -set(PROTOBUF_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE}) - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX11_FLAGS} ${WARN_FLAGS}") - -INCLUDE(CheckCXXSourceCompiles) - -CHECK_CXX_SOURCE_COMPILES(" - #include - #include - int main(int,char*[]){ - int f = open(\"/x/y\", O_RDONLY); - char buf[100]; - return pread(f, buf, 100, 1000) == 0; - }" - HAS_PREAD -) - -CHECK_CXX_SOURCE_COMPILES(" - #include - int main(int,char*[]){ - struct tm time2020; - return !strptime(\"2020-02-02 12:34:56\", \"%Y-%m-%d %H:%M:%S\", &time2020); - }" - HAS_STRPTIME -) - -CHECK_CXX_SOURCE_COMPILES(" - #include - int main(int,char* argv[]){ - return static_cast(std::stoll(argv[0])); - }" - HAS_STOLL -) - -CHECK_CXX_SOURCE_COMPILES(" - #include - #include - int main(int,char*[]){ - int64_t x = 1; printf(\"%lld\",x); - }" - INT64_IS_LL -) - -CHECK_CXX_SOURCE_COMPILES(" - #ifdef __clang__ - #pragma clang diagnostic push - #pragma clang diagnostic ignored \"-Wdeprecated\" - #pragma clang diagnostic pop - #elif defined(__GNUC__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored \"-Wdeprecated\" - #pragma GCC diagnostic pop - #elif defined(_MSC_VER) - #pragma warning( push ) - #pragma warning( disable : 4996 ) - #pragma warning( pop ) - #else - unknownCompiler! - #endif - int main(int, char *[]) {}" - HAS_DIAGNOSTIC_PUSH -) - -CHECK_CXX_SOURCE_COMPILES(" - #include - int main(int, char *[]) { - return std::isnan(1.0f); - }" - HAS_STD_ISNAN -) - -CHECK_CXX_SOURCE_COMPILES(" - #include - int main(int, char *[]) { - std::mutex test_mutex; - std::lock_guard lock_mutex(test_mutex); - }" - HAS_STD_MUTEX -) - -CHECK_CXX_SOURCE_COMPILES(" - #include - std::string func() { - std::string var = \"test\"; - return std::move(var); - } - int main(int, char *[]) {}" - NEEDS_REDUNDANT_MOVE -) - -INCLUDE(CheckCXXSourceRuns) - -CHECK_CXX_SOURCE_RUNS(" - #include - int main(int, char *[]) { - time_t t = -14210715; // 1969-07-20 12:34:45 - struct tm *ptm = gmtime(&t); - return !(ptm && ptm->tm_year == 69); - }" - HAS_PRE_1970 -) - -CHECK_CXX_SOURCE_RUNS(" - #include - #include - int main(int, char *[]) { - setenv(\"TZ\", \"America/Los_Angeles\", 1); - tzset(); - struct tm time2037; - struct tm time2038; - strptime(\"2037-05-05 12:34:56\", \"%Y-%m-%d %H:%M:%S\", &time2037); - strptime(\"2038-05-05 12:34:56\", \"%Y-%m-%d %H:%M:%S\", &time2038); - return mktime(&time2038) - mktime(&time2037) != 31536000; - }" - HAS_POST_2038 -) - -set(CMAKE_REQUIRED_INCLUDES ${ZLIB_INCLUDE_DIR}) -set(CMAKE_REQUIRED_LIBRARIES zlib) -CHECK_CXX_SOURCE_COMPILES(" - #define Z_PREFIX - #include - z_stream strm; - int main(int, char *[]) { - deflateReset(&strm); - }" - NEEDS_Z_PREFIX -) - -configure_file ( - "${LIBRARY_DIR}/Adaptor.hh.in" - "${CMAKE_CURRENT_BINARY_DIR}/Adaptor.hh" - ) - - -add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/orc_proto.pb.h ${CMAKE_CURRENT_BINARY_DIR}/orc_proto.pb.cc - COMMAND ${PROTOBUF_EXECUTABLE} - -I${ClickHouse_SOURCE_DIR}/contrib/orc/proto - --cpp_out="${CMAKE_CURRENT_BINARY_DIR}" - "${ClickHouse_SOURCE_DIR}/contrib/orc/proto/orc_proto.proto" -) - -set(SOURCE_FILES - "${CMAKE_CURRENT_BINARY_DIR}/Adaptor.hh" - ${CMAKE_CURRENT_BINARY_DIR}/orc_proto.pb.h - ${LIBRARY_DIR}/io/InputStream.cc - ${LIBRARY_DIR}/io/OutputStream.cc - ${LIBRARY_DIR}/wrap/orc-proto-wrapper.cc - ${LIBRARY_DIR}/Adaptor.cc - ${LIBRARY_DIR}/ByteRLE.cc - ${LIBRARY_DIR}/ColumnPrinter.cc - ${LIBRARY_DIR}/ColumnReader.cc - ${LIBRARY_DIR}/ColumnWriter.cc - ${LIBRARY_DIR}/Common.cc - ${LIBRARY_DIR}/Compression.cc - ${LIBRARY_DIR}/Exceptions.cc - ${LIBRARY_DIR}/Int128.cc - ${LIBRARY_DIR}/LzoDecompressor.cc - ${LIBRARY_DIR}/MemoryPool.cc - ${LIBRARY_DIR}/OrcFile.cc - ${LIBRARY_DIR}/Reader.cc - ${LIBRARY_DIR}/RLEv1.cc - ${LIBRARY_DIR}/RLEv2.cc - ${LIBRARY_DIR}/RLE.cc - ${LIBRARY_DIR}/Statistics.cc - ${LIBRARY_DIR}/StripeStream.cc - ${LIBRARY_DIR}/Timezone.cc - ${LIBRARY_DIR}/TypeImpl.cc - ${LIBRARY_DIR}/Vector.cc - ${LIBRARY_DIR}/Writer.cc - ) - -if(ORC_CXX_HAS_THREAD_LOCAL AND BUILD_LIBHDFSPP) - set(SOURCE_FILES ${SOURCE_FILES} ${LIBRARY_DIR}/OrcHdfsFile.cc) -endif(ORC_CXX_HAS_THREAD_LOCAL AND BUILD_LIBHDFSPP) - -#list(TRANSFORM SOURCE_FILES PREPEND ${LIBRARY_DIR}/) - -configure_file ( - "${LIBRARY_INCLUDE}/orc/orc-config.hh.in" - "${CMAKE_CURRENT_BINARY_DIR}/orc/orc-config.hh" - ) - -add_library (orc ${SOURCE_FILES}) - -target_include_directories (orc - PRIVATE - ${LIBRARY_INCLUDE} - ${LIBRARY_DIR} - #PUBLIC - ${CMAKE_CURRENT_BINARY_DIR} - PRIVATE - ${PROTOBUF_INCLUDE_DIR} - ${ZLIB_INCLUDE_DIR} - ${SNAPPY_INCLUDE_DIR} - ${LZ4_INCLUDE_DIR} - ${LIBHDFSPP_INCLUDE_DIR} - ) - -target_link_libraries (orc PRIVATE - ${Protobuf_LIBRARY} - ${ZLIB_LIBRARIES} - ${SNAPPY_LIBRARY} - ${LZ4_LIBRARY} - ${LIBHDFSPP_LIBRARIES} - ) - -#install(TARGETS orc DESTINATION lib) - -if(ORC_CXX_HAS_THREAD_LOCAL AND BUILD_LIBHDFSPP) - add_definitions(-DBUILD_LIBHDFSPP) -endif(ORC_CXX_HAS_THREAD_LOCAL AND BUILD_LIBHDFSPP) diff --git a/contrib/poco b/contrib/poco index 6216cc01a10..d478f62bd93 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit 6216cc01a107ce149863411ca29013a224f80343 +Subproject commit d478f62bd93c9cd14eb343756ef73a4ae622ddf5 diff --git a/contrib/protobuf b/contrib/protobuf index 12735370922..d6a10dd3db5 160000 --- a/contrib/protobuf +++ b/contrib/protobuf @@ -1 +1 @@ -Subproject commit 12735370922a35f03999afff478e1c6d7aa917a4 +Subproject commit d6a10dd3db55d8f7f9e464db9151874cde1f79ec diff --git a/contrib/simdjson b/contrib/simdjson index e9be643db5c..60916318f76 160000 --- a/contrib/simdjson +++ b/contrib/simdjson @@ -1 +1 @@ -Subproject commit e9be643db5cf1c29a69bc80ee72d220124a9c50e +Subproject commit 60916318f76432b5d04814c2af50d04ec15664ad diff --git a/contrib/simdjson-cmake/CMakeLists.txt b/contrib/simdjson-cmake/CMakeLists.txt index 747b85e6a94..faf0755cce5 100644 --- a/contrib/simdjson-cmake/CMakeLists.txt +++ b/contrib/simdjson-cmake/CMakeLists.txt @@ -12,4 +12,4 @@ set(SIMDJSON_SRC ) add_library(${SIMDJSON_LIBRARY} ${SIMDJSON_SRC}) -target_include_directories(${SIMDJSON_LIBRARY} SYSTEM PUBLIC "${SIMDJSON_INCLUDE_DIR}") +target_include_directories(${SIMDJSON_LIBRARY} SYSTEM PUBLIC "${SIMDJSON_INCLUDE_DIR}" PRIVATE "${SIMDJSON_SRC_DIR}") diff --git a/contrib/ssl b/contrib/ssl deleted file mode 160000 index ba8de796195..00000000000 --- a/contrib/ssl +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ba8de796195ff9d8bb0249ce289b83226b848b77 diff --git a/contrib/zlib-ng b/contrib/zlib-ng index cff0f500d93..5673222fbd3 160000 --- a/contrib/zlib-ng +++ b/contrib/zlib-ng @@ -1 +1 @@ -Subproject commit cff0f500d9399d7cd3b9461a693d211e4b86fcc9 +Subproject commit 5673222fbd37ea89afb2ea73096f9bf5ec68ea31 diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index c70f3d8a068..295124c6ada 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -1,3 +1,6 @@ +set(ConfigIncludePath ${CMAKE_CURRENT_BINARY_DIR}/includes/configs CACHE INTERNAL "Path to generated configuration files.") +include_directories(${ConfigIncludePath}) + if (USE_INCLUDE_WHAT_YOU_USE) set (CMAKE_CXX_INCLUDE_WHAT_YOU_USE ${IWYU_PATH}) endif () @@ -72,11 +75,86 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") endif () endif () elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wshadow") -endif() + # Add compiler options only to c++ compiler + function(add_cxx_compile_options option) + add_compile_options( + "$<$,CXX>:${option}>" + ) + endfunction() + # Warn about boolean expression compared with an integer value different from true/false + add_cxx_compile_options(-Wbool-compare) + # Warn whenever a pointer is cast such that the required alignment of the target is increased. + add_cxx_compile_options(-Wcast-align) + # Warn whenever a pointer is cast so as to remove a type qualifier from the target type. + add_cxx_compile_options(-Wcast-qual) + # Warn when deleting a pointer to incomplete type, which may cause undefined behavior at runtime + add_cxx_compile_options(-Wdelete-incomplete) + # Warn if a requested optimization pass is disabled. Code is too big or too complex + add_cxx_compile_options(-Wdisabled-optimization) + # Warn about duplicated conditions in an if-else-if chain + add_cxx_compile_options(-Wduplicated-cond) + # Warn about a comparison between values of different enumerated types + add_cxx_compile_options(-Wenum-compare) + # Warn about uninitialized variables that are initialized with themselves + add_cxx_compile_options(-Winit-self) + # Warn about logical not used on the left hand side operand of a comparison + add_cxx_compile_options(-Wlogical-not-parentheses) + # Warn about suspicious uses of logical operators in expressions + add_cxx_compile_options(-Wlogical-op) + # Warn if there exists a path from the function entry to a use of the variable that is uninitialized. + add_cxx_compile_options(-Wmaybe-uninitialized) + # Warn when the indentation of the code does not reflect the block structure + add_cxx_compile_options(-Wmisleading-indentation) + # Warn if a global function is defined without a previous declaration + # add_cxx_compile_options(-Wmissing-declarations) + # Warn if a user-supplied include directory does not exist + # add_cxx_compile_options(-Wmissing-include-dirs) + # Obvious + add_cxx_compile_options(-Wnon-virtual-dtor) + # Obvious + add_cxx_compile_options(-Wno-return-local-addr) + # Obvious + add_cxx_compile_options(-Wnull-dereference) + # Obvious + add_cxx_compile_options(-Wodr) + # Obvious + add_cxx_compile_options(-Wold-style-cast) + # Warn when a function declaration hides virtual functions from a base class + # add_cxx_compile_options(-Woverloaded-virtual) + # Warn about placement new expressions with undefined behavior + add_cxx_compile_options(-Wplacement-new=2) + # Warn about anything that depends on the “size of” a function type or of void + add_cxx_compile_options(-Wpointer-arith) + # Warn if anything is declared more than once in the same scope + add_cxx_compile_options(-Wredundant-decls) + # Member initialization reordering + add_cxx_compile_options(-Wreorder) + # Obvious + add_cxx_compile_options(-Wshadow) + # Warn if left shifting a negative value + add_cxx_compile_options(-Wshift-negative-value) + # Warn about a definition of an unsized deallocation function + add_cxx_compile_options(-Wsized-deallocation) + # Warn when the sizeof operator is applied to a parameter that is declared as an array in a function definition + add_cxx_compile_options(-Wsizeof-array-argument) + # Warn for suspicious length parameters to certain string and memory built-in functions if the argument uses sizeof + add_cxx_compile_options(-Wsizeof-pointer-memaccess) + # Warn about overriding virtual functions that are not marked with the override keyword + # add_cxx_compile_options(-Wsuggest-override) + # Warn whenever a switch statement has an index of boolean type and the case values are outside the range of a boolean type + add_cxx_compile_options(-Wswitch-bool) + # Warn if a self-comparison always evaluates to true or false + add_cxx_compile_options(-Wtautological-compare) + # Warn about trampolines generated for pointers to nested functions + add_cxx_compile_options(-Wtrampolines) + # Obvious + add_cxx_compile_options(-Wunused) + # Warn if vector operation is not implemented via SIMD capabilities of the architecture + add_cxx_compile_options(-Wvector-operation-performance) +endif () if (USE_DEBUG_HELPERS) - set (INCLUDE_DEBUG_HELPERS "-include ${ClickHouse_SOURCE_DIR}/libs/libcommon/include/common/iostream_debug_helpers.h") + set (INCLUDE_DEBUG_HELPERS "-I${ClickHouse_SOURCE_DIR}/libs/libcommon/include -include ${ClickHouse_SOURCE_DIR}/dbms/src/Core/iostream_debug_helpers.h") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${INCLUDE_DEBUG_HELPERS}") endif () @@ -100,7 +178,7 @@ set(dbms_sources) add_headers_and_sources(clickhouse_common_io src/Common) add_headers_and_sources(clickhouse_common_io src/Common/HashTable) add_headers_and_sources(clickhouse_common_io src/IO) -list (REMOVE_ITEM clickhouse_common_io_sources src/Common/new_delete.cpp) +list (REMOVE_ITEM clickhouse_common_io_sources src/Common/malloc.cpp src/Common/new_delete.cpp) if(USE_RDKAFKA) add_headers_and_sources(dbms src/Storages/Kafka) @@ -111,7 +189,7 @@ list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD}) list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON}) list (APPEND dbms_sources src/Functions/IFunction.cpp src/Functions/FunctionFactory.cpp src/Functions/FunctionHelpers.cpp) -list (APPEND dbms_headers src/Functions/IFunction.h src/Functions/FunctionFactory.h src/Functions/FunctionHelpers.h) +list (APPEND dbms_headers src/Functions/IFunctionImpl.h src/Functions/FunctionFactory.h src/Functions/FunctionHelpers.h) list (APPEND dbms_sources src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -140,6 +218,9 @@ endif () add_library(clickhouse_common_io ${clickhouse_common_io_headers} ${clickhouse_common_io_sources}) +add_library (clickhouse_malloc OBJECT src/Common/malloc.cpp) +set_source_files_properties(src/Common/malloc.cpp PROPERTIES COMPILE_FLAGS "-fno-builtin") + add_library (clickhouse_new_delete STATIC src/Common/new_delete.cpp) target_link_libraries (clickhouse_new_delete PRIVATE clickhouse_common_io) @@ -168,6 +249,7 @@ add_object_library(clickhouse_compression src/Compression) add_object_library(clickhouse_datastreams src/DataStreams) add_object_library(clickhouse_datatypes src/DataTypes) add_object_library(clickhouse_databases src/Databases) +add_object_library(clickhouse_disks src/Disks) add_object_library(clickhouse_interpreters src/Interpreters) add_object_library(clickhouse_interpreters_clusterproxy src/Interpreters/ClusterProxy) add_object_library(clickhouse_columns src/Columns) @@ -209,7 +291,6 @@ macro (dbms_target_link_libraries) endmacro () if (USE_EMBEDDED_COMPILER) - llvm_libs_all(REQUIRED_LLVM_LIBRARIES) dbms_target_link_libraries (PRIVATE ${REQUIRED_LLVM_LIBRARIES}) dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS}) endif () @@ -376,6 +457,10 @@ if (USE_POCO_MONGODB) dbms_target_link_libraries (PRIVATE ${Poco_MongoDB_LIBRARY}) endif() +if (USE_POCO_REDIS) + dbms_target_link_libraries (PRIVATE ${Poco_Redis_LIBRARY}) +endif() + if (USE_POCO_NETSSL) target_link_libraries (clickhouse_common_io PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY}) dbms_target_link_libraries (PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY}) @@ -417,6 +502,12 @@ if (USE_HDFS) target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${HDFS3_INCLUDE_DIR}) endif() +if (USE_AWS_S3) + target_link_libraries (clickhouse_common_io PUBLIC ${AWS_S3_LIBRARY}) + target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${AWS_S3_CORE_INCLUDE_DIR}) + target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${AWS_S3_INCLUDE_DIR}) +endif() + if (USE_BROTLI) target_link_libraries (clickhouse_common_io PRIVATE ${BROTLI_LIBRARY}) target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${BROTLI_INCLUDE_DIR}) @@ -425,6 +516,13 @@ endif() if (USE_JEMALLOC) dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # used in Interpreters/AsynchronousMetrics.cpp target_include_directories (clickhouse_new_delete SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) + + if(NOT MAKE_STATIC_LIBRARIES AND ${JEMALLOC_LIBRARIES} MATCHES "${CMAKE_STATIC_LIBRARY_SUFFIX}$") + # mallctl in dbms/src/Interpreters/AsynchronousMetrics.cpp + # Actually we link JEMALLOC to almost all libraries. + # This is just hotfix for some uninvestigated problem. + target_link_libraries(clickhouse_interpreters PRIVATE ${JEMALLOC_LIBRARIES}) + endif() endif () dbms_target_include_directories (PUBLIC ${DBMS_INCLUDE_DIR} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/src/Formats/include) diff --git a/dbms/benchmark/clickhouse/queries.sql b/dbms/benchmark/clickhouse/queries.sql index 05d4e00fb19..726beebb064 100644 --- a/dbms/benchmark/clickhouse/queries.sql +++ b/dbms/benchmark/clickhouse/queries.sql @@ -34,10 +34,10 @@ SELECT WatchID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM SELECT URL, count() AS c FROM {table} GROUP BY URL ORDER BY c DESC LIMIT 10; SELECT 1, URL, count() AS c FROM {table} GROUP BY 1, URL ORDER BY c DESC LIMIT 10; SELECT ClientIP AS x, x - 1, x - 2, x - 3, count() AS c FROM {table} GROUP BY x, x - 1, x - 2, x - 3 ORDER BY c DESC LIMIT 10; -SELECT URL, count() AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND notEmpty(URL) GROUP BY URL ORDER BY PageViews DESC LIMIT 10; -SELECT Title, count() AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND notEmpty(Title) GROUP BY Title ORDER BY PageViews DESC LIMIT 10; -SELECT URL, count() AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; -SELECT TraficSourceID, SearchEngineID, AdvEngineID, ((SearchEngineID = 0 AND AdvEngineID = 0) ? Referer : '') AS Src, URL AS Dst, count() AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000; -SELECT URLHash, EventDate, count() AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = halfMD5('http://example.ru/') GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100; -SELECT WindowClientWidth, WindowClientHeight, count() AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh AND NOT DontCountHits AND URLHash = halfMD5('http://example.ru/') GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; -SELECT toStartOfMinute(EventTime) AS Minute, count() AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-02') AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute; +SELECT URL, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND notEmpty(URL) GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +SELECT Title, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND notEmpty(Title) GROUP BY Title ORDER BY PageViews DESC LIMIT 10; +SELECT URL, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; +SELECT TraficSourceID, SearchEngineID, AdvEngineID, ((SearchEngineID = 0 AND AdvEngineID = 0) ? Referer : '') AS Src, URL AS Dst, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000; +SELECT URLHash, EventDate, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = halfMD5('http://example.ru/') GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100; +SELECT WindowClientWidth, WindowClientHeight, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh AND NOT DontCountHits AND URLHash = halfMD5('http://example.ru/') GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; +SELECT toStartOfMinute(EventTime) AS Minute, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-02') AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute; diff --git a/dbms/benchmark/greenplum/queries.sql b/dbms/benchmark/greenplum/queries.sql index 5aec0e8fafa..8b5ed6ed9c8 100644 --- a/dbms/benchmark/greenplum/queries.sql +++ b/dbms/benchmark/greenplum/queries.sql @@ -34,10 +34,10 @@ SELECT WatchID, ClientIP, count(1) AS c, sum(Refresh), avg(ResolutionWidth) FROM SELECT URL, count(1) AS c FROM {table} GROUP BY URL ORDER BY c DESC LIMIT 10 SELECT 1, URL, count(1) AS c FROM {table} GROUP BY 1, URL ORDER BY c DESC LIMIT 10 SELECT ClientIP AS x, ClientIP - 1, ClientIP - 2, ClientIP - 3, count(1) AS c FROM {table} GROUP BY x, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10 -SELECT URL, count(1) AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate between '2013-07-01'::timestamp AND '2013-07-31'::timestamp AND DontCountHits =0 AND Refresh = 0 AND URL <>'' GROUP BY URL ORDER BY PageViews DESC LIMIT 10 -SELECT Title, count(1) AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate BETWEEN '2013-07-01'::timestamp AND '2013-07-31'::timestamp AND DontCountHits=0 AND Refresh=0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10 -SELECT URL, count(1) AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate between '2013-07-01'::timestamp AND '2013-07-31'::timestamp AND Refresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; -SELECT TraficSourceID, SearchEngineID, AdvEngineID, case when (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END Src, URL AS Dst, count(1) AS PageViews FROM {table} WHERE CounterID = 34 AND eventDate between '2013-07-01'::timestamp AND '2013-07-31'::timestamp AND Refresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000; -SELECT URLHash, EventDate, count(1) AS PageViews FROM {table} WHERE CounterID = 34 AND eventDate between '2013-07-01'::timestamp AND '2013-07-31'::timestamp AND Refresh =0 AND TraficSourceID IN (-1, 6) AND RefererHash = 7135345792483900000 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100 -SELECT WindowClientWidth, WindowClientHeight, count(1) AS PageViews FROM {table} WHERE CounterID = 34 AND eventDate between '2013-07-01'::timestamp AND '2013-07-31'::timestamp AND Refresh =0 AND DontCountHits =0 AND URLHash = 7135345792483900000 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; -SELECT date_trunc('minute', EventTime) AS Minute, count(1) AS PageViews FROM {table} WHERE CounterID = 34 AND eventDate between '2013-07-01'::timestamp AND '2013-07-31'::timestamp AND Refresh =0 AND DontCountHits =0 GROUP BY Minute ORDER BY Minute; +SELECT URL, count(1) AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate between '2013-07-01'::timestamp AND '2013-07-31'::timestamp AND DontCountHits =0 AND Refresh = 0 AND URL <>'' GROUP BY URL ORDER BY PageViews DESC LIMIT 10 +SELECT Title, count(1) AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate BETWEEN '2013-07-01'::timestamp AND '2013-07-31'::timestamp AND DontCountHits=0 AND Refresh=0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10 +SELECT URL, count(1) AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate between '2013-07-01'::timestamp AND '2013-07-31'::timestamp AND Refresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; +SELECT TraficSourceID, SearchEngineID, AdvEngineID, case when (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END Src, URL AS Dst, count(1) AS PageViews FROM {table} WHERE CounterID = 62 AND eventDate between '2013-07-01'::timestamp AND '2013-07-31'::timestamp AND Refresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000; +SELECT URLHash, EventDate, count(1) AS PageViews FROM {table} WHERE CounterID = 62 AND eventDate between '2013-07-01'::timestamp AND '2013-07-31'::timestamp AND Refresh =0 AND TraficSourceID IN (-1, 6) AND RefererHash = 7135345792483900000 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100 +SELECT WindowClientWidth, WindowClientHeight, count(1) AS PageViews FROM {table} WHERE CounterID = 62 AND eventDate between '2013-07-01'::timestamp AND '2013-07-31'::timestamp AND Refresh =0 AND DontCountHits =0 AND URLHash = 7135345792483900000 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; +SELECT date_trunc('minute', EventTime) AS Minute, count(1) AS PageViews FROM {table} WHERE CounterID = 62 AND eventDate between '2013-07-01'::timestamp AND '2013-07-31'::timestamp AND Refresh =0 AND DontCountHits =0 GROUP BY Minute ORDER BY Minute; diff --git a/dbms/benchmark/hive/queries.sql b/dbms/benchmark/hive/queries.sql index 74b0fd92b40..e48aa7057d8 100644 --- a/dbms/benchmark/hive/queries.sql +++ b/dbms/benchmark/hive/queries.sql @@ -92,18 +92,18 @@ SELECT 1, URL, count(*) AS c FROM hits_10m GROUP BY 1, URL ORDER BY c DESC LIMIT SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, count(*) AS c FROM hits_10m GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10; -SELECT URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= TIMESTAMP('2013-07-01') AND EventDate <= TIMESTAMP('2013-07-31') AND NOT DontCountHits != 0 AND NOT Refresh != 0 AND URL != '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +SELECT URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= TIMESTAMP('2013-07-01') AND EventDate <= TIMESTAMP('2013-07-31') AND NOT DontCountHits != 0 AND NOT Refresh != 0 AND URL != '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; -SELECT Title, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= TIMESTAMP('2013-07-01') AND EventDate <= TIMESTAMP('2013-07-31') AND NOT DontCountHits != 0 AND NOT Refresh != 0 AND Title != '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; +SELECT Title, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= TIMESTAMP('2013-07-01') AND EventDate <= TIMESTAMP('2013-07-31') AND NOT DontCountHits != 0 AND NOT Refresh != 0 AND Title != '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; -SELECT URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= TIMESTAMP('2013-07-01') AND EventDate <= TIMESTAMP('2013-07-31') AND NOT Refresh != 0 AND IsLink != 0 AND NOT IsDownload != 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; +SELECT URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= TIMESTAMP('2013-07-01') AND EventDate <= TIMESTAMP('2013-07-31') AND NOT Refresh != 0 AND IsLink != 0 AND NOT IsDownload != 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; -SELECT TraficSourceID, SearchEngineID, AdvEngineID, URL, count(*) as c, if(SearchEngineID = 0 AND AdvEngineID = 0 , Referer, '') as src FROM hits_100m WHERE CounterID = 34 AND EventDate >= TIMESTAMP('2013-07-01') AND EventDate <= TIMESTAMP('2013-07-31') AND NOT Refresh != 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, URL, if(SearchEngineID = 0 AND AdvEngineID = 0 , Referer, '') ORDER BY c DESC LIMIT 1000; +SELECT TraficSourceID, SearchEngineID, AdvEngineID, URL, count(*) as c, if(SearchEngineID = 0 AND AdvEngineID = 0 , Referer, '') as src FROM hits_100m WHERE CounterID = 62 AND EventDate >= TIMESTAMP('2013-07-01') AND EventDate <= TIMESTAMP('2013-07-31') AND NOT Refresh != 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, URL, if(SearchEngineID = 0 AND AdvEngineID = 0 , Referer, '') ORDER BY c DESC LIMIT 1000; -SELECT URLHash, EventDate, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= TIMESTAMP('2013-07-01') AND EventDate <= TIMESTAMP('2013-07-31') AND NOT Refresh != 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 6202628419148573758 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100000; +SELECT URLHash, EventDate, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= TIMESTAMP('2013-07-01') AND EventDate <= TIMESTAMP('2013-07-31') AND NOT Refresh != 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 6202628419148573758 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100000; -SELECT WindowClientWidth, WindowClientHeight, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= TIMESTAMP('2013-07-01') AND EventDate <= TIMESTAMP('2013-07-31') AND NOT Refresh != 0 AND NOT DontCountHits != 0 AND URLHash = 6202628419148573758 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; +SELECT WindowClientWidth, WindowClientHeight, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= TIMESTAMP('2013-07-01') AND EventDate <= TIMESTAMP('2013-07-31') AND NOT Refresh != 0 AND NOT DontCountHits != 0 AND URLHash = 6202628419148573758 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; -SELECT unix_timestamp(EventTime) - SECOND(EventTime) AS m, count(*) FROM hits_10m WHERE CounterID = 34 AND EventDate >= TIMESTAMP('2013-07-01') AND EventDate <= TIMESTAMP('2013-07-02') AND NOT Refresh != 0 AND NOT DontCountHits != 0 GROUP BY unix_timestamp(EventTime) - SECOND(EventTime) ORDER BY m; +SELECT unix_timestamp(EventTime) - SECOND(EventTime) AS m, count(*) FROM hits_10m WHERE CounterID = 62 AND EventDate >= TIMESTAMP('2013-07-01') AND EventDate <= TIMESTAMP('2013-07-02') AND NOT Refresh != 0 AND NOT DontCountHits != 0 GROUP BY unix_timestamp(EventTime) - SECOND(EventTime) ORDER BY m; diff --git a/dbms/benchmark/infinidb/queries.sql b/dbms/benchmark/infinidb/queries.sql index 126e7fc090f..b4da23ac8c8 100644 --- a/dbms/benchmark/infinidb/queries.sql +++ b/dbms/benchmark/infinidb/queries.sql @@ -92,20 +92,20 @@ SELECT SQL_NO_CACHE 1, URL, count(*) FROM hits_10m GROUP BY 1, URL ORDER BY coun SELECT SQL_NO_CACHE ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, count(*) FROM hits_10m GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY count(*) DESC LIMIT 10; -SELECT SQL_NO_CACHE URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND URL != '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +SELECT SQL_NO_CACHE URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND URL != '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; -SELECT SQL_NO_CACHE Title, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND Title != '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; +SELECT SQL_NO_CACHE Title, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND Title != '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; -SELECT SQL_NO_CACHE URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; +SELECT SQL_NO_CACHE URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; -SELECT SQL_NO_CACHE TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN SearchEngineID = 0 AND AdvEngineID = 0 THEN Referer ELSE '' END AS Src, URL AS Dst, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000; +SELECT SQL_NO_CACHE TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN SearchEngineID = 0 AND AdvEngineID = 0 THEN Referer ELSE '' END AS Src, URL AS Dst, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000; -SELECT SQL_NO_CACHE URLHash, EventDate, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = 6202628419148573758 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100000; +SELECT SQL_NO_CACHE URLHash, EventDate, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = 6202628419148573758 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100000; -SELECT SQL_NO_CACHE WindowClientWidth, WindowClientHeight, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND NOT DontCountHits AND URLHash = 6202628419148573758 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; +SELECT SQL_NO_CACHE WindowClientWidth, WindowClientHeight, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND NOT DontCountHits AND URLHash = 6202628419148573758 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; -SELECT SQL_NO_CACHE EventTime - INTERVAL SECOND(EventTime) SECOND AS Minute, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-02') AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute; \ No newline at end of file +SELECT SQL_NO_CACHE EventTime - INTERVAL SECOND(EventTime) SECOND AS Minute, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-02') AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute; \ No newline at end of file diff --git a/dbms/benchmark/infobright/queries.sql b/dbms/benchmark/infobright/queries.sql index 0e8028f9040..f5a14c5908e 100644 --- a/dbms/benchmark/infobright/queries.sql +++ b/dbms/benchmark/infobright/queries.sql @@ -94,20 +94,20 @@ SELECT 1, URL, count(*) FROM hits_10m GROUP BY 1, URL ORDER BY count(*) DESC LIM SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, count(*) FROM hits_10m GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY count(*) DESC LIMIT 10; -SELECT URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND URL != '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +SELECT URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND URL != '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; -SELECT Title, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND Title != '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; +SELECT Title, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND Title != '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; -SELECT URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; +SELECT URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; -SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN SearchEngineID = 0 AND AdvEngineID = 0 THEN Referer ELSE '' END AS Src, URL AS Dst, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000; +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN SearchEngineID = 0 AND AdvEngineID = 0 THEN Referer ELSE '' END AS Src, URL AS Dst, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000; -SELECT URLHash, EventDate, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = 6202628419148573758 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100000; +SELECT URLHash, EventDate, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = 6202628419148573758 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100000; -SELECT WindowClientWidth, WindowClientHeight, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND NOT DontCountHits AND URLHash = 6202628419148573758 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; +SELECT WindowClientWidth, WindowClientHeight, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND NOT DontCountHits AND URLHash = 6202628419148573758 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; -SELECT EventTime - INTERVAL SECOND(EventTime) SECOND AS Minute, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-02') AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute; \ No newline at end of file +SELECT EventTime - INTERVAL SECOND(EventTime) SECOND AS Minute, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-02') AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute; \ No newline at end of file diff --git a/dbms/benchmark/memsql/queries.sql b/dbms/benchmark/memsql/queries.sql index 1c8b0b8798f..6069bad1b44 100644 --- a/dbms/benchmark/memsql/queries.sql +++ b/dbms/benchmark/memsql/queries.sql @@ -34,10 +34,10 @@ SELECT WatchID, ClientIP, count(*) AS c, sum(Refresh), avg(ResolutionWidth) FROM SELECT URL, count(*) FROM hits_10m GROUP BY URL ORDER BY count(*) DESC LIMIT 10; SELECT 1, URL, count(*) FROM hits_10m GROUP BY 1, URL ORDER BY count(*) DESC LIMIT 10; SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, count(*) FROM hits_10m GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY count(*) DESC LIMIT 10; -SELECT URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND URL != '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; -SELECT Title, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND Title != '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; -SELECT URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; -SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN SearchEngineID = 0 AND AdvEngineID = 0 THEN Referer ELSE '' END AS Src, URL AS Dst, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000; -SELECT URLHash, EventDate, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = 6202628419148573758 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100000; -SELECT WindowClientWidth, WindowClientHeight, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND NOT DontCountHits AND URLHash = 6202628419148573758 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; -SELECT EventTime - INTERVAL SECOND(EventTime) SECOND AS Minute, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-02') AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute; +SELECT URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND URL != '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +SELECT Title, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND Title != '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; +SELECT URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN SearchEngineID = 0 AND AdvEngineID = 0 THEN Referer ELSE '' END AS Src, URL AS Dst, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000; +SELECT URLHash, EventDate, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = 6202628419148573758 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100000; +SELECT WindowClientWidth, WindowClientHeight, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND NOT DontCountHits AND URLHash = 6202628419148573758 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; +SELECT EventTime - INTERVAL SECOND(EventTime) SECOND AS Minute, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-02') AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute; diff --git a/dbms/benchmark/monetdb/queries.sql b/dbms/benchmark/monetdb/queries.sql index 2d05167ef76..c17868bb31a 100644 --- a/dbms/benchmark/monetdb/queries.sql +++ b/dbms/benchmark/monetdb/queries.sql @@ -92,20 +92,20 @@ SELECT 1, URL, count(*) FROM hits_10m GROUP BY 1, URL ORDER BY count(*) DESC LIM SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, count(*) FROM hits_10m GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY count(*) DESC LIMIT 10; -SELECT URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT DontCountHits AND NOT Refresh AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +SELECT URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT DontCountHits AND NOT Refresh AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; -SELECT Title, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT DontCountHits AND NOT Refresh AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; +SELECT Title, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT DontCountHits AND NOT Refresh AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; -SELECT URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; +SELECT URL, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; -SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN SearchEngineID = 0 AND AdvEngineID = 0 THEN Referer ELSE '' END AS Src, URL AS Dst, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000; +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN SearchEngineID = 0 AND AdvEngineID = 0 THEN Referer ELSE '' END AS Src, URL AS Dst, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000; -SELECT URLHash, EventDate, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = 6202628419148573758 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100000; +SELECT URLHash, EventDate, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = 6202628419148573758 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100000; -SELECT WindowClientWidth, WindowClientHeight, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh AND NOT DontCountHits AND URLHash = 6202628419148573758 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; +SELECT WindowClientWidth, WindowClientHeight, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh AND NOT DontCountHits AND URLHash = 6202628419148573758 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; -SELECT EventTime - extract (SECOND from EventTime) AS M, count(*) AS PageViews FROM hits_10m WHERE CounterID = 34 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-02' AND NOT Refresh AND NOT DontCountHits GROUP BY M ORDER BY M; \ No newline at end of file +SELECT EventTime - extract (SECOND from EventTime) AS M, count(*) AS PageViews FROM hits_10m WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-02' AND NOT Refresh AND NOT DontCountHits GROUP BY M ORDER BY M; \ No newline at end of file diff --git a/dbms/benchmark/vertica/queries.sql b/dbms/benchmark/vertica/queries.sql index cd276338f69..e7906ce65e3 100644 --- a/dbms/benchmark/vertica/queries.sql +++ b/dbms/benchmark/vertica/queries.sql @@ -34,10 +34,10 @@ SELECT WatchID, ClientIP, count(*) AS c, sum(Refresh), avg(ResolutionWidth) FROM SELECT URL, count(*) FROM {table} GROUP BY URL ORDER BY count(*) DESC LIMIT 10; SELECT 1, URL, count(*) FROM {table} GROUP BY 1, URL ORDER BY count(*) DESC LIMIT 10; SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, count(*) FROM {table} GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY count(*) DESC LIMIT 10; -SELECT URL, count(*) AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND URL != '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; -SELECT Title, count(*) AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND Title != '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; -SELECT URL, count(*) AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; -SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN SearchEngineID = 0 AND AdvEngineID = 0 THEN Referer ELSE '' END AS Src, URL AS Dst, count(*) AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000; -SELECT URLHash, EventDate, count(*) AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = 6202628419148573758 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100000; -SELECT WindowClientWidth, WindowClientHeight, count(*) AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND NOT DontCountHits AND URLHash = 6202628419148573758 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; -SELECT TIME_SLICE(EventTime, 1, 'MINUTE') AS Minute, count(*) AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-02') AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute; +SELECT URL, count(*) AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND URL != '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +SELECT Title, count(*) AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND Title != '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; +SELECT URL, count(*) AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN SearchEngineID = 0 AND AdvEngineID = 0 THEN Referer ELSE '' END AS Src, URL AS Dst, count(*) AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000; +SELECT URLHash, EventDate, count(*) AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = 6202628419148573758 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100000; +SELECT WindowClientWidth, WindowClientHeight, count(*) AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-31') AND NOT Refresh AND NOT DontCountHits AND URLHash = 6202628419148573758 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; +SELECT TIME_SLICE(EventTime, 1, 'MINUTE') AS Minute, count(*) AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= DATE('2013-07-01') AND EventDate <= DATE('2013-07-02') AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute; diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index a544274826e..220af3d87dc 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -1,11 +1,11 @@ # This strings autochanged from release_lib.sh: -set(VERSION_REVISION 54428) +set(VERSION_REVISION 54430) set(VERSION_MAJOR 19) -set(VERSION_MINOR 17) +set(VERSION_MINOR 19) set(VERSION_PATCH 1) -set(VERSION_GITHASH 5286d0afb285a5fbf3d320af3daa6de6b1841374) -set(VERSION_DESCRIBE v19.17.1.1-prestable) -set(VERSION_STRING 19.17.1.1) +set(VERSION_GITHASH 8bd9709d1dec3366e35d2efeab213435857f67a9) +set(VERSION_DESCRIBE v19.19.1.1-prestable) +set(VERSION_STRING 19.19.1.1) # end of autochange set(VERSION_EXTRA "" CACHE STRING "") diff --git a/dbms/programs/CMakeLists.txt b/dbms/programs/CMakeLists.txt index 138321360f3..48fb30a8058 100644 --- a/dbms/programs/CMakeLists.txt +++ b/dbms/programs/CMakeLists.txt @@ -19,7 +19,7 @@ if(NOT (MAKE_STATIC_LIBRARIES OR SPLIT_SHARED_LIBRARIES)) set(CLICKHOUSE_ONE_SHARED 1) endif() -configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h) +configure_file (config_tools.h.in ${ConfigIncludePath}/config_tools.h) macro(clickhouse_target_link_split_lib target name) diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp index c0b2eccfd29..6f08475f934 100644 --- a/dbms/programs/benchmark/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -504,6 +504,7 @@ public: #ifndef __clang__ #pragma GCC optimize("-fno-var-tracking-assignments") #endif +#pragma GCC diagnostic ignored "-Wmissing-declarations" int mainEntryClickHouseBenchmark(int argc, char ** argv) { diff --git a/dbms/programs/benchmark/clickhouse-benchmark.cpp b/dbms/programs/benchmark/clickhouse-benchmark.cpp index 96715419b1b..6bcb6e19b88 100644 --- a/dbms/programs/benchmark/clickhouse-benchmark.cpp +++ b/dbms/programs/benchmark/clickhouse-benchmark.cpp @@ -1,2 +1,2 @@ -int mainEntryClickHouseBenchmark(int argc, char ** argv); +extern int mainEntryClickHouseBenchmark(int argc, char ** argv); int main(int argc_, char ** argv_) { return mainEntryClickHouseBenchmark(argc_, argv_); } diff --git a/dbms/programs/client/CMakeLists.txt b/dbms/programs/client/CMakeLists.txt index c9996de1e9b..dc5cf787adf 100644 --- a/dbms/programs/client/CMakeLists.txt +++ b/dbms/programs/client/CMakeLists.txt @@ -12,7 +12,7 @@ endif () include(CheckSymbolExists) check_symbol_exists(readpassphrase readpassphrase.h HAVE_READPASSPHRASE) -configure_file(config_client.h.in ${CMAKE_CURRENT_BINARY_DIR}/include/config_client.h) +configure_file(config_client.h.in ${ConfigIncludePath}/config_client.h) if(NOT HAVE_READPASSPHRASE) add_subdirectory(readpassphrase) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index a2826bc8cd2..4b9cee29ff6 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -89,6 +89,39 @@ #define DISABLE_LINE_WRAPPING "\033[?7l" #define ENABLE_LINE_WRAPPING "\033[?7h" +#if USE_READLINE && RL_VERSION_MAJOR >= 7 + +#define BRACK_PASTE_PREF "\033[200~" +#define BRACK_PASTE_SUFF "\033[201~" + +#define BRACK_PASTE_LAST '~' +#define BRACK_PASTE_SLEN 6 + +/// This handler bypasses some unused macro/event checkings. +static int clickhouse_rl_bracketed_paste_begin(int /* count */, int /* key */) +{ + std::string buf; + buf.reserve(128); + + RL_SETSTATE(RL_STATE_MOREINPUT); + SCOPE_EXIT(RL_UNSETSTATE(RL_STATE_MOREINPUT)); + int c; + while ((c = rl_read_key()) >= 0) + { + if (c == '\r') + c = '\n'; + buf.push_back(c); + if (buf.size() >= BRACK_PASTE_SLEN && c == BRACK_PASTE_LAST && buf.substr(buf.size() - BRACK_PASTE_SLEN) == BRACK_PASTE_SUFF) + { + buf.resize(buf.size() - BRACK_PASTE_SLEN); + break; + } + } + return static_cast(rl_insert_text(buf.c_str())) == buf.size() ? 0 : 1; +} + +#endif + namespace DB { @@ -462,6 +495,24 @@ private: if (rl_initialize()) throw Exception("Cannot initialize readline", ErrorCodes::CANNOT_READLINE); +#if RL_VERSION_MAJOR >= 7 + /// Enable bracketed-paste-mode only when multiquery is enabled and multiline is + /// disabled, so that we are able to paste and execute multiline queries in a whole + /// instead of erroring out, while be less intrusive. + if (config().has("multiquery") && !config().has("multiline")) + { + /// When bracketed paste mode is set, pasted text is bracketed with control sequences so + /// that the program can differentiate pasted text from typed-in text. This helps + /// clickhouse-client so that without -m flag, one can still paste multiline queries, and + /// possibly get better pasting performance. See https://cirw.in/blog/bracketed-paste for + /// more details. + rl_variable_bind("enable-bracketed-paste", "on"); + + /// Use our bracketed paste handler to get better user experience. See comments above. + rl_bind_keyseq(BRACK_PASTE_PREF, clickhouse_rl_bracketed_paste_begin); + } +#endif + auto clear_prompt_or_exit = [](int) { /// This is signal safe. @@ -632,7 +683,8 @@ private: /// If the user restarts the client then after pressing the "up" button /// every line of the query will be displayed separately. std::string logged_query = input; - std::replace(logged_query.begin(), logged_query.end(), '\n', ' '); + if (config().has("multiline")) + std::replace(logged_query.begin(), logged_query.end(), '\n', ' '); add_history(logged_query.c_str()); #if USE_READLINE && HAVE_READLINE_HISTORY @@ -704,6 +756,9 @@ private: bool process(const String & text) { + if (exit_strings.end() != exit_strings.find(trim(text, [](char c){ return isWhitespaceASCII(c) || c == ';'; }))) + return false; + const bool test_mode = config().has("testmode"); if (config().has("multiquery")) { @@ -798,9 +853,6 @@ private: bool processSingleQuery(const String & line, ASTPtr parsed_query_ = nullptr) { - if (exit_strings.end() != exit_strings.find(trim(line, [](char c){ return isWhitespaceASCII(c) || c == ';'; }))) - return false; - resetOutput(); got_exception = false; @@ -1059,7 +1111,14 @@ private: /// Check if server send Exception packet auto packet_type = connection->checkPacket(); if (packet_type && *packet_type == Protocol::Server::Exception) + { + /* + * We're exiting with error, so it makes sense to kill the + * input stream without waiting for it to complete. + */ + async_block_input->cancel(true); return; + } connection->sendData(block); processed_rows += block.rows(); @@ -1173,7 +1232,7 @@ private: /// Returns true if one should continue receiving packets. bool receiveAndProcessPacket() { - Connection::Packet packet = connection->receivePacket(); + Packet packet = connection->receivePacket(); switch (packet.type) { @@ -1221,7 +1280,7 @@ private: { while (true) { - Connection::Packet packet = connection->receivePacket(); + Packet packet = connection->receivePacket(); switch (packet.type) { @@ -1255,7 +1314,7 @@ private: { while (true) { - Connection::Packet packet = connection->receivePacket(); + Packet packet = connection->receivePacket(); switch (packet.type) { @@ -1883,6 +1942,9 @@ public: } +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wmissing-declarations" + int mainEntryClickHouseClient(int argc, char ** argv) { try diff --git a/dbms/programs/client/Suggest.h b/dbms/programs/client/Suggest.h index 57895b38764..78cc8d94db0 100644 --- a/dbms/programs/client/Suggest.h +++ b/dbms/programs/client/Suggest.h @@ -113,7 +113,7 @@ private: while (true) { - Connection::Packet packet = connection.receivePacket(); + Packet packet = connection.receivePacket(); switch (packet.type) { case Protocol::Server::Data: diff --git a/dbms/programs/client/readpassphrase/CMakeLists.txt b/dbms/programs/client/readpassphrase/CMakeLists.txt index b7ff6db31ee..94ed9f54bdb 100644 --- a/dbms/programs/client/readpassphrase/CMakeLists.txt +++ b/dbms/programs/client/readpassphrase/CMakeLists.txt @@ -6,5 +6,8 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-result -Wno-reserved-id-macro") configure_file(includes.h.in ${CMAKE_CURRENT_BINARY_DIR}/include/includes.h) add_library(readpassphrase ${CMAKE_CURRENT_SOURCE_DIR}/readpassphrase.c) +set_target_properties(readpassphrase + PROPERTIES LINKER_LANGUAGE C + ) # . to allow #include target_include_directories(readpassphrase PUBLIC . ${CMAKE_CURRENT_BINARY_DIR}/include ${CMAKE_CURRENT_BINARY_DIR}/../include) diff --git a/dbms/programs/compressor/Compressor.cpp b/dbms/programs/compressor/Compressor.cpp index a073a79b416..9c4699b610a 100644 --- a/dbms/programs/compressor/Compressor.cpp +++ b/dbms/programs/compressor/Compressor.cpp @@ -57,6 +57,8 @@ void checkAndWriteHeader(DB::ReadBuffer & in, DB::WriteBuffer & out) } +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wmissing-declarations" int mainEntryClickHouseCompressor(int argc, char ** argv) { diff --git a/dbms/programs/copier/ClusterCopier.cpp b/dbms/programs/copier/ClusterCopier.cpp index 5fc1d76b542..a095e99fe22 100644 --- a/dbms/programs/copier/ClusterCopier.cpp +++ b/dbms/programs/copier/ClusterCopier.cpp @@ -1,6 +1,7 @@ #include "ClusterCopier.h" #include +#include #include #include #include @@ -64,6 +65,7 @@ #include #include #include +#include #include #include @@ -178,7 +180,9 @@ struct ShardPartition ShardPartition(TaskShard & parent, const String & name_quoted_) : task_shard(parent), name(name_quoted_) {} String getPartitionPath() const; + String getPartitionCleanStartPath() const; String getCommonPartitionIsDirtyPath() const; + String getCommonPartitionIsCleanedPath() const; String getPartitionActiveWorkersPath() const; String getActiveWorkerPath() const; String getPartitionShardsPath() const; @@ -259,6 +263,8 @@ struct TaskTable String getPartitionPath(const String & partition_name) const; String getPartitionIsDirtyPath(const String & partition_name) const; + String getPartitionIsCleanedPath(const String & partition_name) const; + String getPartitionTaskStatusPath(const String & partition_name) const; String name_in_config; @@ -369,23 +375,6 @@ struct MultiTransactionInfo Coordination::Responses responses; }; - -/// Atomically checks that is_dirty node is not exists, and made the remaining op -/// Returns relative number of failed operation in the second field (the passed op has 0 index) -static MultiTransactionInfo checkNoNodeAndCommit( - const zkutil::ZooKeeperPtr & zookeeper, - const String & checking_node_path, - Coordination::RequestPtr && op) -{ - MultiTransactionInfo info; - info.requests.emplace_back(zkutil::makeCreateRequest(checking_node_path, "", zkutil::CreateMode::Persistent)); - info.requests.emplace_back(zkutil::makeRemoveRequest(checking_node_path, -1)); - info.requests.emplace_back(std::move(op)); - info.code = zookeeper->tryMulti(info.requests, info.responses); - return info; -} - - // Creates AST representing 'ENGINE = Distributed(cluster, db, table, [sharding_key]) std::shared_ptr createASTStorageDistributed( const String & cluster_name, const String & database, const String & table, const ASTPtr & sharding_key_ast = nullptr) @@ -431,6 +420,11 @@ String TaskTable::getPartitionPath(const String & partition_name) const + "/" + escapeForFileName(partition_name); // 201701 } +String ShardPartition::getPartitionCleanStartPath() const +{ + return getPartitionPath() + "/clean_start"; +} + String ShardPartition::getPartitionPath() const { return task_shard.task_table.getPartitionPath(name); @@ -438,8 +432,9 @@ String ShardPartition::getPartitionPath() const String ShardPartition::getShardStatusPath() const { - // /root/table_test.hits/201701/1 - return getPartitionPath() + "/shards/" + toString(task_shard.numberInCluster()); + // schema: //tables///shards/ + // e.g. /root/table_test.hits/201701/shards/1 + return getPartitionShardsPath() + "/" + toString(task_shard.numberInCluster()); } String ShardPartition::getPartitionShardsPath() const @@ -462,11 +457,26 @@ String ShardPartition::getCommonPartitionIsDirtyPath() const return getPartitionPath() + "/is_dirty"; } +String ShardPartition::getCommonPartitionIsCleanedPath() const +{ + return getCommonPartitionIsDirtyPath() + "/cleaned"; +} + String TaskTable::getPartitionIsDirtyPath(const String & partition_name) const { return getPartitionPath(partition_name) + "/is_dirty"; } +String TaskTable::getPartitionIsCleanedPath(const String & partition_name) const +{ + return getPartitionIsDirtyPath(partition_name) + "/cleaned"; +} + +String TaskTable::getPartitionTaskStatusPath(const String & partition_name) const +{ + return getPartitionPath(partition_name) + "/shards"; +} + String DB::TaskShard::getDescription() const { std::stringstream ss; @@ -1129,9 +1139,9 @@ protected: } /** Checks that the whole partition of a table was copied. We should do it carefully due to dirty lock. - * State of some task could be changed during the processing. - * We have to ensure that all shards have the finished state and there are no dirty flag. - * Moreover, we have to check status twice and check zxid, because state could be changed during the checking. + * State of some task could change during the processing. + * We have to ensure that all shards have the finished state and there is no dirty flag. + * Moreover, we have to check status twice and check zxid, because state can change during the checking. */ bool checkPartitionIsDone(const TaskTable & task_table, const String & partition_name, const TasksShard & shards_with_partition) { @@ -1170,10 +1180,22 @@ protected: } // Check that partition is not dirty - if (zookeeper->exists(task_table.getPartitionIsDirtyPath(partition_name))) { - LOG_INFO(log, "Partition " << partition_name << " become dirty"); - return false; + CleanStateClock clean_state_clock ( + zookeeper, + task_table.getPartitionIsDirtyPath(partition_name), + task_table.getPartitionIsCleanedPath(partition_name) + ); + Coordination::Stat stat; + LogicalClock task_start_clock; + if (zookeeper->exists(task_table.getPartitionTaskStatusPath(partition_name), &stat)) + task_start_clock = LogicalClock(stat.mzxid); + zookeeper->get(task_table.getPartitionTaskStatusPath(partition_name), &stat); + if (!clean_state_clock.is_clean() || task_start_clock <= clean_state_clock.discovery_zxid) + { + LOG_INFO(log, "Partition " << partition_name << " become dirty"); + return false; + } } get_futures.clear(); @@ -1260,17 +1282,188 @@ protected: return res; } - bool tryDropPartition(ShardPartition & task_partition, const zkutil::ZooKeeperPtr & zookeeper) + /** Allows to compare two incremental counters of type UInt32 in presence of possible overflow. + * We assume that we compare values that are not too far away. + * For example, when we increment 0xFFFFFFFF, we get 0. So, 0xFFFFFFFF is less than 0. + */ + class WrappingUInt32 + { + public: + UInt32 value; + + WrappingUInt32(UInt32 _value) + : value(_value) + {} + + bool operator<(const WrappingUInt32 & other) const + { + return value != other.value && *this <= other; + } + + bool operator<=(const WrappingUInt32 & other) const + { + const UInt32 HALF = 1 << 31; + return (value <= other.value && other.value - value < HALF) + || (value > other.value && value - other.value > HALF); + } + + bool operator==(const WrappingUInt32 & other) const + { + return value == other.value; + } + }; + + /** Conforming Zxid definition. + * cf. https://github.com/apache/zookeeper/blob/631d1b284f0edb1c4f6b0fb221bf2428aec71aaa/zookeeper-docs/src/main/resources/markdown/zookeeperInternals.md#guarantees-properties-and-definitions + */ + class Zxid + { + public: + WrappingUInt32 epoch; + WrappingUInt32 counter; + Zxid(UInt64 _zxid) + : epoch(_zxid >> 32) + , counter(_zxid) + {} + + bool operator<=(const Zxid & other) const + { + return (epoch < other.epoch) + || (epoch == other.epoch && counter <= other.counter); + } + + bool operator==(const Zxid & other) const + { + return epoch == other.epoch && counter == other.counter; + } + }; + + class LogicalClock + { + public: + std::optional zxid; + + LogicalClock() = default; + + LogicalClock(UInt64 _zxid) + : zxid(_zxid) + {} + + bool hasHappened() const + { + return bool(zxid); + } + + // happens-before relation with a reasonable time bound + bool happensBefore(const LogicalClock & other) const + { + return !zxid + || (other.zxid && *zxid <= *other.zxid); + } + + bool operator<=(const LogicalClock & other) const + { + return happensBefore(other); + } + + // strict equality check + bool operator==(const LogicalClock & other) const + { + return zxid == other.zxid; + } + }; + + class CleanStateClock + { + public: + LogicalClock discovery_zxid; + std::optional discovery_version; + + LogicalClock clean_state_zxid; + std::optional clean_state_version; + + std::shared_ptr stale; + + bool is_clean() const + { + return + !is_stale() + && ( + !discovery_zxid.hasHappened() + || (clean_state_zxid.hasHappened() && discovery_zxid <= clean_state_zxid)); + } + + bool is_stale() const + { + return stale->load(); + } + + CleanStateClock( + const zkutil::ZooKeeperPtr & zookeeper, + const String & discovery_path, + const String & clean_state_path) + : stale(std::make_shared(false)) + { + Coordination::Stat stat; + String _some_data; + auto watch_callback = + [stale = stale] (const Coordination::WatchResponse & rsp) + { + auto logger = &Poco::Logger::get("ClusterCopier"); + if (rsp.error == Coordination::ZOK) + { + switch (rsp.type) + { + case Coordination::CREATED: + LOG_DEBUG(logger, "CleanStateClock change: CREATED, at " << rsp.path); + stale->store(true); + break; + case Coordination::CHANGED: + LOG_DEBUG(logger, "CleanStateClock change: CHANGED, at" << rsp.path); + stale->store(true); + } + } + }; + if (zookeeper->tryGetWatch(discovery_path, _some_data, &stat, watch_callback)) + { + discovery_zxid = LogicalClock(stat.mzxid); + discovery_version = stat.version; + } + if (zookeeper->tryGetWatch(clean_state_path, _some_data, &stat, watch_callback)) + { + clean_state_zxid = LogicalClock(stat.mzxid); + clean_state_version = stat.version; + } + } + + bool operator==(const CleanStateClock & other) const + { + return !is_stale() + && !other.is_stale() + && discovery_zxid == other.discovery_zxid + && discovery_version == other.discovery_version + && clean_state_zxid == other.clean_state_zxid + && clean_state_version == other.clean_state_version; + } + + bool operator!=(const CleanStateClock & other) const + { + return !(*this == other); + } + }; + + bool tryDropPartition(ShardPartition & task_partition, const zkutil::ZooKeeperPtr & zookeeper, const CleanStateClock & clean_state_clock) { if (is_safe_mode) throw Exception("DROP PARTITION is prohibited in safe mode", ErrorCodes::NOT_IMPLEMENTED); TaskTable & task_table = task_partition.task_shard.task_table; - String current_shards_path = task_partition.getPartitionShardsPath(); - String current_partition_active_workers_dir = task_partition.getPartitionActiveWorkersPath(); - String is_dirty_flag_path = task_partition.getCommonPartitionIsDirtyPath(); - String dirt_cleaner_path = is_dirty_flag_path + "/cleaner"; + const String current_shards_path = task_partition.getPartitionShardsPath(); + const String current_partition_active_workers_dir = task_partition.getPartitionActiveWorkersPath(); + const String is_dirty_flag_path = task_partition.getCommonPartitionIsDirtyPath(); + const String dirt_cleaner_path = is_dirty_flag_path + "/cleaner"; + const String is_dirt_cleaned_path = task_partition.getCommonPartitionIsCleanedPath(); zkutil::EphemeralNodeHolder::Ptr cleaner_holder; try @@ -1294,44 +1487,92 @@ protected: { if (stat.numChildren != 0) { - LOG_DEBUG(log, "Partition " << task_partition.name << " contains " << stat.numChildren << " active workers, sleep"); + LOG_DEBUG(log, "Partition " << task_partition.name << " contains " << stat.numChildren << " active workers while trying to drop it. Going to sleep."); std::this_thread::sleep_for(default_sleep_time); return false; } + else + { + zookeeper->remove(current_partition_active_workers_dir); + } } - /// Remove all status nodes - zookeeper->tryRemoveRecursive(current_shards_path); - - String query = "ALTER TABLE " + getQuotedTable(task_table.table_push); - query += " DROP PARTITION " + task_partition.name + ""; - - /// TODO: use this statement after servers will be updated up to 1.1.54310 - // query += " DROP PARTITION ID '" + task_partition.name + "'"; - - ClusterPtr & cluster_push = task_table.cluster_push; - Settings settings_push = task_cluster->settings_push; - - /// It is important, DROP PARTITION must be done synchronously - settings_push.replication_alter_partitions_sync = 2; - - LOG_DEBUG(log, "Execute distributed DROP PARTITION: " << query); - /// Limit number of max executing replicas to 1 - UInt64 num_shards = executeQueryOnCluster(cluster_push, query, nullptr, &settings_push, PoolMode::GET_ONE, 1); - - if (num_shards < cluster_push->getShardCount()) { - LOG_INFO(log, "DROP PARTITION wasn't successfully executed on " << cluster_push->getShardCount() - num_shards << " shards"); - return false; + zkutil::EphemeralNodeHolder::Ptr active_workers_lock; + try + { + active_workers_lock = zkutil::EphemeralNodeHolder::create(current_partition_active_workers_dir, *zookeeper, host_id); + } + catch (const Coordination::Exception & e) + { + if (e.code == Coordination::ZNODEEXISTS) + { + LOG_DEBUG(log, "Partition " << task_partition.name << " is being filled now by somebody, sleep"); + return false; + } + + throw; + } + + // Lock the dirty flag + zookeeper->set(is_dirty_flag_path, host_id, clean_state_clock.discovery_version.value()); + zookeeper->tryRemove(task_partition.getPartitionCleanStartPath()); + CleanStateClock my_clock(zookeeper, is_dirty_flag_path, is_dirt_cleaned_path); + + /// Remove all status nodes + { + Strings children; + if (zookeeper->tryGetChildren(current_shards_path, children) == Coordination::ZOK) + for (const auto & child : children) + { + zookeeper->removeRecursive(current_shards_path + "/" + child); + } + } + + String query = "ALTER TABLE " + getQuotedTable(task_table.table_push); + query += " DROP PARTITION " + task_partition.name + ""; + + /// TODO: use this statement after servers will be updated up to 1.1.54310 + // query += " DROP PARTITION ID '" + task_partition.name + "'"; + + ClusterPtr & cluster_push = task_table.cluster_push; + Settings settings_push = task_cluster->settings_push; + + /// It is important, DROP PARTITION must be done synchronously + settings_push.replication_alter_partitions_sync = 2; + + LOG_DEBUG(log, "Execute distributed DROP PARTITION: " << query); + /// Limit number of max executing replicas to 1 + UInt64 num_shards = executeQueryOnCluster(cluster_push, query, nullptr, &settings_push, PoolMode::GET_ONE, 1); + + if (num_shards < cluster_push->getShardCount()) + { + LOG_INFO(log, "DROP PARTITION wasn't successfully executed on " << cluster_push->getShardCount() - num_shards << " shards"); + return false; + } + + /// Update the locking node + if (!my_clock.is_stale()) + { + zookeeper->set(is_dirty_flag_path, host_id, my_clock.discovery_version.value()); + if (my_clock.clean_state_version) + zookeeper->set(is_dirt_cleaned_path, host_id, my_clock.clean_state_version.value()); + else + zookeeper->create(is_dirt_cleaned_path, host_id, zkutil::CreateMode::Persistent); + } + else + { + LOG_DEBUG(log, "Clean state is altered when dropping the partition, cowardly bailing"); + /// clean state is stale + return false; + } + + LOG_INFO(log, "Partition " << task_partition.name << " was dropped on cluster " << task_table.cluster_push_name); + if (zookeeper->tryCreate(current_shards_path, host_id, zkutil::CreateMode::Persistent) == Coordination::ZNODEEXISTS) + zookeeper->set(current_shards_path, host_id); } - /// Remove the locking node - Coordination::Requests requests; - requests.emplace_back(zkutil::makeRemoveRequest(dirt_cleaner_path, -1)); - requests.emplace_back(zkutil::makeRemoveRequest(is_dirty_flag_path, -1)); - zookeeper->multi(requests); - - LOG_INFO(log, "Partition " << task_partition.name << " was dropped on cluster " << task_table.cluster_push_name); + LOG_INFO(log, "Partition " << task_partition.name << " is safe for work now."); return true; } @@ -1362,6 +1603,7 @@ protected: /// Process each source shard having current partition and copy current partition /// NOTE: shards are sorted by "distance" to current host + bool has_shard_to_process = false; for (const TaskShardPtr & shard : task_table.all_shards) { /// Does shard have a node with current partition? @@ -1405,6 +1647,7 @@ protected: bool is_unprioritized_task = !previous_shard_is_instantly_finished && shard->priority.is_remote; PartitionTaskStatus task_status = PartitionTaskStatus::Error; bool was_error = false; + has_shard_to_process = true; for (UInt64 try_num = 0; try_num < max_shard_partition_tries; ++try_num) { task_status = tryProcessPartitionTask(timeouts, partition, is_unprioritized_task); @@ -1432,11 +1675,13 @@ protected: cluster_partition.elapsed_time_seconds += watch.elapsedSeconds(); /// Check that whole cluster partition is done - /// Firstly check number failed partition tasks, than look into ZooKeeper and ensure that each partition is done + /// Firstly check the number of failed partition tasks, then look into ZooKeeper and ensure that each partition is done bool partition_is_done = num_failed_shards == 0; try { - partition_is_done = partition_is_done && checkPartitionIsDone(task_table, partition_name, expected_shards); + partition_is_done = + !has_shard_to_process + || (partition_is_done && checkPartitionIsDone(task_table, partition_name, expected_shards)); } catch (...) { @@ -1526,20 +1771,35 @@ protected: TaskTable & task_table = task_shard.task_table; ClusterPartition & cluster_partition = task_table.getClusterPartition(task_partition.name); + /// We need to update table definitions for each partition, it could be changed after ALTER + createShardInternalTables(timeouts, task_shard); + auto zookeeper = context.getZooKeeper(); - String is_dirty_flag_path = task_partition.getCommonPartitionIsDirtyPath(); - String current_task_is_active_path = task_partition.getActiveWorkerPath(); - String current_task_status_path = task_partition.getShardStatusPath(); + const String is_dirty_flag_path = task_partition.getCommonPartitionIsDirtyPath(); + const String is_dirt_cleaned_path = task_partition.getCommonPartitionIsCleanedPath(); + const String current_task_is_active_path = task_partition.getActiveWorkerPath(); + const String current_task_status_path = task_partition.getShardStatusPath(); /// Auxiliary functions: /// Creates is_dirty node to initialize DROP PARTITION - auto create_is_dirty_node = [&] () + auto create_is_dirty_node = [&, this] (const CleanStateClock & clock) { - auto code = zookeeper->tryCreate(is_dirty_flag_path, current_task_status_path, zkutil::CreateMode::Persistent); - if (code && code != Coordination::ZNODEEXISTS) - throw Coordination::Exception(code, is_dirty_flag_path); + if (clock.is_stale()) + LOG_DEBUG(log, "Clean state clock is stale while setting dirty flag, cowardly bailing"); + else if (!clock.is_clean()) + LOG_DEBUG(log, "Thank you, Captain Obvious"); + else if (clock.discovery_version) + { + LOG_DEBUG(log, "Updating clean state clock"); + zookeeper->set(is_dirty_flag_path, host_id, clock.discovery_version.value()); + } + else + { + LOG_DEBUG(log, "Creating clean state clock"); + zookeeper->create(is_dirty_flag_path, host_id, zkutil::CreateMode::Persistent); + } }; /// Returns SELECT query filtering current partition and applying user filter @@ -1563,14 +1823,29 @@ protected: LOG_DEBUG(log, "Processing " << current_task_status_path); + CleanStateClock clean_state_clock (zookeeper, is_dirty_flag_path, is_dirt_cleaned_path); + + LogicalClock task_start_clock; + { + Coordination::Stat stat; + if (zookeeper->exists(task_partition.getPartitionShardsPath(), &stat)) + task_start_clock = LogicalClock(stat.mzxid); + } + /// Do not start if partition is dirty, try to clean it - if (zookeeper->exists(is_dirty_flag_path)) + if (clean_state_clock.is_clean() + && (!task_start_clock.hasHappened() || clean_state_clock.discovery_zxid <= task_start_clock)) + { + LOG_DEBUG(log, "Partition " << task_partition.name << " appears to be clean"); + zookeeper->createAncestors(current_task_status_path); + } + else { LOG_DEBUG(log, "Partition " << task_partition.name << " is dirty, try to drop it"); try { - tryDropPartition(task_partition, zookeeper); + tryDropPartition(task_partition, zookeeper, clean_state_clock); } catch (...) { @@ -1598,7 +1873,8 @@ protected: throw; } - /// Exit if task has been already processed, create blocking node if it is abandoned + /// Exit if task has been already processed; + /// create blocking node to signal cleaning up if it is abandoned { String status_data; if (zookeeper->tryGet(current_task_status_path, status_data)) @@ -1611,21 +1887,21 @@ protected: } // Task is abandoned, initialize DROP PARTITION - LOG_DEBUG(log, "Task " << current_task_status_path << " has not been successfully finished by " << status.owner); + LOG_DEBUG(log, "Task " << current_task_status_path << " has not been successfully finished by " << status.owner << ". Partition will be dropped and refilled."); - create_is_dirty_node(); + create_is_dirty_node(clean_state_clock); return PartitionTaskStatus::Error; } } - zookeeper->createAncestors(current_task_status_path); - - /// We need to update table definitions for each partition, it could be changed after ALTER - createShardInternalTables(timeouts, task_shard); - /// Check that destination partition is empty if we are first worker /// NOTE: this check is incorrect if pull and push tables have different partition key! + String clean_start_status; + if (!zookeeper->tryGet(task_partition.getPartitionCleanStartPath(), clean_start_status) || clean_start_status != "ok") { + zookeeper->createIfNotExists(task_partition.getPartitionCleanStartPath(), ""); + auto checker = zkutil::EphemeralNodeHolder::create(task_partition.getPartitionCleanStartPath() + "/checker", *zookeeper, host_id); + // Maybe we are the first worker ASTPtr query_select_ast = get_select_query(task_shard.table_split_shard, "count()"); UInt64 count; { @@ -1643,36 +1919,38 @@ protected: Coordination::Stat stat_shards; zookeeper->get(task_partition.getPartitionShardsPath(), &stat_shards); + /// NOTE: partition is still fresh if dirt discovery happens before cleaning if (stat_shards.numChildren == 0) { - LOG_WARNING(log, "There are no any workers for partition " << task_partition.name + LOG_WARNING(log, "There are no workers for partition " << task_partition.name << ", but destination table contains " << count << " rows" << ". Partition will be dropped and refilled."); - create_is_dirty_node(); + create_is_dirty_node(clean_state_clock); return PartitionTaskStatus::Error; } } + zookeeper->set(task_partition.getPartitionCleanStartPath(), "ok"); } + /// At this point, we need to sync that the destination table is clean + /// before any actual work /// Try start processing, create node about it { String start_state = TaskStateWithOwner::getData(TaskState::Started, host_id); - auto op_create = zkutil::makeCreateRequest(current_task_status_path, start_state, zkutil::CreateMode::Persistent); - MultiTransactionInfo info = checkNoNodeAndCommit(zookeeper, is_dirty_flag_path, std::move(op_create)); - - if (info.code) + CleanStateClock new_clean_state_clock (zookeeper, is_dirty_flag_path, is_dirt_cleaned_path); + if (clean_state_clock != new_clean_state_clock) { - zkutil::KeeperMultiException exception(info.code, info.requests, info.responses); - - if (exception.getPathForFirstFailedOp() == is_dirty_flag_path) - { - LOG_INFO(log, "Partition " << task_partition.name << " is dirty and will be dropped and refilled"); - return PartitionTaskStatus::Error; - } - - throw exception; + LOG_INFO(log, "Partition " << task_partition.name << " clean state changed, cowardly bailing"); + return PartitionTaskStatus::Error; } + else if (!new_clean_state_clock.is_clean()) + { + LOG_INFO(log, "Partition " << task_partition.name << " is dirty and will be dropped and refilled"); + create_is_dirty_node(new_clean_state_clock); + return PartitionTaskStatus::Error; + } + zookeeper->create(current_task_status_path, start_state, zkutil::CreateMode::Persistent); } /// Try create table (if not exists) on each shard @@ -1733,12 +2011,13 @@ protected: output = io_insert.out; } + /// Fail-fast optimization to abort copying when the current clean state expires std::future future_is_dirty_checker; Stopwatch watch(CLOCK_MONOTONIC_COARSE); constexpr UInt64 check_period_milliseconds = 500; - /// Will asynchronously check that ZooKeeper connection and is_dirty flag appearing while copy data + /// Will asynchronously check that ZooKeeper connection and is_dirty flag appearing while copying data auto cancel_check = [&] () { if (zookeeper->expired()) @@ -1754,7 +2033,12 @@ protected: Coordination::ExistsResponse status = future_is_dirty_checker.get(); if (status.error != Coordination::ZNONODE) + { + LogicalClock dirt_discovery_epoch (status.stat.mzxid); + if (dirt_discovery_epoch == clean_state_clock.discovery_zxid) + return false; throw Exception("Partition is dirty, cancel INSERT SELECT", ErrorCodes::UNFINISHED); + } } return false; @@ -1789,20 +2073,19 @@ protected: /// Finalize the processing, change state of current partition task (and also check is_dirty flag) { String state_finished = TaskStateWithOwner::getData(TaskState::Finished, host_id); - auto op_set = zkutil::makeSetRequest(current_task_status_path, state_finished, 0); - MultiTransactionInfo info = checkNoNodeAndCommit(zookeeper, is_dirty_flag_path, std::move(op_set)); - - if (info.code) + CleanStateClock new_clean_state_clock (zookeeper, is_dirty_flag_path, is_dirt_cleaned_path); + if (clean_state_clock != new_clean_state_clock) { - zkutil::KeeperMultiException exception(info.code, info.requests, info.responses); - - if (exception.getPathForFirstFailedOp() == is_dirty_flag_path) - LOG_INFO(log, "Partition " << task_partition.name << " became dirty and will be dropped and refilled"); - else - LOG_INFO(log, "Someone made the node abandoned. Will refill partition. " << zkutil::ZooKeeper::error2string(info.code)); - + LOG_INFO(log, "Partition " << task_partition.name << " clean state changed, cowardly bailing"); return PartitionTaskStatus::Error; } + else if (!new_clean_state_clock.is_clean()) + { + LOG_INFO(log, "Partition " << task_partition.name << " became dirty and will be dropped and refilled"); + create_is_dirty_node(new_clean_state_clock); + return PartitionTaskStatus::Error; + } + zookeeper->set(current_task_status_path, state_finished, 0); } LOG_INFO(log, "Partition " << task_partition.name << " copied"); @@ -2181,6 +2464,7 @@ void ClusterCopierApp::mainImpl() registerTableFunctions(); registerStorages(); registerDictionaries(); + registerDisks(); static const std::string default_database = "_local"; context->addDatabase(default_database, std::make_shared(default_database)); @@ -2229,6 +2513,8 @@ int ClusterCopierApp::main(const std::vector &) } +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wmissing-declarations" int mainEntryClickHouseClusterCopier(int argc, char ** argv) { diff --git a/dbms/programs/extract-from-config/ExtractFromConfig.cpp b/dbms/programs/extract-from-config/ExtractFromConfig.cpp index af9550e4547..dff7e81c430 100644 --- a/dbms/programs/extract-from-config/ExtractFromConfig.cpp +++ b/dbms/programs/extract-from-config/ExtractFromConfig.cpp @@ -44,6 +44,9 @@ static std::string extractFromConfig( return configuration->getString(key); } +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wmissing-declarations" + int mainEntryClickHouseExtractFromConfig(int argc, char ** argv) { bool print_stacktrace = false; diff --git a/dbms/programs/format/Format.cpp b/dbms/programs/format/Format.cpp index ff415d88e1b..f826d6394bc 100644 --- a/dbms/programs/format/Format.cpp +++ b/dbms/programs/format/Format.cpp @@ -8,6 +8,9 @@ #include #include +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wmissing-declarations" + int mainEntryClickHouseFormat(int argc, char ** argv) { using namespace DB; diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp index c3dfcacf3f3..f84d9d4b6ac 100644 --- a/dbms/programs/local/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -152,6 +153,7 @@ try registerTableFunctions(); registerStorages(); registerDictionaries(); + registerDisks(); /// Maybe useless if (config().has("macros")) @@ -441,7 +443,7 @@ void LocalServer::init(int argc, char ** argv) exit(0); } - if (options.count("help")) + if (options.empty() || options.count("help")) { std::cout << getHelpHeader() << "\n"; std::cout << description << "\n"; @@ -497,6 +499,9 @@ void LocalServer::applyCmdOptions() } +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wmissing-declarations" + int mainEntryClickHouseLocal(int argc, char ** argv) { DB::LocalServer app; diff --git a/dbms/programs/main.cpp b/dbms/programs/main.cpp index 760eae4298b..a063463d7c9 100644 --- a/dbms/programs/main.cpp +++ b/dbms/programs/main.cpp @@ -14,10 +14,6 @@ #include "config_core.h" #endif -#if USE_TCMALLOC -#include -#endif - #include #include @@ -153,14 +149,6 @@ int main(int argc_, char ** argv_) /// will work only after additional call of this function. updatePHDRCache(); -#if USE_TCMALLOC - /** Without this option, tcmalloc returns memory to OS too frequently for medium-sized memory allocations - * (like IO buffers, column vectors, hash tables, etc.), - * that lead to page faults and significantly hurts performance. - */ - MallocExtension::instance()->SetNumericProperty("tcmalloc.aggressive_memory_decommit", false); -#endif - std::vector argv(argv_, argv_ + argc_); /// Print a basic help if nothing was matched diff --git a/dbms/programs/obfuscator/Obfuscator.cpp b/dbms/programs/obfuscator/Obfuscator.cpp index be6125d77bf..f267acc1f01 100644 --- a/dbms/programs/obfuscator/Obfuscator.cpp +++ b/dbms/programs/obfuscator/Obfuscator.cpp @@ -123,14 +123,14 @@ UInt64 hash(Ts... xs) } -UInt64 maskBits(UInt64 x, size_t num_bits) +static UInt64 maskBits(UInt64 x, size_t num_bits) { return x & ((1ULL << num_bits) - 1); } /// Apply Feistel network round to least significant num_bits part of x. -UInt64 feistelRound(UInt64 x, size_t num_bits, UInt64 seed, size_t round) +static UInt64 feistelRound(UInt64 x, size_t num_bits, UInt64 seed, size_t round) { size_t num_bits_left_half = num_bits / 2; size_t num_bits_right_half = num_bits - num_bits_left_half; @@ -146,7 +146,7 @@ UInt64 feistelRound(UInt64 x, size_t num_bits, UInt64 seed, size_t round) /// Apply Feistel network with num_rounds to least significant num_bits part of x. -UInt64 feistelNetwork(UInt64 x, size_t num_bits, UInt64 seed, size_t num_rounds = 4) +static UInt64 feistelNetwork(UInt64 x, size_t num_bits, UInt64 seed, size_t num_rounds = 4) { UInt64 bits = maskBits(x, num_bits); for (size_t i = 0; i < num_rounds; ++i) @@ -156,7 +156,7 @@ UInt64 feistelNetwork(UInt64 x, size_t num_bits, UInt64 seed, size_t num_rounds /// Pseudorandom permutation within set of numbers with the same log2(x). -UInt64 transform(UInt64 x, UInt64 seed) +static UInt64 transform(UInt64 x, UInt64 seed) { /// Keep 0 and 1 as is. if (x == 0 || x == 1) @@ -199,7 +199,7 @@ public: /// Keep sign and apply pseudorandom permutation after converting to unsigned as above. -Int64 transformSigned(Int64 x, UInt64 seed) +static Int64 transformSigned(Int64 x, UInt64 seed) { if (x >= 0) return transform(x, seed); @@ -298,7 +298,7 @@ public: /// Pseudorandom function, but keep word characters as word characters. -void transformFixedString(const UInt8 * src, UInt8 * dst, size_t size, UInt64 seed) +static void transformFixedString(const UInt8 * src, UInt8 * dst, size_t size, UInt64 seed) { { SipHash hash; @@ -579,7 +579,7 @@ public: { for (auto & elem : table) { - Histogram & histogram = elem.getSecond(); + Histogram & histogram = elem.getMapped(); if (histogram.buckets.size() < params.num_buckets_cutoff) { @@ -593,7 +593,7 @@ public: { for (auto & elem : table) { - Histogram & histogram = elem.getSecond(); + Histogram & histogram = elem.getMapped(); if (!histogram.total) continue; @@ -625,7 +625,7 @@ public: { for (auto & elem : table) { - Histogram & histogram = elem.getSecond(); + Histogram & histogram = elem.getMapped(); if (!histogram.total) continue; @@ -641,7 +641,7 @@ public: { for (auto & elem : table) { - Histogram & histogram = elem.getSecond(); + Histogram & histogram = elem.getMapped(); if (!histogram.total) continue; @@ -676,7 +676,7 @@ public: while (true) { it = table.find(hashContext(code_points.data() + code_points.size() - context_size, code_points.data() + code_points.size())); - if (it && lookupResultGetMapped(it)->total + lookupResultGetMapped(it)->count_end != 0) + if (it && it->getMapped().total + it->getMapped().count_end != 0) break; if (context_size == 0) @@ -710,7 +710,7 @@ public: if (num_bytes_after_desired_size > 0) end_probability_multiplier = std::pow(1.25, num_bytes_after_desired_size); - CodePoint code = lookupResultGetMapped(it)->sample(determinator, end_probability_multiplier); + CodePoint code = it->getMapped().sample(determinator, end_probability_multiplier); if (code == END) break; @@ -943,6 +943,8 @@ public: } +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wmissing-declarations" int mainEntryClickHouseObfuscator(int argc, char ** argv) try diff --git a/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp b/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp index 594cddfd3db..3dadc7632de 100644 --- a/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -95,6 +96,7 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques std::string schema_name = ""; std::string table_name = params.get("table"); std::string connection_string = params.get("connection_string"); + if (params.has("schema")) { schema_name = params.get("schema"); @@ -106,6 +108,8 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques try { + const bool external_table_functions_use_nulls = Poco::NumberParser::parseBool(params.get("external_table_functions_use_nulls", "false")); + POCO_SQL_ODBC_CLASS::SessionImpl session(validateODBCConnectionString(connection_string), DBMS_DEFAULT_CONNECT_TIMEOUT_SEC); SQLHDBC hdbc = session.dbc().handle(); @@ -160,13 +164,13 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques /// TODO Why 301? SQLCHAR column_name[301]; - SQLSMALLINT nullable; - const auto result = POCO_SQL_ODBC_CLASS::SQLDescribeCol(hstmt, ncol, column_name, sizeof(column_name), nullptr, &type, nullptr, nullptr, &nullable); + SQLSMALLINT is_nullable; + const auto result = POCO_SQL_ODBC_CLASS::SQLDescribeCol(hstmt, ncol, column_name, sizeof(column_name), nullptr, &type, nullptr, nullptr, &is_nullable); if (POCO_SQL_ODBC_CLASS::Utility::isError(result)) throw POCO_SQL_ODBC_CLASS::StatementException(hstmt); auto column_type = getDataType(type); - if (nullable == SQL_NULLABLE) + if (external_table_functions_use_nulls && is_nullable == SQL_NULLABLE) { column_type = std::make_shared(column_type); } diff --git a/dbms/programs/odbc-bridge/MainHandler.cpp b/dbms/programs/odbc-bridge/MainHandler.cpp index 162e93dc3db..73480bf884f 100644 --- a/dbms/programs/odbc-bridge/MainHandler.cpp +++ b/dbms/programs/odbc-bridge/MainHandler.cpp @@ -35,7 +35,7 @@ using PocoSessionPoolConstructor = std::function createAndCheckResizePocoSessionPool(PocoSessionPoolConstructor pool_constr) +static std::shared_ptr createAndCheckResizePocoSessionPool(PocoSessionPoolConstructor pool_constr) { static std::mutex mutex; diff --git a/dbms/programs/odbc-bridge/ODBCBridge.cpp b/dbms/programs/odbc-bridge/ODBCBridge.cpp index c84452b691e..453ee499784 100644 --- a/dbms/programs/odbc-bridge/ODBCBridge.cpp +++ b/dbms/programs/odbc-bridge/ODBCBridge.cpp @@ -193,6 +193,7 @@ int ODBCBridge::main(const std::vector & /*args*/) } } +#pragma GCC diagnostic ignored "-Wmissing-declarations" int mainEntryClickHouseODBCBridge(int argc, char ** argv) { DB::ODBCBridge app; diff --git a/dbms/programs/performance-test/JSONString.h b/dbms/programs/performance-test/JSONString.h index 5695145442e..ebd850877d7 100644 --- a/dbms/programs/performance-test/JSONString.h +++ b/dbms/programs/performance-test/JSONString.h @@ -22,7 +22,7 @@ public: void set(const std::string & key, std::string value, bool wrap = true); template - std::enable_if_t> set(const std::string key, T value) + std::enable_if_t> set(const std::string key, T value) { set(key, std::to_string(value), /*wrap= */ false); } diff --git a/dbms/programs/performance-test/PerformanceTest.cpp b/dbms/programs/performance-test/PerformanceTest.cpp index ab55cd3d6cf..689f68f8d5e 100644 --- a/dbms/programs/performance-test/PerformanceTest.cpp +++ b/dbms/programs/performance-test/PerformanceTest.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -35,7 +36,7 @@ void waitQuery(Connection & connection) if (!connection.poll(1000000)) continue; - Connection::Packet packet = connection.receivePacket(); + Packet packet = connection.receivePacket(); switch (packet.type) { case Protocol::Server::EndOfStream: @@ -120,7 +121,7 @@ bool PerformanceTest::checkPreconditions() const while (true) { - Connection::Packet packet = connection.receivePacket(); + Packet packet = connection.receivePacket(); if (packet.type == Protocol::Server::Data) { @@ -142,7 +143,7 @@ bool PerformanceTest::checkPreconditions() const if (!exist) { - LOG_WARNING(log, "Table " << table_to_check << " doesn't exist"); + LOG_WARNING(log, "Table " << backQuote(table_to_check) << " doesn't exist"); return false; } } diff --git a/dbms/programs/performance-test/PerformanceTestSuite.cpp b/dbms/programs/performance-test/PerformanceTestSuite.cpp index eaa4e24cde9..594f04a3906 100644 --- a/dbms/programs/performance-test/PerformanceTestSuite.cpp +++ b/dbms/programs/performance-test/PerformanceTestSuite.cpp @@ -294,7 +294,7 @@ static std::vector getInputFiles(const po::variables_map & options, return input_files; } -std::unordered_map> getTestQueryIndexes(const po::basic_parsed_options & parsed_opts) +static std::unordered_map> getTestQueryIndexes(const po::basic_parsed_options & parsed_opts) { std::unordered_map> result; const auto & options = parsed_opts.options; @@ -319,6 +319,9 @@ std::unordered_map> getTestQueryIndexes(co return result; } +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wmissing-declarations" + int mainEntryClickHousePerformanceTest(int argc, char ** argv) try { diff --git a/dbms/programs/performance-test/applySubstitutions.cpp b/dbms/programs/performance-test/applySubstitutions.cpp index b8c1d4b6059..a18e066fb01 100644 --- a/dbms/programs/performance-test/applySubstitutions.cpp +++ b/dbms/programs/performance-test/applySubstitutions.cpp @@ -30,7 +30,7 @@ void constructSubstitutions(ConfigurationPtr & substitutions_view, StringToVecto /// Recursive method which goes through all substitution blocks in xml /// and replaces property {names} by their values -void runThroughAllOptionsAndPush(StringToVector::iterator substitutions_left, +static void runThroughAllOptionsAndPush(StringToVector::iterator substitutions_left, StringToVector::iterator substitutions_right, const std::string & template_query, Strings & out_queries) diff --git a/dbms/programs/server/CMakeLists.txt b/dbms/programs/server/CMakeLists.txt index 16aa7131291..e36307198b4 100644 --- a/dbms/programs/server/CMakeLists.txt +++ b/dbms/programs/server/CMakeLists.txt @@ -1,22 +1,23 @@ set(CLICKHOUSE_SERVER_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/HTTPHandler.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/HTTPHandlerFactory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/InterserverIOHTTPHandler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/MetricsTransmitter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/NotFoundHandler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/PingRequestHandler.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/PrometheusMetricsWriter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/PrometheusRequestHandler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ReplicasStatusHandler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/RootRequestHandler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Server.cpp ${CMAKE_CURRENT_SOURCE_DIR}/TCPHandler.cpp ) -if (USE_SSL) - set(CLICKHOUSE_SERVER_SOURCES - ${CLICKHOUSE_SERVER_SOURCES} - ${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandler.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandlerFactory.cpp - ) -endif () +set(CLICKHOUSE_SERVER_SOURCES + ${CLICKHOUSE_SERVER_SOURCES} + ${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandler.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandlerFactory.cpp +) set(CLICKHOUSE_SERVER_LINK PRIVATE clickhouse_dictionaries clickhouse_common_io clickhouse_common_config clickhouse_common_zookeeper clickhouse_parsers string_utils PUBLIC daemon PRIVATE clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY}) if (USE_POCO_NETSSL) diff --git a/dbms/programs/server/HTTPHandler.cpp b/dbms/programs/server/HTTPHandler.cpp index 7ab1105e453..29d186def2d 100644 --- a/dbms/programs/server/HTTPHandler.cpp +++ b/dbms/programs/server/HTTPHandler.cpp @@ -34,7 +34,6 @@ #include #include #include -#include #include #include @@ -407,16 +406,16 @@ void HTTPHandler::processQuery( { if (http_request_compression_method_str == "gzip") { - in_post = std::make_unique(*in_post_raw, CompressionMethod::Gzip); + in_post = std::make_unique(std::move(in_post_raw), CompressionMethod::Gzip); } else if (http_request_compression_method_str == "deflate") { - in_post = std::make_unique(*in_post_raw, CompressionMethod::Zlib); + in_post = std::make_unique(std::move(in_post_raw), CompressionMethod::Zlib); } #if USE_BROTLI else if (http_request_compression_method_str == "br") { - in_post = std::make_unique(*in_post_raw); + in_post = std::make_unique(std::move(in_post_raw)); } #endif else diff --git a/dbms/programs/server/HTTPHandlerFactory.cpp b/dbms/programs/server/HTTPHandlerFactory.cpp new file mode 100644 index 00000000000..ab8fb4f7336 --- /dev/null +++ b/dbms/programs/server/HTTPHandlerFactory.cpp @@ -0,0 +1,43 @@ +#include "HTTPHandlerFactory.h" + + +namespace DB +{ + +HTTPRequestHandlerFactoryMain::HTTPRequestHandlerFactoryMain(IServer & server_, const std::string & name_) + : server(server_), log(&Logger::get(name_)), name(name_) +{ +} + +Poco::Net::HTTPRequestHandler * HTTPRequestHandlerFactoryMain::createRequestHandler( + const Poco::Net::HTTPServerRequest & request) // override +{ + LOG_TRACE(log, "HTTP Request for " << name << ". " + << "Method: " + << request.getMethod() + << ", Address: " + << request.clientAddress().toString() + << ", User-Agent: " + << (request.has("User-Agent") ? request.get("User-Agent") : "none") + << (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : ("")) + << ", Content Type: " << request.getContentType() + << ", Transfer Encoding: " << request.getTransferEncoding()); + + for (auto & handlerFactory: child_handler_factories) + { + auto handler = handlerFactory->createRequestHandler(request); + if (handler != nullptr) + return handler; + } + + if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET + || request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD + || request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST) + { + return new NotFoundHandler; + } + + return nullptr; +} + +} diff --git a/dbms/programs/server/HTTPHandlerFactory.h b/dbms/programs/server/HTTPHandlerFactory.h index ce65f4476c0..fcd7fb5d4a2 100644 --- a/dbms/programs/server/HTTPHandlerFactory.h +++ b/dbms/programs/server/HTTPHandlerFactory.h @@ -9,6 +9,7 @@ #include "InterserverIOHTTPHandler.h" #include "NotFoundHandler.h" #include "PingRequestHandler.h" +#include "PrometheusRequestHandler.h" #include "ReplicasStatusHandler.h" #include "RootRequestHandler.h" @@ -16,60 +17,111 @@ namespace DB { -template -class HTTPRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory +/// Handle request using child handlers +class HTTPRequestHandlerFactoryMain : public Poco::Net::HTTPRequestHandlerFactory { private: + using TThis = HTTPRequestHandlerFactoryMain; + IServer & server; Logger * log; std::string name; + std::vector> child_handler_factories; + public: - HTTPRequestHandlerFactory(IServer & server_, const std::string & name_) : server(server_), log(&Logger::get(name_)), name(name_) + HTTPRequestHandlerFactoryMain(IServer & server_, const std::string & name_); + + Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest & request) override; + + template + TThis * addHandler(TArgs &&... args) { + child_handler_factories.emplace_back(std::make_unique(server, std::forward(args)...)); + return this; } +}; + + +/// Handle POST or GET with params +template +class HTTPQueryRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory +{ +private: + IServer & server; + +public: + HTTPQueryRequestHandlerFactory(IServer & server_) : server(server_) {} Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest & request) override { - LOG_TRACE(log, "HTTP Request for " << name << ". " - << "Method: " - << request.getMethod() - << ", Address: " - << request.clientAddress().toString() - << ", User-Agent: " - << (request.has("User-Agent") ? request.get("User-Agent") : "none") - << (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : ("")) - << ", Content Type: " << request.getContentType() - << ", Transfer Encoding: " << request.getTransferEncoding()); + if (request.getURI().find('?') != std::string::npos || request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST) + return new HandleType(server); + return nullptr; + } +}; - const auto & uri = request.getURI(); - if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET || request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD) - { - if (uri == "/") - return new RootRequestHandler(server); - if (uri == "/ping") - return new PingRequestHandler(server); - else if (startsWith(uri, "/replicas_status")) - return new ReplicasStatusHandler(server.context()); - } +/// Handle GET or HEAD endpoint on specified path +template +class HTTPGetRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory +{ +private: + IServer & server; +public: + HTTPGetRequestHandlerFactory(IServer & server_) : server(server_) {} - if (uri.find('?') != std::string::npos || request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST) - { - return new HandlerType(server); - } + Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest & request) override + { + auto & method = request.getMethod(); + if (!(method == Poco::Net::HTTPRequest::HTTP_GET || method == Poco::Net::HTTPRequest::HTTP_HEAD)) + return nullptr; - if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET || request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD - || request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST) - { - return new NotFoundHandler; - } + auto & uri = request.getURI(); + bool uri_match = TGetEndpoint::strict_path ? uri == TGetEndpoint::path : startsWith(uri, TGetEndpoint::path); + if (uri_match) + return new typename TGetEndpoint::HandleType(server); return nullptr; } }; -using HTTPHandlerFactory = HTTPRequestHandlerFactory; -using InterserverIOHTTPHandlerFactory = HTTPRequestHandlerFactory; + +struct RootEndpoint +{ + static constexpr auto path = "/"; + static constexpr auto strict_path = true; + using HandleType = RootRequestHandler; +}; + +struct PingEndpoint +{ + static constexpr auto path = "/ping"; + static constexpr auto strict_path = true; + using HandleType = PingRequestHandler; +}; + +struct ReplicasStatusEndpoint +{ + static constexpr auto path = "/replicas_status"; + static constexpr auto strict_path = false; + using HandleType = ReplicasStatusHandler; +}; + +using HTTPRootRequestHandlerFactory = HTTPGetRequestHandlerFactory; +using HTTPPingRequestHandlerFactory = HTTPGetRequestHandlerFactory; +using HTTPReplicasStatusRequestHandlerFactory = HTTPGetRequestHandlerFactory; + +template +HTTPRequestHandlerFactoryMain * createDefaultHandlerFatory(IServer & server, const std::string & name) +{ + auto handlerFactory = new HTTPRequestHandlerFactoryMain(server, name); + handlerFactory->addHandler() + ->addHandler() + ->addHandler() + ->addHandler>(); + return handlerFactory; +} + } diff --git a/dbms/programs/server/MySQLHandler.cpp b/dbms/programs/server/MySQLHandler.cpp index f7429ebf2a7..a147ccafba0 100644 --- a/dbms/programs/server/MySQLHandler.cpp +++ b/dbms/programs/server/MySQLHandler.cpp @@ -1,7 +1,6 @@ #include -#if USE_SSL -#include "MySQLHandler.h" +#include "MySQLHandler.h" #include #include #include @@ -15,37 +14,40 @@ #include #include #include -#include -#include +#include +#include + +#if USE_POCO_NETSSL #include #include -#include - +#include +#include +#endif namespace DB { using namespace MySQLProtocol; - +#if USE_POCO_NETSSL using Poco::Net::SecureStreamSocket; using Poco::Net::SSLManager; - +#endif namespace ErrorCodes { extern const int MYSQL_CLIENT_INSUFFICIENT_CAPABILITIES; extern const int OPENSSL_ERROR; + extern const int SUPPORT_IS_DISABLED; } -MySQLHandler::MySQLHandler(IServer & server_, const Poco::Net::StreamSocket & socket_, RSA & public_key_, RSA & private_key_, bool ssl_enabled, size_t connection_id_) +MySQLHandler::MySQLHandler(IServer & server_, const Poco::Net::StreamSocket & socket_, + bool ssl_enabled, size_t connection_id_) : Poco::Net::TCPServerConnection(socket_) , server(server_) , log(&Poco::Logger::get("MySQLHandler")) , connection_context(server.context()) , connection_id(connection_id_) - , public_key(public_key_) - , private_key(private_key_) , auth_plugin(new MySQLProtocol::Authentication::Native41()) { server_capability_flags = CLIENT_PROTOCOL_41 | CLIENT_SECURE_CONNECTION | CLIENT_PLUGIN_AUTH | CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA | CLIENT_CONNECT_WITH_DB | CLIENT_DEPRECATE_EOF; @@ -197,21 +199,7 @@ void MySQLHandler::finishHandshake(MySQLProtocol::HandshakeResponse & packet) if (payload_size == SSL_REQUEST_PAYLOAD_SIZE) { - read_bytes(packet_size); /// Reading rest SSLRequest. - SSLRequest ssl_request; - ReadBufferFromMemory payload(buf, pos); - payload.ignore(PACKET_HEADER_SIZE); - ssl_request.readPayload(payload); - connection_context.mysql.client_capabilities = ssl_request.capability_flags; - connection_context.mysql.max_packet_size = ssl_request.max_packet_size ? ssl_request.max_packet_size : MAX_PACKET_LENGTH; - secure_connection = true; - ss = std::make_shared(SecureStreamSocket::attach(socket(), SSLManager::instance().defaultServerContext())); - in = std::make_shared(*ss); - out = std::make_shared(*ss); - connection_context.mysql.sequence_id = 2; - packet_sender = std::make_shared(*in, *out, connection_context.mysql.sequence_id); - packet_sender->max_packet_size = connection_context.mysql.max_packet_size; - packet_sender->receivePacket(packet); /// Reading HandshakeResponse from secure socket. + finishHandshakeSSL(packet_size, buf, pos, read_bytes, packet); } else { @@ -229,14 +217,18 @@ void MySQLHandler::finishHandshake(MySQLProtocol::HandshakeResponse & packet) void MySQLHandler::authenticate(const String & user_name, const String & auth_plugin_name, const String & initial_auth_response) { - // For compatibility with JavaScript MySQL client, Native41 authentication plugin is used when possible (if password is specified using double SHA1). Otherwise SHA256 plugin is used. - auto user = connection_context.getUser(user_name); - if (user->authentication.getType() != DB::Authentication::DOUBLE_SHA1_PASSWORD) - auth_plugin = std::make_unique(public_key, private_key, log); + try + { + // For compatibility with JavaScript MySQL client, Native41 authentication plugin is used when possible (if password is specified using double SHA1). Otherwise SHA256 plugin is used. + auto user = connection_context.getUser(user_name); + const DB::Authentication::Type user_auth_type = user->authentication.getType(); + if (user_auth_type != DB::Authentication::DOUBLE_SHA1_PASSWORD && user_auth_type != DB::Authentication::PLAINTEXT_PASSWORD && user_auth_type != DB::Authentication::NO_PASSWORD) + { + authPluginSSL(); + } - try { std::optional auth_response = auth_plugin_name == auth_plugin->getName() ? std::make_optional(initial_auth_response) : std::nullopt; - auth_plugin->authenticate(user_name, auth_response, connection_context, packet_sender, secure_connection, socket().address()); + auth_plugin->authenticate(user_name, auth_response, connection_context, packet_sender, secure_connection, socket().peerAddress()); } catch (const Exception & exc) { @@ -277,30 +269,121 @@ void MySQLHandler::comPing() packet_sender->sendPacket(OK_Packet(0x0, client_capability_flags, 0, 0, 0), true); } +static bool isFederatedServerSetupCommand(const String & query); + void MySQLHandler::comQuery(ReadBuffer & payload) { - bool with_output = false; - std::function set_content_type = [&with_output](const String &) -> void { - with_output = true; - }; + String query = String(payload.position(), payload.buffer().end()); - const String query("select ''"); - ReadBufferFromString empty_select(query); - - bool should_replace = false; - // Translate query from MySQL to ClickHouse. - // This is a temporary workaround until ClickHouse supports the syntax "@@var_name". - if (std::string(payload.position(), payload.buffer().end()) == "select @@version_comment limit 1") // MariaDB client starts session with that query + // This is a workaround in order to support adding ClickHouse to MySQL using federated server. + // As Clickhouse doesn't support these statements, we just send OK packet in response. + if (isFederatedServerSetupCommand(query)) { - should_replace = true; - } - - Context query_context = connection_context; - executeQuery(should_replace ? empty_select : payload, *out, true, query_context, set_content_type, nullptr); - - if (!with_output) packet_sender->sendPacket(OK_Packet(0x00, client_capability_flags, 0, 0, 0), true); + } + else + { + bool with_output = false; + std::function set_content_type = [&with_output](const String &) -> void { + with_output = true; + }; + + String replacement_query = "select ''"; + bool should_replace = false; + + // Translate query from MySQL to ClickHouse. + // This is a temporary workaround until ClickHouse supports the syntax "@@var_name". + if (query == "select @@version_comment limit 1") // MariaDB client starts session with that query + { + should_replace = true; + } + // This is a workaround in order to support adding ClickHouse to MySQL using federated server. + if (0 == strncasecmp("SHOW TABLE STATUS LIKE", query.c_str(), 22)) + { + should_replace = true; + replacement_query = boost::replace_all_copy(query, "SHOW TABLE STATUS LIKE ", show_table_status_replacement_query); + } + + ReadBufferFromString replacement(replacement_query); + + Context query_context = connection_context; + executeQuery(should_replace ? replacement : payload, *out, true, query_context, set_content_type, nullptr); + + if (!with_output) + packet_sender->sendPacket(OK_Packet(0x00, client_capability_flags, 0, 0, 0), true); + } } +void MySQLHandler::authPluginSSL() +{ + throw Exception("ClickHouse was built without SSL support. Try specifying password using double SHA1 in users.xml.", ErrorCodes::SUPPORT_IS_DISABLED); } + +void MySQLHandler::finishHandshakeSSL([[maybe_unused]] size_t packet_size, [[maybe_unused]] char * buf, [[maybe_unused]] size_t pos, [[maybe_unused]] std::function read_bytes, [[maybe_unused]] MySQLProtocol::HandshakeResponse & packet) +{ + throw Exception("Client requested SSL, while it is disabled.", ErrorCodes::SUPPORT_IS_DISABLED); +} + +#if USE_SSL && USE_POCO_NETSSL +MySQLHandlerSSL::MySQLHandlerSSL(IServer & server_, const Poco::Net::StreamSocket & socket_, bool ssl_enabled, size_t connection_id_, RSA & public_key_, RSA & private_key_) + : MySQLHandler(server_, socket_, ssl_enabled, connection_id_) + , public_key(public_key_) + , private_key(private_key_) +{} + +void MySQLHandlerSSL::authPluginSSL() +{ + auth_plugin = std::make_unique(public_key, private_key, log); +} + +void MySQLHandlerSSL::finishHandshakeSSL(size_t packet_size, char * buf, size_t pos, std::function read_bytes, MySQLProtocol::HandshakeResponse & packet) +{ + read_bytes(packet_size); /// Reading rest SSLRequest. + SSLRequest ssl_request; + ReadBufferFromMemory payload(buf, pos); + payload.ignore(PACKET_HEADER_SIZE); + ssl_request.readPayload(payload); + connection_context.mysql.client_capabilities = ssl_request.capability_flags; + connection_context.mysql.max_packet_size = ssl_request.max_packet_size ? ssl_request.max_packet_size : MAX_PACKET_LENGTH; + secure_connection = true; + ss = std::make_shared(SecureStreamSocket::attach(socket(), SSLManager::instance().defaultServerContext())); + in = std::make_shared(*ss); + out = std::make_shared(*ss); + connection_context.mysql.sequence_id = 2; + packet_sender = std::make_shared(*in, *out, connection_context.mysql.sequence_id); + packet_sender->max_packet_size = connection_context.mysql.max_packet_size; + packet_sender->receivePacket(packet); /// Reading HandshakeResponse from secure socket. +} + #endif + +static bool isFederatedServerSetupCommand(const String & query) +{ + return 0 == strncasecmp("SET NAMES", query.c_str(), 9) || 0 == strncasecmp("SET character_set_results", query.c_str(), 25) + || 0 == strncasecmp("SET FOREIGN_KEY_CHECKS", query.c_str(), 22) || 0 == strncasecmp("SET AUTOCOMMIT", query.c_str(), 14) + || 0 == strncasecmp("SET SESSION TRANSACTION ISOLATION LEVEL", query.c_str(), 39); +} + +const String MySQLHandler::show_table_status_replacement_query("SELECT" + " name AS Name," + " engine AS Engine," + " '10' AS Version," + " 'Dynamic' AS Row_format," + " 0 AS Rows," + " 0 AS Avg_row_length," + " 0 AS Data_length," + " 0 AS Max_data_length," + " 0 AS Index_length," + " 0 AS Data_free," + " 'NULL' AS Auto_increment," + " metadata_modification_time AS Create_time," + " metadata_modification_time AS Update_time," + " metadata_modification_time AS Check_time," + " 'utf8_bin' AS Collation," + " 'NULL' AS Checksum," + " '' AS Create_options," + " '' AS Comment" + " FROM system.tables" + " WHERE name LIKE "); + +} diff --git a/dbms/programs/server/MySQLHandler.h b/dbms/programs/server/MySQLHandler.h index 9d51667925d..42629470632 100644 --- a/dbms/programs/server/MySQLHandler.h +++ b/dbms/programs/server/MySQLHandler.h @@ -1,22 +1,21 @@ #pragma once #include -#if USE_SSL - #include -#include #include #include #include "IServer.h" +#if USE_POCO_NETSSL +#include +#endif namespace DB { - /// Handler for MySQL wire protocol connections. Allows to connect to ClickHouse using MySQL client. class MySQLHandler : public Poco::Net::TCPServerConnection { public: - MySQLHandler(IServer & server_, const Poco::Net::StreamSocket & socket_, RSA & public_key_, RSA & private_key_, bool ssl_enabled, size_t connection_id_); + MySQLHandler(IServer & server_, const Poco::Net::StreamSocket & socket_, bool ssl_enabled, size_t connection_id_); void run() final; @@ -34,28 +33,50 @@ private: void authenticate(const String & user_name, const String & auth_plugin_name, const String & auth_response); + virtual void authPluginSSL(); + virtual void finishHandshakeSSL(size_t packet_size, char * buf, size_t pos, std::function read_bytes, MySQLProtocol::HandshakeResponse & packet); + IServer & server; + +protected: Poco::Logger * log; + Context connection_context; std::shared_ptr packet_sender; +private: size_t connection_id = 0; size_t server_capability_flags = 0; size_t client_capability_flags = 0; - RSA & public_key; - RSA & private_key; - +protected: std::unique_ptr auth_plugin; - std::shared_ptr ss; std::shared_ptr in; std::shared_ptr out; bool secure_connection = false; + +private: + static const String show_table_status_replacement_query; }; -} +#if USE_SSL && USE_POCO_NETSSL +class MySQLHandlerSSL : public MySQLHandler +{ +public: + MySQLHandlerSSL(IServer & server_, const Poco::Net::StreamSocket & socket_, bool ssl_enabled, size_t connection_id_, RSA & public_key_, RSA & private_key_); + +private: + void authPluginSSL() override; + void finishHandshakeSSL(size_t packet_size, char * buf, size_t pos, std::function read_bytes, MySQLProtocol::HandshakeResponse & packet) override; + + RSA & public_key; + RSA & private_key; + std::shared_ptr ss; +}; #endif + +} diff --git a/dbms/programs/server/MySQLHandlerFactory.cpp b/dbms/programs/server/MySQLHandlerFactory.cpp index 752d88e05d8..987efbfa347 100644 --- a/dbms/programs/server/MySQLHandlerFactory.cpp +++ b/dbms/programs/server/MySQLHandlerFactory.cpp @@ -1,7 +1,5 @@ #include "MySQLHandlerFactory.h" -#if USE_POCO_NETSSL && USE_SSL #include -#include #include #include #include @@ -9,6 +7,10 @@ #include "IServer.h" #include "MySQLHandler.h" +#if USE_POCO_NETSSL +#include +#endif + namespace DB { @@ -24,6 +26,8 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_) : server(server_) , log(&Logger::get("MySQLHandlerFactory")) { + +#if USE_POCO_NETSSL try { Poco::Net::SSLManager::instance().defaultServerContext(); @@ -33,7 +37,9 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_) LOG_INFO(log, "Failed to create SSL context. SSL will be disabled. Error: " << getCurrentExceptionMessage(false)); ssl_enabled = false; } +#endif +#if USE_SSL /// Reading rsa keys for SHA256 authentication plugin. try { @@ -44,8 +50,10 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_) LOG_WARNING(log, "Failed to read RSA keys. Error: " << getCurrentExceptionMessage(false)); generateRSAKeys(); } +#endif } +#if USE_SSL void MySQLHandlerFactory::readRSAKeys() { const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config(); @@ -113,13 +121,18 @@ void MySQLHandlerFactory::generateRSAKeys() if (!private_key) throw Exception("Failed to copy RSA key. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR); } +#endif Poco::Net::TCPServerConnection * MySQLHandlerFactory::createConnection(const Poco::Net::StreamSocket & socket) { size_t connection_id = last_connection_id++; LOG_TRACE(log, "MySQL connection. Id: " << connection_id << ". Address: " << socket.peerAddress().toString()); - return new MySQLHandler(server, socket, *public_key, *private_key, ssl_enabled, connection_id); +#if USE_POCO_NETSSL && USE_SSL + return new MySQLHandlerSSL(server, socket, ssl_enabled, connection_id, *public_key, *private_key); +#else + return new MySQLHandler(server, socket, ssl_enabled, connection_id); +#endif + } } -#endif diff --git a/dbms/programs/server/MySQLHandlerFactory.h b/dbms/programs/server/MySQLHandlerFactory.h index 78022ddce37..b7df9fa60e0 100644 --- a/dbms/programs/server/MySQLHandlerFactory.h +++ b/dbms/programs/server/MySQLHandlerFactory.h @@ -1,12 +1,12 @@ #pragma once #include -#if USE_POCO_NETSSL && USE_SSL - #include #include -#include #include "IServer.h" +#if USE_SSL +#include +#endif namespace DB { @@ -17,6 +17,7 @@ private: IServer & server; Poco::Logger * log; +#if USE_SSL struct RSADeleter { void operator()(RSA * ptr) { RSA_free(ptr); } @@ -27,6 +28,9 @@ private: RSAPtr private_key; bool ssl_enabled = true; +#else + bool ssl_enabled = false; +#endif std::atomic last_connection_id = 0; public: @@ -40,4 +44,3 @@ public: }; } -#endif diff --git a/dbms/programs/server/PrometheusMetricsWriter.cpp b/dbms/programs/server/PrometheusMetricsWriter.cpp new file mode 100644 index 00000000000..b45e66f7056 --- /dev/null +++ b/dbms/programs/server/PrometheusMetricsWriter.cpp @@ -0,0 +1,90 @@ +#include "PrometheusMetricsWriter.h" + +#include + +namespace +{ + +template +void writeOutLine(DB::WriteBuffer & wb, T && val) +{ + DB::writeText(std::forward(val), wb); + DB::writeChar('\n', wb); +} + +template +void writeOutLine(DB::WriteBuffer & wb, T && val, TArgs &&... args) +{ + DB::writeText(std::forward(val), wb); + DB::writeChar(' ', wb); + writeOutLine(wb, std::forward(args)...); +} + +} + + +namespace DB +{ + +PrometheusMetricsWriter::PrometheusMetricsWriter( + const Poco::Util::AbstractConfiguration & config, const std::string & config_name, + const AsynchronousMetrics & async_metrics_) + : async_metrics(async_metrics_) + , send_events(config.getBool(config_name + ".events", true)) + , send_metrics(config.getBool(config_name + ".metrics", true)) + , send_asynchronous_metrics(config.getBool(config_name + ".asynchronous_metrics", true)) +{ +} + +void PrometheusMetricsWriter::write(WriteBuffer & wb) const +{ + if (send_events) + { + for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i) + { + const auto counter = ProfileEvents::global_counters[i].load(std::memory_order_relaxed); + + std::string metric_name{ProfileEvents::getName(static_cast(i))}; + std::string metric_doc{ProfileEvents::getDocumentation(static_cast(i))}; + + std::string key{profile_events_prefix + metric_name}; + + writeOutLine(wb, "# HELP", key, metric_doc); + writeOutLine(wb, "# TYPE", key, "counter"); + writeOutLine(wb, key, counter); + } + } + + if (send_metrics) + { + for (size_t i = 0, end = CurrentMetrics::end(); i < end; ++i) + { + const auto value = CurrentMetrics::values[i].load(std::memory_order_relaxed); + + std::string metric_name{CurrentMetrics::getName(static_cast(i))}; + std::string metric_doc{CurrentMetrics::getDocumentation(static_cast(i))}; + + std::string key{current_metrics_prefix + metric_name}; + + writeOutLine(wb, "# HELP", key, metric_doc); + writeOutLine(wb, "# TYPE", key, "gauge"); + writeOutLine(wb, key, value); + } + } + + if (send_asynchronous_metrics) + { + auto async_metrics_values = async_metrics.getValues(); + for (const auto & name_value : async_metrics_values) + { + std::string key{asynchronous_metrics_prefix + name_value.first}; + auto value = name_value.second; + + // TODO: add HELP section? asynchronous_metrics contains only key and value + writeOutLine(wb, "# TYPE", key, "gauge"); + writeOutLine(wb, key, value); + } + } +} + +} diff --git a/dbms/programs/server/PrometheusMetricsWriter.h b/dbms/programs/server/PrometheusMetricsWriter.h new file mode 100644 index 00000000000..1b253b2b954 --- /dev/null +++ b/dbms/programs/server/PrometheusMetricsWriter.h @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include + +#include + +#include + +namespace DB +{ + +/// Write metrics in Prometheus format +class PrometheusMetricsWriter +{ +public: + PrometheusMetricsWriter( + const Poco::Util::AbstractConfiguration & config, const std::string & config_name, + const AsynchronousMetrics & async_metrics_); + + void write(WriteBuffer & wb) const; + +private: + const AsynchronousMetrics & async_metrics; + + const bool send_events; + const bool send_metrics; + const bool send_asynchronous_metrics; + + static inline constexpr auto profile_events_prefix = "ClickHouseProfileEvents"; + static inline constexpr auto current_metrics_prefix = "ClickHouseMetrics"; + static inline constexpr auto asynchronous_metrics_prefix = "ClickHouseAsyncMetrics"; +}; + +} diff --git a/dbms/programs/server/PrometheusRequestHandler.cpp b/dbms/programs/server/PrometheusRequestHandler.cpp new file mode 100644 index 00000000000..b5a48d13b64 --- /dev/null +++ b/dbms/programs/server/PrometheusRequestHandler.cpp @@ -0,0 +1,42 @@ +#include "PrometheusRequestHandler.h" + +#include + +#include + +#include +#include + +#include +#include + +#include + + +namespace DB +{ + +void PrometheusRequestHandler::handleRequest( + Poco::Net::HTTPServerRequest & request, + Poco::Net::HTTPServerResponse & response) +{ + try + { + const auto & config = server.config(); + unsigned keep_alive_timeout = config.getUInt("keep_alive_timeout", 10); + + setResponseDefaultHeaders(response, keep_alive_timeout); + + response.setContentType("text/plain; version=0.0.4; charset=UTF-8"); + + auto wb = WriteBufferFromHTTPServerResponse(request, response, keep_alive_timeout); + metrics_writer.write(wb); + wb.finalize(); + } + catch (...) + { + tryLogCurrentException("PrometheusRequestHandler"); + } +} + +} diff --git a/dbms/programs/server/PrometheusRequestHandler.h b/dbms/programs/server/PrometheusRequestHandler.h new file mode 100644 index 00000000000..439a01c7d6f --- /dev/null +++ b/dbms/programs/server/PrometheusRequestHandler.h @@ -0,0 +1,61 @@ +#pragma once + +#include "IServer.h" +#include "PrometheusMetricsWriter.h" + +#include +#include +#include +#include + +namespace DB +{ + +class PrometheusRequestHandler : public Poco::Net::HTTPRequestHandler +{ +private: + IServer & server; + const PrometheusMetricsWriter & metrics_writer; + +public: + explicit PrometheusRequestHandler(IServer & server_, PrometheusMetricsWriter & metrics_writer_) + : server(server_) + , metrics_writer(metrics_writer_) + { + } + + void handleRequest( + Poco::Net::HTTPServerRequest & request, + Poco::Net::HTTPServerResponse & response) override; +}; + + +template +class PrometeusRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory +{ +private: + IServer & server; + std::string endpoint_path; + PrometheusMetricsWriter metrics_writer; + +public: + PrometeusRequestHandlerFactory(IServer & server_, const AsynchronousMetrics & async_metrics_) + : server(server_) + , endpoint_path(server_.config().getString("prometheus.endpoint", "/metrics")) + , metrics_writer(server_.config(), "prometheus", async_metrics_) + { + } + + Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest & request) override + { + if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET + && request.getURI() == endpoint_path) + return new HandlerType(server, metrics_writer); + + return nullptr; + } +}; + +using PrometeusHandlerFactory = PrometeusRequestHandlerFactory; + +} diff --git a/dbms/programs/server/ReplicasStatusHandler.cpp b/dbms/programs/server/ReplicasStatusHandler.cpp index 22e059d9ccc..4d72c6da3cf 100644 --- a/dbms/programs/server/ReplicasStatusHandler.cpp +++ b/dbms/programs/server/ReplicasStatusHandler.cpp @@ -15,8 +15,8 @@ namespace DB { -ReplicasStatusHandler::ReplicasStatusHandler(Context & context_) - : context(context_) +ReplicasStatusHandler::ReplicasStatusHandler(IServer & server) + : context(server.context()) { } diff --git a/dbms/programs/server/ReplicasStatusHandler.h b/dbms/programs/server/ReplicasStatusHandler.h index 02759bbc8ac..479b013cfe8 100644 --- a/dbms/programs/server/ReplicasStatusHandler.h +++ b/dbms/programs/server/ReplicasStatusHandler.h @@ -1,5 +1,7 @@ #pragma once +#include "IServer.h" + #include @@ -15,7 +17,7 @@ private: Context & context; public: - explicit ReplicasStatusHandler(Context & context_); + explicit ReplicasStatusHandler(IServer & server); void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; }; diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index e274dca6bdc..ae1d995d363 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -50,6 +50,7 @@ #include #include #include +#include #include #include "HTTPHandlerFactory.h" #include "MetricsTransmitter.h" @@ -57,7 +58,7 @@ #include "TCPHandlerFactory.h" #include "Common/config_version.h" #include - +#include "MySQLHandlerFactory.h" #if defined(OS_LINUX) #include @@ -65,7 +66,6 @@ #endif #if USE_POCO_NETSSL -#include "MySQLHandlerFactory.h" #include #include #endif @@ -188,6 +188,7 @@ int Server::main(const std::vector & /*args*/) registerTableFunctions(); registerStorages(); registerDictionaries(); + registerDisks(); CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get()); CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger()); @@ -244,6 +245,8 @@ int Server::main(const std::vector & /*args*/) } #endif + global_context->setRemoteHostFilter(config()); + std::string path = getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH)); std::string default_database = config().getString("default_database", "default"); @@ -372,6 +375,12 @@ int Server::main(const std::vector & /*args*/) Poco::File(user_files_path).createDirectories(); } + { + std::string dictionaries_lib_path = config().getString("dictionaries_lib_path", path + "dictionaries_lib/"); + global_context->setDictionariesLibPath(dictionaries_lib_path); + Poco::File(dictionaries_lib_path).createDirectories(); + } + if (config().has("interserver_http_port") && config().has("interserver_https_port")) throw Exception("Both http and https interserver ports are specified", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); @@ -439,6 +448,13 @@ int Server::main(const std::vector & /*args*/) buildLoggers(*config, logger()); global_context->setClustersConfig(config); global_context->setMacros(std::make_unique(*config, "macros")); + + /// Setup protection to avoid accidental DROP for big tables (that are greater than 50 GB by default) + if (config->has("max_table_size_to_drop")) + global_context->setMaxTableSizeToDrop(config->getUInt64("max_table_size_to_drop")); + + if (config->has("max_partition_size_to_drop")) + global_context->setMaxPartitionSizeToDrop(config->getUInt64("max_partition_size_to_drop")); }, /* already_loaded = */ true); @@ -470,13 +486,6 @@ int Server::main(const std::vector & /*args*/) /// Limit on total number of concurrently executed queries. global_context->getProcessList().setMaxSize(config().getInt("max_concurrent_queries", 0)); - /// Setup protection to avoid accidental DROP for big tables (that are greater than 50 GB by default) - if (config().has("max_table_size_to_drop")) - global_context->setMaxTableSizeToDrop(config().getUInt64("max_table_size_to_drop")); - - if (config().has("max_partition_size_to_drop")) - global_context->setMaxPartitionSizeToDrop(config().getUInt64("max_partition_size_to_drop")); - /// Set up caches. /// Lower cache size on low-memory systems. @@ -669,6 +678,10 @@ int Server::main(const std::vector & /*args*/) return address; }; + /// This object will periodically calculate some metrics. + AsynchronousMetrics async_metrics(*global_context); + attachSystemTablesAsync(*global_context->getDatabase("system"), async_metrics); + for (const auto & listen_host : listen_hosts) { auto create_server = [&](const char * port_name, auto && func) @@ -708,13 +721,17 @@ int Server::main(const std::vector & /*args*/) auto address = socket_bind_listen(socket, listen_host, port); socket.setReceiveTimeout(settings.http_receive_timeout); socket.setSendTimeout(settings.http_send_timeout); + auto handler_factory = createDefaultHandlerFatory(*this, "HTTPHandler-factory"); + if (config().has("prometheus") && config().getInt("prometheus.port", 0) == 0) + handler_factory->addHandler(async_metrics); + servers.emplace_back(std::make_unique( - new HTTPHandlerFactory(*this, "HTTPHandler-factory"), + handler_factory, server_pool, socket, http_params)); - LOG_INFO(log, "Listening http://" + address.toString()); + LOG_INFO(log, "Listening for http://" + address.toString()); }); /// HTTPS @@ -726,12 +743,12 @@ int Server::main(const std::vector & /*args*/) socket.setReceiveTimeout(settings.http_receive_timeout); socket.setSendTimeout(settings.http_send_timeout); servers.emplace_back(std::make_unique( - new HTTPHandlerFactory(*this, "HTTPSHandler-factory"), + createDefaultHandlerFatory(*this, "HTTPSHandler-factory"), server_pool, socket, http_params)); - LOG_INFO(log, "Listening https://" + address.toString()); + LOG_INFO(log, "Listening for https://" + address.toString()); #else UNUSED(port); throw Exception{"HTTPS protocol is disabled because Poco library was built without NetSSL support.", @@ -784,12 +801,12 @@ int Server::main(const std::vector & /*args*/) socket.setReceiveTimeout(settings.http_receive_timeout); socket.setSendTimeout(settings.http_send_timeout); servers.emplace_back(std::make_unique( - new InterserverIOHTTPHandlerFactory(*this, "InterserverIOHTTPHandler-factory"), + createDefaultHandlerFatory(*this, "InterserverIOHTTPHandler-factory"), server_pool, socket, http_params)); - LOG_INFO(log, "Listening for replica communication (interserver) http://" + address.toString()); + LOG_INFO(log, "Listening for replica communication (interserver): http://" + address.toString()); }); create_server("interserver_https_port", [&](UInt16 port) @@ -800,12 +817,12 @@ int Server::main(const std::vector & /*args*/) socket.setReceiveTimeout(settings.http_receive_timeout); socket.setSendTimeout(settings.http_send_timeout); servers.emplace_back(std::make_unique( - new InterserverIOHTTPHandlerFactory(*this, "InterserverIOHTTPHandler-factory"), + createDefaultHandlerFatory(*this, "InterserverIOHTTPHandler-factory"), server_pool, socket, http_params)); - LOG_INFO(log, "Listening for secure replica communication (interserver) https://" + address.toString()); + LOG_INFO(log, "Listening for secure replica communication (interserver): https://" + address.toString()); #else UNUSED(port); throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.", @@ -815,7 +832,6 @@ int Server::main(const std::vector & /*args*/) create_server("mysql_port", [&](UInt16 port) { -#if USE_SSL Poco::Net::ServerSocket socket; auto address = socket_bind_listen(socket, listen_host, port, /* secure = */ true); socket.setReceiveTimeout(Poco::Timespan()); @@ -827,11 +843,24 @@ int Server::main(const std::vector & /*args*/) new Poco::Net::TCPServerParams)); LOG_INFO(log, "Listening for MySQL compatibility protocol: " + address.toString()); -#else - UNUSED(port); - throw Exception{"SSL support for MySQL protocol is disabled because Poco library was built without NetSSL support.", - ErrorCodes::SUPPORT_IS_DISABLED}; -#endif + }); + + /// Prometheus (if defined and not setup yet with http_port) + create_server("prometheus.port", [&](UInt16 port) + { + Poco::Net::ServerSocket socket; + auto address = socket_bind_listen(socket, listen_host, port); + socket.setReceiveTimeout(settings.http_receive_timeout); + socket.setSendTimeout(settings.http_send_timeout); + auto handler_factory = new HTTPRequestHandlerFactoryMain(*this, "PrometheusHandler-factory"); + handler_factory->addHandler(async_metrics); + servers.emplace_back(std::make_unique( + handler_factory, + server_pool, + socket, + http_params)); + + LOG_INFO(log, "Listening for Prometheus: http://" + address.toString()); }); } @@ -935,10 +964,6 @@ int Server::main(const std::vector & /*args*/) throw; } - /// This object will periodically calculate some metrics. - AsynchronousMetrics async_metrics(*global_context); - attachSystemTablesAsync(*global_context->getDatabase("system"), async_metrics); - std::vector> metrics_transmitters; for (const auto & graphite_key : DB::getMultipleKeysFromConfig(config(), "", "graphite")) { @@ -955,6 +980,9 @@ int Server::main(const std::vector & /*args*/) } } +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wmissing-declarations" + int mainEntryClickHouseServer(int argc, char ** argv) { DB::Server app; diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 88b1eb6ae3e..cb215eb0af8 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -111,7 +110,7 @@ void TCPHandler::runImpl() { if (!connection_context.isDatabaseExist(default_database)) { - Exception e("Database " + default_database + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); + Exception e("Database " + backQuote(default_database) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); LOG_ERROR(log, "Code: " << e.code() << ", e.displayText() = " << e.displayText() << ", Stack trace:\n\n" << e.getStackTrace().toString()); sendException(e, connection_context.getSettingsRef().calculate_text_stack_trace); @@ -201,6 +200,8 @@ void TCPHandler::runImpl() /// So, the stream has been marked as cancelled and we can't read from it anymore. state.block_in.reset(); state.maybe_compressed_in.reset(); /// For more accurate accounting by MemoryTracker. + + state.temporary_tables_read = true; }); /// Send structure of columns to client for function input() @@ -340,6 +341,18 @@ void TCPHandler::runImpl() LOG_WARNING(log, "Client has gone away."); } + try + { + if (exception && !state.temporary_tables_read) + query_context->initializeExternalTablesIfSet(); + } + catch (...) + { + network_error = true; + LOG_WARNING(log, "Can't read external tables after query failure."); + } + + try { query_scope.reset(); @@ -471,76 +484,63 @@ void TCPHandler::processOrdinaryQuery() /// Pull query execution result, if exists, and send it to network. if (state.io.in) { - /// Send header-block, to allow client to prepare output format for data to send. - { - Block header = state.io.in->getHeader(); - - if (header) - sendData(header); - } + /// This allows the client to prepare output format + if (Block header = state.io.in->getHeader()) + sendData(header); + /// Use of async mode here enables reporting progress and monitoring client cancelling the query AsynchronousBlockInputStream async_in(state.io.in); - async_in.readPrefix(); + async_in.readPrefix(); while (true) { - Block block; - - while (true) + if (isQueryCancelled()) { - if (isQueryCancelled()) - { - /// A packet was received requesting to stop execution of the request. - async_in.cancel(false); - break; - } - else - { - if (after_send_progress.elapsed() / 1000 >= query_context->getSettingsRef().interactive_delay) - { - /// Some time passed and there is a progress. - after_send_progress.restart(); - sendProgress(); - } - - sendLogs(); - - if (async_in.poll(query_context->getSettingsRef().interactive_delay / 1000)) - { - /// There is the following result block. - block = async_in.read(); - break; - } - } - } - - /** If data has run out, we will send the profiling data and total values to - * the last zero block to be able to use - * this information in the suffix output of stream. - * If the request was interrupted, then `sendTotals` and other methods could not be called, - * because we have not read all the data yet, - * and there could be ongoing calculations in other threads at the same time. - */ - if (!block && !isQueryCancelled()) - { - sendTotals(state.io.in->getTotals()); - sendExtremes(state.io.in->getExtremes()); - sendProfileInfo(state.io.in->getProfileInfo()); - sendProgress(); - sendLogs(); - } - - sendData(block); - if (!block) + async_in.cancel(false); break; + } + + if (after_send_progress.elapsed() / 1000 >= query_context->getSettingsRef().interactive_delay) + { + /// Some time passed and there is a progress. + after_send_progress.restart(); + sendProgress(); + } + + sendLogs(); + + if (async_in.poll(query_context->getSettingsRef().interactive_delay / 1000)) + { + const auto block = async_in.read(); + if (!block) + break; + + if (!state.io.null_format) + sendData(block); + } + } + async_in.readSuffix(); + + /** When the data has run out, we send the profiling data and totals up to the terminating empty block, + * so that this information can be used in the suffix output of stream. + * If the request has been interrupted, then sendTotals and other methods should not be called, + * because we have not read all the data. + */ + if (!isQueryCancelled()) + { + sendTotals(state.io.in->getTotals()); + sendExtremes(state.io.in->getExtremes()); + sendProfileInfo(state.io.in->getProfileInfo()); + sendProgress(); } - async_in.readSuffix(); + sendData({}); } state.io.onFinish(); } + void TCPHandler::processOrdinaryQueryWithProcessors(size_t num_threads) { auto & pipeline = state.io.pipeline; @@ -919,8 +919,15 @@ void TCPHandler::receiveQuery() } /// Per query settings. + Settings custom_settings{}; + auto settings_format = (client_revision >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) ? SettingsBinaryFormat::STRINGS + : SettingsBinaryFormat::OLD; + custom_settings.deserialize(*in, settings_format); + auto settings_changes = custom_settings.changes(); + query_context->checkSettingsConstraints(settings_changes); + query_context->applySettingsChanges(settings_changes); + Settings & settings = query_context->getSettingsRef(); - settings.deserialize(*in); /// Sync timeouts on client and server during current query to avoid dangling queries on server /// NOTE: We use settings.send_timeout for the receive timeout and vice versa (change arguments ordering in TimeoutSetter), @@ -949,7 +956,9 @@ void TCPHandler::receiveUnexpectedQuery() skip_client_info.read(*in, client_revision); Settings & skip_settings = query_context->getSettingsRef(); - skip_settings.deserialize(*in); + auto settings_format = (client_revision >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) ? SettingsBinaryFormat::STRINGS + : SettingsBinaryFormat::OLD; + skip_settings.deserialize(*in, settings_format); readVarUInt(skip_uint_64, *in); readVarUInt(skip_uint_64, *in); diff --git a/dbms/programs/server/TCPHandler.h b/dbms/programs/server/TCPHandler.h index 561ed4d0eca..4ab9097b9bb 100644 --- a/dbms/programs/server/TCPHandler.h +++ b/dbms/programs/server/TCPHandler.h @@ -63,6 +63,8 @@ struct QueryState bool sent_all_data = false; /// Request requires data from the client (INSERT, but not INSERT SELECT). bool need_receive_data_for_insert = false; + /// Temporary tables read + bool temporary_tables_read = false; /// Request requires data from client for function input() bool need_receive_data_for_input = false; diff --git a/dbms/programs/server/config.xml b/dbms/programs/server/config.xml index 6e9bb527c97..725a2428411 100644 --- a/dbms/programs/server/config.xml +++ b/dbms/programs/server/config.xml @@ -3,6 +3,25 @@ NOTE: User and query level settings are set up in "users.xml" file. --> + + + + + + + trace @@ -15,7 +34,6 @@ 8123 9000 - + + + @@ -411,7 +447,7 @@ + + 127.0.0.1 + clientA1.com + clientA3.com + clientB\d+\.ru + clientC\d+\.ru$ + ^clientD\d+\.ru$ + + + + diff --git a/dbms/tests/integration/test_allowed_client_hosts/configs/users.xml b/dbms/tests/integration/test_allowed_client_hosts/configs/users.xml new file mode 100644 index 00000000000..3142ec5355a --- /dev/null +++ b/dbms/tests/integration/test_allowed_client_hosts/configs/users.xml @@ -0,0 +1,13 @@ + + + + + + + + + default + + + + diff --git a/dbms/tests/integration/test_allowed_client_hosts/test.py b/dbms/tests/integration/test_allowed_client_hosts/test.py new file mode 100644 index 00000000000..fcdf408c88a --- /dev/null +++ b/dbms/tests/integration/test_allowed_client_hosts/test.py @@ -0,0 +1,60 @@ +import os +import pytest +from helpers.cluster import ClickHouseCluster + + +cluster = ClickHouseCluster(__file__) +server = cluster.add_instance('server', config_dir="configs") + +clientA1 = cluster.add_instance('clientA1', hostname = 'clientA1.com') +clientA2 = cluster.add_instance('clientA2', hostname = 'clientA2.com') +clientA3 = cluster.add_instance('clientA3', hostname = 'clientA3.com') +clientB1 = cluster.add_instance('clientB1', hostname = 'clientB001.ru') +clientB2 = cluster.add_instance('clientB2', hostname = 'clientB002.ru') +clientB3 = cluster.add_instance('clientB3', hostname = 'xxx.clientB003.rutracker.com') +clientC1 = cluster.add_instance('clientC1', hostname = 'clientC01.ru') +clientC2 = cluster.add_instance('clientC2', hostname = 'xxx.clientC02.ru') +clientC3 = cluster.add_instance('clientC3', hostname = 'xxx.clientC03.rutracker.com') +clientD1 = cluster.add_instance('clientD1', hostname = 'clientD0001.ru') +clientD2 = cluster.add_instance('clientD2', hostname = 'xxx.clientD0002.ru') +clientD3 = cluster.add_instance('clientD3', hostname = 'clientD0003.ru') + + +def query_from_one_node_to_another(client_node, server_node, query): + return client_node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host {} --query {!r}".format(server_node.hostname, query)]) + + +def query(node, query): + return query_from_one_node_to_another(node, node, query) + + +@pytest.fixture(scope="module", autouse=True) +def setup_nodes(): + try: + cluster.start() + query(server, "CREATE TABLE test_table (x Int32) ENGINE = MergeTree() ORDER BY tuple()") + query(server, "INSERT INTO test_table VALUES (5)") + + yield cluster + + finally: + cluster.shutdown() + + +def test_allowed_host(): + expected_to_pass = [clientA1, clientA3] + expected_to_fail = [clientA2] + + # Reverse DNS lookup currently isn't working as expected in this test. + # For example, it gives something like "vitbartestallowedclienthosts_clientB1_1.vitbartestallowedclienthosts_default" instead of "clientB001.ru". + # Maybe we should setup the test network better. + #expected_to_pass.extend([clientB1, clientB2, clientB3, clientC1, clientC2, clientD1, clientD3]) + #expected_to_fail.extend([clientC3, clientD2]) + + for client_node in expected_to_pass: + assert query_from_one_node_to_another(client_node, server, "SELECT * FROM test_table") == "5\n" + + for client_node in expected_to_fail: + with pytest.raises(Exception) as e: + query_from_one_node_to_another(client_node, server, "SELECT * FROM test_table") + assert "User default is not allowed to connect from address" in str(e) diff --git a/dbms/tests/integration/test_allowed_url_from_config/__init__.py b/dbms/tests/integration/test_allowed_url_from_config/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_allowed_url_from_config/configs/config_for_redirect.xml b/dbms/tests/integration/test_allowed_url_from_config/configs/config_for_redirect.xml new file mode 100644 index 00000000000..eceaf7a7838 --- /dev/null +++ b/dbms/tests/integration/test_allowed_url_from_config/configs/config_for_redirect.xml @@ -0,0 +1,5 @@ + + + hdfs1:50070 + + diff --git a/dbms/tests/integration/test_allowed_url_from_config/configs/config_for_remote.xml b/dbms/tests/integration/test_allowed_url_from_config/configs/config_for_remote.xml new file mode 100644 index 00000000000..b6074fc8a2d --- /dev/null +++ b/dbms/tests/integration/test_allowed_url_from_config/configs/config_for_remote.xml @@ -0,0 +1,10 @@ + + + localhost:9000 + localhost:9440 + example01-01-1 + example01-01-2 + example01-02-1 + example01-02-2 + + diff --git a/dbms/tests/integration/test_allowed_url_from_config/configs/config_with_hosts.xml b/dbms/tests/integration/test_allowed_url_from_config/configs/config_with_hosts.xml new file mode 100644 index 00000000000..b35fa733ae6 --- /dev/null +++ b/dbms/tests/integration/test_allowed_url_from_config/configs/config_with_hosts.xml @@ -0,0 +1,7 @@ + + + host:80 + ^[a-z]*\.ru$ + + + diff --git a/dbms/tests/integration/test_allowed_url_from_config/configs/config_with_only_primary_hosts.xml b/dbms/tests/integration/test_allowed_url_from_config/configs/config_with_only_primary_hosts.xml new file mode 100644 index 00000000000..a84d864bd0d --- /dev/null +++ b/dbms/tests/integration/test_allowed_url_from_config/configs/config_with_only_primary_hosts.xml @@ -0,0 +1,8 @@ + + + host:80 + host:123 + yandex.ru + + + diff --git a/dbms/tests/integration/test_allowed_url_from_config/configs/config_with_only_regexp_hosts.xml b/dbms/tests/integration/test_allowed_url_from_config/configs/config_with_only_regexp_hosts.xml new file mode 100644 index 00000000000..b807672c2c1 --- /dev/null +++ b/dbms/tests/integration/test_allowed_url_from_config/configs/config_with_only_regexp_hosts.xml @@ -0,0 +1,7 @@ + + + ^[a-z]*:80$ + ^[a-z]*\.ru$ + + + diff --git a/dbms/tests/integration/test_allowed_url_from_config/configs/config_without_allowed_hosts.xml b/dbms/tests/integration/test_allowed_url_from_config/configs/config_without_allowed_hosts.xml new file mode 100644 index 00000000000..1855c56fc03 --- /dev/null +++ b/dbms/tests/integration/test_allowed_url_from_config/configs/config_without_allowed_hosts.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/dbms/tests/integration/test_allowed_url_from_config/test.py b/dbms/tests/integration/test_allowed_url_from_config/test.py new file mode 100644 index 00000000000..688f94cb058 --- /dev/null +++ b/dbms/tests/integration/test_allowed_url_from_config/test.py @@ -0,0 +1,74 @@ +import time +import pytest + +from helpers.hdfs_api import HDFSApi +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', main_configs=['configs/config_with_hosts.xml']) +node2 = cluster.add_instance('node2', main_configs=['configs/config_with_only_primary_hosts.xml']) +node3 = cluster.add_instance('node3', main_configs=['configs/config_with_only_regexp_hosts.xml']) +node4 = cluster.add_instance('node4', main_configs=['configs/config_without_allowed_hosts.xml']) +node6 = cluster.add_instance('node6', main_configs=['configs/config_for_remote.xml']) +node7 = cluster.add_instance('node7', main_configs=['configs/config_for_redirect.xml'], with_hdfs=True) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + +def test_config_with_hosts(start_cluster): + assert node1.query("CREATE TABLE table_test_1_1 (word String) Engine=URL('http://host:80', HDFS)") == "" + assert node1.query("CREATE TABLE table_test_1_2 (word String) Engine=URL('https://yandex.ru', CSV)") == "" + assert "not allowed" in node1.query_and_get_error("CREATE TABLE table_test_1_4 (word String) Engine=URL('https://host:123', S3)") + assert "not allowed" in node1.query_and_get_error("CREATE TABLE table_test_1_4 (word String) Engine=URL('https://yandex2.ru', CSV)") + +def test_config_with_only_primary_hosts(start_cluster): + assert node2.query("CREATE TABLE table_test_2_1 (word String) Engine=URL('https://host:80', CSV)") == "" + assert node2.query("CREATE TABLE table_test_2_2 (word String) Engine=URL('https://host:123', S3)") == "" + assert node2.query("CREATE TABLE table_test_2_3 (word String) Engine=URL('https://yandex.ru', CSV)") == "" + assert node2.query("CREATE TABLE table_test_2_4 (word String) Engine=URL('https://yandex.ru:87', HDFS)") == "" + assert "not allowed" in node2.query_and_get_error("CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host', HDFS)") + assert "not allowed" in node2.query_and_get_error("CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host:234', CSV)") + assert "not allowed" in node2.query_and_get_error("CREATE TABLE table_test_2_6 (word String) Engine=URL('https://yandex2.ru', S3)") + +def test_config_with_only_regexp_hosts(start_cluster): + assert node3.query("CREATE TABLE table_test_3_1 (word String) Engine=URL('https://host:80', HDFS)") == "" + assert node3.query("CREATE TABLE table_test_3_2 (word String) Engine=URL('https://yandex.ru', CSV)") == "" + assert "not allowed" in node3.query_and_get_error("CREATE TABLE table_test_3_3 (word String) Engine=URL('https://host', CSV)") + assert "not allowed" in node3.query_and_get_error("CREATE TABLE table_test_3_4 (word String) Engine=URL('https://yandex2.ru', S3)") + +def test_config_without_allowed_hosts(start_cluster): + assert node4.query("CREATE TABLE table_test_4_1 (word String) Engine=URL('https://host:80', CSV)") == "" + assert node4.query("CREATE TABLE table_test_4_2 (word String) Engine=URL('https://host', HDFS)") == "" + assert node4.query("CREATE TABLE table_test_4_3 (word String) Engine=URL('https://yandex.ru', CSV)") == "" + assert node4.query("CREATE TABLE table_test_4_4 (word String) Engine=URL('ftp://something.com', S3)") == "" + +def test_table_function_remote(start_cluster): + assert node6.query("SELECT * FROM remote('localhost', system, events)") != "" + assert node6.query("SELECT * FROM remoteSecure('localhost', system, metrics)") != "" + assert "URL \"localhost:800\" is not allowed in config.xml" in node6.query_and_get_error("SELECT * FROM remoteSecure('localhost:800', system, events)") + assert "URL \"localhost:800\" is not allowed in config.xml" in node6.query_and_get_error("SELECT * FROM remote('localhost:800', system, metrics)") + assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-01-1,example01-02-1', system, events)") + assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remote('example01-0{1,2}-1', system, events") + assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-01-{1|2}', system, events)") + assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remote('example01-0{1,2}-{1|2}', system, events)") + assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-{01..02}-{1|2}', system, events)") + assert "not allowed" in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-01-1,example01-03-1', system, events)") + assert "not allowed" in node6.query_and_get_error("SELECT * FROM remote('example01-01-{1|3}', system, events)") + assert "not allowed" in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-0{1,3}-1', system, metrics)") + +def test_redirect(start_cluster): + hdfs_api = HDFSApi("root") + hdfs_api.write_data("/simple_storage", "1\t\n") + assert hdfs_api.read_data("/simple_storage") == "1\t\n" + node7.query("CREATE TABLE table_test_7_1 (word String) ENGINE=URL('http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', CSV)") + assert "not allowed" in node7.query_and_get_error("SET max_http_get_redirects=1; SELECT * from table_test_7_1") + +def test_HDFS(start_cluster): + assert "not allowed" in node7.query_and_get_error("CREATE TABLE table_test_7_2 (word String) ENGINE=HDFS('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'CSV')") + assert "not allowed" in node7.query_and_get_error("SELECT * FROM hdfs('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV', 'word String')") diff --git a/dbms/tests/integration/test_backup_restore/__init__.py b/dbms/tests/integration/test_backup_restore/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_backup_restore/test.py b/dbms/tests/integration/test_backup_restore/test.py new file mode 100644 index 00000000000..c63e056414a --- /dev/null +++ b/dbms/tests/integration/test_backup_restore/test.py @@ -0,0 +1,151 @@ +import os.path +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV + + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance('instance') +q = instance.query +path_to_data = '/var/lib/clickhouse/' + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + q('CREATE DATABASE test') + + yield cluster + + finally: + cluster.shutdown() + + +def exec_bash(cmd): + cmd = '/bin/bash -c "{}"'.format(cmd.replace('"', '\\"')) + return instance.exec_in_container(cmd) + + +def copy_backup_to_detached(database, src_table, dst_table): + fp_increment = os.path.join(path_to_data, 'shadow/increment.txt') + increment = exec_bash('cat ' + fp_increment).strip() + fp_backup = os.path.join(path_to_data, 'shadow', increment, 'data', database, src_table) + fp_detached = os.path.join(path_to_data, 'data', database, dst_table, 'detached') + exec_bash('cp -r {}/* {}/'.format(fp_backup, fp_detached)) + + +@pytest.fixture +def backup_restore(started_cluster): + q("DROP TABLE IF EXISTS test.tbl") + q("CREATE TABLE test.tbl (p Date, k Int8) ENGINE = MergeTree PARTITION BY toYYYYMM(p) ORDER BY p") + for i in range(1, 4): + q('INSERT INTO test.tbl (p, k) VALUES(toDate({}), {})'.format(i, i)) + for i in range(31, 34): + q('INSERT INTO test.tbl (p, k) VALUES(toDate({}), {})'.format(i, i)) + + expected = TSV('1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33') + res = q("SELECT * FROM test.tbl ORDER BY p") + assert(TSV(res) == expected) + + q("ALTER TABLE test.tbl FREEZE") + + yield + + q("DROP TABLE IF EXISTS test.tbl") + + +def test_restore(backup_restore): + q("CREATE TABLE test.tbl1 AS test.tbl") + + copy_backup_to_detached('test', 'tbl', 'tbl1') + + # The data_version of parts to be attached are larger than the newly created table's data_version. + q("ALTER TABLE test.tbl1 ATTACH PARTITION 197001") + q("ALTER TABLE test.tbl1 ATTACH PARTITION 197002") + q("SELECT sleep(2)") + + # Validate the attached parts are identical to the backup. + expected = TSV('1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33') + res = q("SELECT * FROM test.tbl1 ORDER BY p") + assert(TSV(res) == expected) + + q("ALTER TABLE test.tbl1 UPDATE k=10 WHERE 1") + q("SELECT sleep(2)") + + # Validate mutation has been applied to all attached parts. + expected = TSV('1970-01-02\t10\n1970-01-03\t10\n1970-01-04\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10') + res = q("SELECT * FROM test.tbl1 ORDER BY p") + assert(TSV(res) == expected) + + q("DROP TABLE IF EXISTS test.tbl1") + + +def test_attach_partition(backup_restore): + q("CREATE TABLE test.tbl2 AS test.tbl") + for i in range(3, 5): + q('INSERT INTO test.tbl2(p, k) VALUES(toDate({}), {})'.format(i, i)) + for i in range(33, 35): + q('INSERT INTO test.tbl2(p, k) VALUES(toDate({}), {})'.format(i, i)) + + expected = TSV('1970-01-04\t3\n1970-01-05\t4\n1970-02-03\t33\n1970-02-04\t34') + res = q("SELECT * FROM test.tbl2 ORDER BY p") + assert(TSV(res) == expected) + + copy_backup_to_detached('test', 'tbl', 'tbl2') + + # The data_version of parts to be attached + # - may be less than, equal to or larger than the current table's data_version. + # - may intersect with the existing parts of a partition. + q("ALTER TABLE test.tbl2 ATTACH PARTITION 197001") + q("ALTER TABLE test.tbl2 ATTACH PARTITION 197002") + q("SELECT sleep(2)") + + expected = TSV('1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-01-04\t3\n1970-01-05\t4\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33\n1970-02-03\t33\n1970-02-04\t34') + res = q("SELECT * FROM test.tbl2 ORDER BY p") + assert(TSV(res) == expected) + + q("ALTER TABLE test.tbl2 UPDATE k=10 WHERE 1") + q("SELECT sleep(2)") + + # Validate mutation has been applied to all attached parts. + expected = TSV('1970-01-02\t10\n1970-01-03\t10\n1970-01-04\t10\n1970-01-04\t10\n1970-01-05\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10\n1970-02-03\t10\n1970-02-04\t10') + res = q("SELECT * FROM test.tbl2 ORDER BY p") + assert(TSV(res) == expected) + + q("DROP TABLE IF EXISTS test.tbl2") + + +def test_replace_partition(backup_restore): + q("CREATE TABLE test.tbl3 AS test.tbl") + for i in range(3, 5): + q('INSERT INTO test.tbl3(p, k) VALUES(toDate({}), {})'.format(i, i)) + for i in range(33, 35): + q('INSERT INTO test.tbl3(p, k) VALUES(toDate({}), {})'.format(i, i)) + + expected = TSV('1970-01-04\t3\n1970-01-05\t4\n1970-02-03\t33\n1970-02-04\t34') + res = q("SELECT * FROM test.tbl3 ORDER BY p") + assert(TSV(res) == expected) + + copy_backup_to_detached('test', 'tbl', 'tbl3') + + # The data_version of parts to be copied + # - may be less than, equal to or larger than the current table data_version. + # - may intersect with the existing parts of a partition. + q("ALTER TABLE test.tbl3 REPLACE PARTITION 197002 FROM test.tbl") + q("SELECT sleep(2)") + + expected = TSV('1970-01-04\t3\n1970-01-05\t4\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33') + res = q("SELECT * FROM test.tbl3 ORDER BY p") + assert(TSV(res) == expected) + + q("ALTER TABLE test.tbl3 UPDATE k=10 WHERE 1") + q("SELECT sleep(2)") + + # Validate mutation has been applied to all copied parts. + expected = TSV('1970-01-04\t10\n1970-01-05\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10') + res = q("SELECT * FROM test.tbl3 ORDER BY p") + assert(TSV(res) == expected) + + q("DROP TABLE IF EXISTS test.tbl3") diff --git a/dbms/tests/integration/test_dictionaries_ddl/configs/config.xml b/dbms/tests/integration/test_dictionaries_ddl/configs/config.xml index e24857fa806..6ecc7c089ca 100644 --- a/dbms/tests/integration/test_dictionaries_ddl/configs/config.xml +++ b/dbms/tests/integration/test_dictionaries_ddl/configs/config.xml @@ -16,4 +16,8 @@ ./clickhouse/ users.xml /etc/clickhouse-server/config.d/*.xml + + + node1 + diff --git a/dbms/tests/integration/test_dictionaries_ddl/test.py b/dbms/tests/integration/test_dictionaries_ddl/test.py index a949bee136a..ad50a8c7daf 100644 --- a/dbms/tests/integration/test_dictionaries_ddl/test.py +++ b/dbms/tests/integration/test_dictionaries_ddl/test.py @@ -182,3 +182,35 @@ def test_conflicting_name(started_cluster): # old version still works node3.query("select dictGetUInt8('test.conflicting_dictionary', 'SomeValue1', toUInt64(17))") == '17\n' + +def test_http_dictionary_restrictions(started_cluster): + try: + node3.query(""" + CREATE DICTIONARY test.restricted_http_dictionary ( + id UInt64, + value String + ) + PRIMARY KEY id + LAYOUT(FLAT()) + SOURCE(HTTP(URL 'http://somehost.net' FORMAT TabSeparated)) + LIFETIME(1) + """) + node3.query("SELECT dictGetString('test.restricted_http_dictionary', 'value', toUInt64(1))") + except QueryRuntimeException as ex: + assert 'is not allowed in config.xml' in str(ex) + +def test_file_dictionary_restrictions(started_cluster): + try: + node3.query(""" + CREATE DICTIONARY test.restricted_file_dictionary ( + id UInt64, + value String + ) + PRIMARY KEY id + LAYOUT(FLAT()) + SOURCE(FILE(PATH '/usr/bin/cat' FORMAT TabSeparated)) + LIFETIME(1) + """) + node3.query("SELECT dictGetString('test.restricted_file_dictionary', 'value', toUInt64(1))") + except QueryRuntimeException as ex: + assert 'is not inside' in str(ex) diff --git a/dbms/tests/integration/test_dictionaries_update_and_reload/test.py b/dbms/tests/integration/test_dictionaries_update_and_reload/test.py index b972dc6c918..434ebc7d505 100644 --- a/dbms/tests/integration/test_dictionaries_update_and_reload/test.py +++ b/dbms/tests/integration/test_dictionaries_update_and_reload/test.py @@ -2,6 +2,7 @@ import pytest import os import time from helpers.cluster import ClickHouseCluster +from helpers.client import QueryTimeoutExceedException from helpers.test_tools import assert_eq_with_retry SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -53,8 +54,9 @@ def test_reload_while_loading(started_cluster): assert get_status('slow') == "NOT_LOADED" assert get_loading_duration('slow') == 0 - # It's not possible to get a value from the dictionary within 1.0 second, so the following query fails by timeout. - assert query("SELECT dictGetInt32('slow', 'a', toUInt64(5))", timeout = 1, ignore_error = True) == "" + # It's not possible to get a value from the dictionary within 0.5 second, so the following query fails by timeout. + with pytest.raises(QueryTimeoutExceedException): + query("SELECT dictGetInt32('slow', 'a', toUInt64(5))", timeout = 0.5) # The dictionary is now loading. assert get_status('slow') == "LOADING" @@ -69,7 +71,8 @@ def test_reload_while_loading(started_cluster): assert duration >= prev_duration # SYSTEM RELOAD DICTIONARY should restart loading. - query("SYSTEM RELOAD DICTIONARY 'slow'") + with pytest.raises(QueryTimeoutExceedException): + query("SYSTEM RELOAD DICTIONARY 'slow'", timeout = 0.5) assert get_status('slow') == "LOADING" prev_start_time, prev_duration = start_time, duration start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow') @@ -83,15 +86,7 @@ def test_reload_while_loading(started_cluster): assert start_time == prev_start_time assert duration >= prev_duration - # SYSTEM RELOAD DICTIONARIES should restart loading again. - query("SYSTEM RELOAD DICTIONARIES") - assert get_status('slow') == "LOADING" - prev_start_time, prev_duration = start_time, duration - start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow') - assert start_time > prev_start_time - assert duration < prev_duration - - # Changing the configuration file should restart loading one more time. + # Changing the configuration file should restart loading again. replace_in_file_in_container('/etc/clickhouse-server/config.d/slow.xml', 'sleep 100', 'sleep 0') time.sleep(5) # Configuration files are reloaded once in 5 seconds. @@ -141,13 +136,13 @@ def test_reload_after_fail_by_system_reload(started_cluster): assert get_status("no_file") == "NOT_LOADED" # We expect an error because the file source doesn't exist. - expected_error = "No such file" - assert expected_error in instance.query_and_get_error("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") + no_such_file_error = "No such file" + assert no_such_file_error in instance.query_and_get_error("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") assert get_status("no_file") == "FAILED" # SYSTEM RELOAD should not change anything now, the status is still FAILED. - query("SYSTEM RELOAD DICTIONARY 'no_file'") - assert expected_error in instance.query_and_get_error("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") + assert no_such_file_error in instance.query_and_get_error("SYSTEM RELOAD DICTIONARY 'no_file'") + assert no_such_file_error in instance.query_and_get_error("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") assert get_status("no_file") == "FAILED" # Creating the file source makes the dictionary able to load. @@ -158,7 +153,7 @@ def test_reload_after_fail_by_system_reload(started_cluster): # Removing the file source should not spoil the loaded dictionary. instance.exec_in_container("rm /etc/clickhouse-server/config.d/no_file.txt") - query("SYSTEM RELOAD DICTIONARY 'no_file'") + assert no_such_file_error in instance.query_and_get_error("SYSTEM RELOAD DICTIONARY 'no_file'") query("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") == "10\n" assert get_status("no_file") == "LOADED" diff --git a/dbms/tests/integration/test_dictionary_ddl_on_cluster/__init__.py b/dbms/tests/integration/test_dictionary_ddl_on_cluster/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_dictionary_ddl_on_cluster/configs/config.d/clusters.xml b/dbms/tests/integration/test_dictionary_ddl_on_cluster/configs/config.d/clusters.xml new file mode 100644 index 00000000000..bead63a1641 --- /dev/null +++ b/dbms/tests/integration/test_dictionary_ddl_on_cluster/configs/config.d/clusters.xml @@ -0,0 +1,28 @@ + + + + + true + + ch1 + 9000 + + + ch2 + 9000 + + + + true + + ch3 + 9000 + + + ch4 + 9000 + + + + + diff --git a/dbms/tests/integration/test_dictionary_ddl_on_cluster/configs/config.d/ddl.xml b/dbms/tests/integration/test_dictionary_ddl_on_cluster/configs/config.d/ddl.xml new file mode 100644 index 00000000000..451e407c160 --- /dev/null +++ b/dbms/tests/integration/test_dictionary_ddl_on_cluster/configs/config.d/ddl.xml @@ -0,0 +1,8 @@ + + + /clickhouse/task_queue/ddl + 10 + 3600 + 5 + + diff --git a/dbms/tests/integration/test_dictionary_ddl_on_cluster/test.py b/dbms/tests/integration/test_dictionary_ddl_on_cluster/test.py new file mode 100644 index 00000000000..31ee90de472 --- /dev/null +++ b/dbms/tests/integration/test_dictionary_ddl_on_cluster/test.py @@ -0,0 +1,62 @@ +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.client import QueryRuntimeException + +cluster = ClickHouseCluster(__file__) +ch1 = cluster.add_instance('ch1', config_dir="configs", with_zookeeper=True) +ch2 = cluster.add_instance('ch2', config_dir="configs", with_zookeeper=True) +ch3 = cluster.add_instance('ch3', config_dir="configs", with_zookeeper=True) +ch4 = cluster.add_instance('ch4', config_dir="configs", with_zookeeper=True) + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + ch1.query("CREATE TABLE sometbl ON CLUSTER 'cluster' (key UInt64, value String) ENGINE = MergeTree ORDER by key") + yield cluster + + finally: + cluster.shutdown() + + +def test_dictionary_ddl_on_cluster(started_cluster): + for node in [ch1, ch2, ch3, ch4]: + assert node.query("SELECT count() from sometbl") == "0\n" + + for num, node in enumerate([ch1, ch2, ch3, ch4]): + node.query("insert into sometbl values ({}, '{}')".format(num, node.name)) + + + ch1.query( + """ + CREATE DICTIONARY somedict ON CLUSTER 'cluster' ( + key UInt64, + value String + ) + PRIMARY KEY key + LAYOUT(FLAT()) + SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'sometbl' DB 'default')) + LIFETIME(10) + """) + + for num, node in enumerate([ch1, ch2, ch3, ch4]): + assert node.query("SELECT count() from sometbl") == "1\n" + assert node.query("SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == node.name + '\n' + + ch1.query("DETACH DICTIONARY default.somedict ON CLUSTER 'cluster'") + + for node in [ch1, ch2, ch3, ch4]: + with pytest.raises(QueryRuntimeException): + node.query("SELECT dictGetString('default.somedict', 'value', toUInt64(1))") + + ch1.query("ATTACH DICTIONARY default.somedict ON CLUSTER 'cluster'") + + for num, node in enumerate([ch1, ch2, ch3, ch4]): + assert node.query("SELECT count() from sometbl") == "1\n" + assert node.query("SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == node.name + '\n' + + ch1.query("DROP DICTIONARY default.somedict ON CLUSTER 'cluster'") + + for node in [ch1, ch2, ch3, ch4]: + with pytest.raises(QueryRuntimeException): + node.query("SELECT dictGetString('default.somedict', 'value', toUInt64(1))") diff --git a/dbms/tests/integration/test_distributed_ddl/cluster.py b/dbms/tests/integration/test_distributed_ddl/cluster.py new file mode 100644 index 00000000000..28071381586 --- /dev/null +++ b/dbms/tests/integration/test_distributed_ddl/cluster.py @@ -0,0 +1,109 @@ +import os +import os.path as p +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager +from helpers.test_tools import TSV + + +class ClickHouseClusterWithDDLHelpers(ClickHouseCluster): + def __init__(self, base_path, config_dir): + ClickHouseCluster.__init__(self, base_path) + + self.test_config_dir = config_dir + + def prepare(self, replace_hostnames_with_ips=True): + try: + for i in xrange(4): + self.add_instance( + 'ch{}'.format(i+1), + config_dir=self.test_config_dir, + macros={"layer": 0, "shard": i/2 + 1, "replica": i%2 + 1}, + with_zookeeper=True) + + self.start() + + # Replace config files for testing ability to set host in DNS and IP formats + if replace_hostnames_with_ips: + self.replace_domains_to_ip_addresses_in_cluster_config(['ch1', 'ch3']) + + # Select sacrifice instance to test CONNECTION_LOSS and server fail on it + sacrifice = self.instances['ch4'] + self.pm_random_drops = PartitionManager() + self.pm_random_drops._add_rule({'probability': 0.01, 'destination': sacrifice.ip_address, 'source_port': 2181, 'action': 'REJECT --reject-with tcp-reset'}) + self.pm_random_drops._add_rule({'probability': 0.01, 'source': sacrifice.ip_address, 'destination_port': 2181, 'action': 'REJECT --reject-with tcp-reset'}) + + # Initialize databases and service tables + instance = self.instances['ch1'] + + self.ddl_check_query(instance, """ + CREATE TABLE IF NOT EXISTS all_tables ON CLUSTER 'cluster_no_replicas' + (database String, name String, engine String, metadata_modification_time DateTime) + ENGINE = Distributed('cluster_no_replicas', 'system', 'tables') + """) + + self.ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS test ON CLUSTER 'cluster'") + + except Exception as e: + print e + raise + + def sync_replicas(self, table, timeout=5): + for instance in self.instances.values(): + instance.query("SYSTEM SYNC REPLICA {}".format(table), timeout=timeout) + + def check_all_hosts_successfully_executed(self, tsv_content, num_hosts=None): + if num_hosts is None: + num_hosts = len(self.instances) + + M = TSV.toMat(tsv_content) + hosts = [(l[0], l[1]) for l in M] # (host, port) + codes = [l[2] for l in M] + messages = [l[3] for l in M] + + assert len(hosts) == num_hosts and len(set(hosts)) == num_hosts, "\n" + tsv_content + assert len(set(codes)) == 1, "\n" + tsv_content + assert codes[0] == "0", "\n" + tsv_content + + def ddl_check_query(self, instance, query, num_hosts=None): + contents = instance.query(query) + self.check_all_hosts_successfully_executed(contents, num_hosts) + return contents + + def replace_domains_to_ip_addresses_in_cluster_config(self, instances_to_replace): + clusters_config = open(p.join(self.base_dir, '{}/config.d/clusters.xml'.format(self.test_config_dir))).read() + + for inst_name, inst in self.instances.items(): + clusters_config = clusters_config.replace(inst_name, str(inst.ip_address)) + + for inst_name in instances_to_replace: + inst = self.instances[inst_name] + self.instances[inst_name].exec_in_container(['bash', '-c', 'echo "$NEW_CONFIG" > /etc/clickhouse-server/config.d/clusters.xml'], environment={"NEW_CONFIG": clusters_config}, privileged=True) + # print cluster.instances[inst_name].exec_in_container(['cat', "/etc/clickhouse-server/config.d/clusters.xml"]) + + @staticmethod + def ddl_check_there_are_no_dublicates(instance): + query = "SELECT max(c), argMax(q, c) FROM (SELECT lower(query) AS q, count() AS c FROM system.query_log WHERE type=2 AND q LIKE '/* ddl_entry=query-%' GROUP BY query)" + rows = instance.query(query) + assert len(rows) > 0 and rows[0][0] == "1", "dublicates on {} {}, query {}".format(instance.name, instance.ip_address, query) + + @staticmethod + def insert_reliable(instance, query_insert): + """ + Make retries in case of UNKNOWN_STATUS_OF_INSERT or zkutil::KeeperException errors + """ + + for i in xrange(100): + try: + instance.query(query_insert) + return + except Exception as e: + last_exception = e + s = str(e) + if not (s.find('Unknown status, client must retry') >= 0 or s.find('zkutil::KeeperException')): + raise e + + raise last_exception diff --git a/dbms/tests/integration/test_distributed_ddl_secure/configs/config.d/clusters.xml b/dbms/tests/integration/test_distributed_ddl/configs_secure/config.d/clusters.xml similarity index 100% rename from dbms/tests/integration/test_distributed_ddl_secure/configs/config.d/clusters.xml rename to dbms/tests/integration/test_distributed_ddl/configs_secure/config.d/clusters.xml diff --git a/dbms/tests/integration/test_distributed_ddl_secure/configs/config.d/ddl.xml b/dbms/tests/integration/test_distributed_ddl/configs_secure/config.d/ddl.xml similarity index 100% rename from dbms/tests/integration/test_distributed_ddl_secure/configs/config.d/ddl.xml rename to dbms/tests/integration/test_distributed_ddl/configs_secure/config.d/ddl.xml diff --git a/dbms/tests/integration/test_distributed_ddl_secure/configs/config.d/macro.xml b/dbms/tests/integration/test_distributed_ddl/configs_secure/config.d/macro.xml similarity index 100% rename from dbms/tests/integration/test_distributed_ddl_secure/configs/config.d/macro.xml rename to dbms/tests/integration/test_distributed_ddl/configs_secure/config.d/macro.xml diff --git a/dbms/tests/integration/test_distributed_ddl_secure/configs/config.d/query_log.xml b/dbms/tests/integration/test_distributed_ddl/configs_secure/config.d/query_log.xml similarity index 100% rename from dbms/tests/integration/test_distributed_ddl_secure/configs/config.d/query_log.xml rename to dbms/tests/integration/test_distributed_ddl/configs_secure/config.d/query_log.xml diff --git a/dbms/tests/integration/test_distributed_ddl_secure/configs/config.d/ssl_conf.xml b/dbms/tests/integration/test_distributed_ddl/configs_secure/config.d/ssl_conf.xml similarity index 100% rename from dbms/tests/integration/test_distributed_ddl_secure/configs/config.d/ssl_conf.xml rename to dbms/tests/integration/test_distributed_ddl/configs_secure/config.d/ssl_conf.xml diff --git a/dbms/tests/integration/test_distributed_ddl_secure/configs/config.d/zookeeper_session_timeout.xml b/dbms/tests/integration/test_distributed_ddl/configs_secure/config.d/zookeeper_session_timeout.xml similarity index 100% rename from dbms/tests/integration/test_distributed_ddl_secure/configs/config.d/zookeeper_session_timeout.xml rename to dbms/tests/integration/test_distributed_ddl/configs_secure/config.d/zookeeper_session_timeout.xml diff --git a/dbms/tests/integration/test_distributed_ddl_secure/configs/dhparam.pem b/dbms/tests/integration/test_distributed_ddl/configs_secure/dhparam.pem similarity index 100% rename from dbms/tests/integration/test_distributed_ddl_secure/configs/dhparam.pem rename to dbms/tests/integration/test_distributed_ddl/configs_secure/dhparam.pem diff --git a/dbms/tests/integration/test_distributed_ddl_secure/configs/server.crt b/dbms/tests/integration/test_distributed_ddl/configs_secure/server.crt similarity index 100% rename from dbms/tests/integration/test_distributed_ddl_secure/configs/server.crt rename to dbms/tests/integration/test_distributed_ddl/configs_secure/server.crt diff --git a/dbms/tests/integration/test_distributed_ddl_secure/configs/server.key b/dbms/tests/integration/test_distributed_ddl/configs_secure/server.key similarity index 100% rename from dbms/tests/integration/test_distributed_ddl_secure/configs/server.key rename to dbms/tests/integration/test_distributed_ddl/configs_secure/server.key diff --git a/dbms/tests/integration/test_distributed_ddl_secure/configs/users.d/query_log.xml b/dbms/tests/integration/test_distributed_ddl/configs_secure/users.d/query_log.xml similarity index 100% rename from dbms/tests/integration/test_distributed_ddl_secure/configs/users.d/query_log.xml rename to dbms/tests/integration/test_distributed_ddl/configs_secure/users.d/query_log.xml diff --git a/dbms/tests/integration/test_distributed_ddl_secure/configs/users.d/restricted_user.xml b/dbms/tests/integration/test_distributed_ddl/configs_secure/users.d/restricted_user.xml similarity index 100% rename from dbms/tests/integration/test_distributed_ddl_secure/configs/users.d/restricted_user.xml rename to dbms/tests/integration/test_distributed_ddl/configs_secure/users.d/restricted_user.xml diff --git a/dbms/tests/integration/test_distributed_ddl/test.py b/dbms/tests/integration/test_distributed_ddl/test.py index a6eda22df4f..e30880e6ea4 100755 --- a/dbms/tests/integration/test_distributed_ddl/test.py +++ b/dbms/tests/integration/test_distributed_ddl/test.py @@ -1,122 +1,34 @@ import os -import os.path as p import sys import time -import datetime -import pytest from contextlib import contextmanager +import pytest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from helpers.cluster import ClickHouseCluster -from helpers.network import PartitionManager, PartitionManagerDisbaler + +from helpers.network import PartitionManager from helpers.test_tools import TSV +from .cluster import ClickHouseClusterWithDDLHelpers -def check_all_hosts_sucesfully_executed(tsv_content, num_hosts=None): - if num_hosts is None: - num_hosts = len(cluster.instances) +@pytest.fixture(scope="module", params=["configs", "configs_secure"]) +def test_cluster(request): + cluster = ClickHouseClusterWithDDLHelpers(__file__, request.param) - M = TSV.toMat(tsv_content) - hosts = [(l[0], l[1]) for l in M] # (host, port) - codes = [l[2] for l in M] - messages = [l[3] for l in M] - - assert len(hosts) == num_hosts and len(set(hosts)) == num_hosts, "\n" + tsv_content - assert len(set(codes)) == 1, "\n" + tsv_content - assert codes[0] == "0", "\n" + tsv_content - - -def ddl_check_query(instance, query, num_hosts=None): - contents = instance.query(query) - check_all_hosts_sucesfully_executed(contents, num_hosts) - return contents - -def ddl_check_there_are_no_dublicates(instance): - rows = instance.query("SELECT max(c), argMax(q, c) FROM (SELECT lower(query) AS q, count() AS c FROM system.query_log WHERE type=2 AND q LIKE '/* ddl_entry=query-%' GROUP BY query)") - assert len(rows) > 0 and rows[0][0] == "1", "dublicates on {} {}, query {}".format(instance.name, instance.ip_address) - -# Make retries in case of UNKNOWN_STATUS_OF_INSERT or zkutil::KeeperException errors -def insert_reliable(instance, query_insert): - for i in xrange(100): - try: - instance.query(query_insert) - return - except Exception as e: - last_exception = e - s = str(e) - if not (s.find('Unknown status, client must retry') >= 0 or s.find('zkutil::KeeperException')): - raise e - - raise last_exception - - -TEST_REPLICATED_ALTERS=False # TODO: Check code and turn on -cluster = ClickHouseCluster(__file__) - - -def replace_domains_to_ip_addresses_in_cluster_config(instances_to_replace): - clusters_config = open(p.join(cluster.base_dir, 'configs/config.d/clusters.xml')).read() - - for inst_name, inst in cluster.instances.items(): - clusters_config = clusters_config.replace(inst_name, str(inst.ip_address)) - - for inst_name in instances_to_replace: - inst = cluster.instances[inst_name] - cluster.instances[inst_name].exec_in_container(['bash', '-c', 'echo "$NEW_CONFIG" > /etc/clickhouse-server/config.d/clusters.xml'], environment={"NEW_CONFIG": clusters_config}, privileged=True) - # print cluster.instances[inst_name].exec_in_container(['cat', "/etc/clickhouse-server/config.d/clusters.xml"]) - - -def init_cluster(cluster): try: - for i in xrange(4): - cluster.add_instance( - 'ch{}'.format(i+1), - config_dir="configs", - macros={"layer": 0, "shard": i/2 + 1, "replica": i%2 + 1}, - with_zookeeper=True) - - cluster.start() - - # Replace config files for testing ability to set host in DNS and IP formats - replace_domains_to_ip_addresses_in_cluster_config(['ch1', 'ch3']) - - # Select sacrifice instance to test CONNECTION_LOSS and server fail on it - sacrifice = cluster.instances['ch4'] - cluster.pm_random_drops = PartitionManager() - cluster.pm_random_drops._add_rule({'probability': 0.01, 'destination': sacrifice.ip_address, 'source_port': 2181, 'action': 'REJECT --reject-with tcp-reset'}) - cluster.pm_random_drops._add_rule({'probability': 0.01, 'source': sacrifice.ip_address, 'destination_port': 2181, 'action': 'REJECT --reject-with tcp-reset'}) - - # Initialize databases and service tables - instance = cluster.instances['ch1'] - - ddl_check_query(instance, """ -CREATE TABLE IF NOT EXISTS all_tables ON CLUSTER 'cluster_no_replicas' - (database String, name String, engine String, metadata_modification_time DateTime) - ENGINE = Distributed('cluster_no_replicas', 'system', 'tables') - """) - - ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS test ON CLUSTER 'cluster'") - - except Exception as e: - print e - raise - - -@pytest.fixture(scope="module") -def started_cluster(): - try: - init_cluster(cluster) + cluster.prepare() yield cluster instance = cluster.instances['ch1'] - ddl_check_query(instance, "DROP DATABASE test ON CLUSTER 'cluster'") - ddl_check_query(instance, "DROP DATABASE IF EXISTS test2 ON CLUSTER 'cluster'") + cluster.ddl_check_query(instance, "DROP DATABASE test ON CLUSTER 'cluster'") + cluster.ddl_check_query(instance, "DROP DATABASE IF EXISTS test2 ON CLUSTER 'cluster'") # Check query log to ensure that DDL queries are not executed twice time.sleep(1.5) for instance in cluster.instances.values(): - ddl_check_there_are_no_dublicates(instance) + cluster.ddl_check_there_are_no_dublicates(instance) cluster.pm_random_drops.heal_all() @@ -124,57 +36,57 @@ def started_cluster(): cluster.shutdown() -def test_default_database(started_cluster): - instance = cluster.instances['ch3'] +def test_default_database(test_cluster): + instance = test_cluster.instances['ch3'] - ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS test2 ON CLUSTER 'cluster' FORMAT TSV") - ddl_check_query(instance, "DROP TABLE IF EXISTS null ON CLUSTER 'cluster' FORMAT TSV") - ddl_check_query(instance, "CREATE TABLE null ON CLUSTER 'cluster2' (s String DEFAULT 'escape\t\nme') ENGINE = Null") + test_cluster.ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS test2 ON CLUSTER 'cluster' FORMAT TSV") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS null ON CLUSTER 'cluster' FORMAT TSV") + test_cluster.ddl_check_query(instance, "CREATE TABLE null ON CLUSTER 'cluster2' (s String DEFAULT 'escape\t\nme') ENGINE = Null") contents = instance.query("SELECT hostName() AS h, database FROM all_tables WHERE name = 'null' ORDER BY h") assert TSV(contents) == TSV("ch1\tdefault\nch2\ttest2\nch3\tdefault\nch4\ttest2\n") - ddl_check_query(instance, "DROP TABLE IF EXISTS null ON CLUSTER cluster2") - ddl_check_query(instance, "DROP DATABASE IF EXISTS test2 ON CLUSTER 'cluster'") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS null ON CLUSTER cluster2") + test_cluster.ddl_check_query(instance, "DROP DATABASE IF EXISTS test2 ON CLUSTER 'cluster'") -def test_create_view(started_cluster): - instance = cluster.instances['ch3'] - ddl_check_query(instance, "CREATE VIEW test.super_simple_view ON CLUSTER 'cluster' AS SELECT * FROM system.numbers FORMAT TSV") - ddl_check_query(instance, "CREATE MATERIALIZED VIEW test.simple_mat_view ON CLUSTER 'cluster' ENGINE = Memory AS SELECT * FROM system.numbers FORMAT TSV") - ddl_check_query(instance, "DROP TABLE test.simple_mat_view ON CLUSTER 'cluster' FORMAT TSV") - ddl_check_query(instance, "DROP TABLE IF EXISTS test.super_simple_view2 ON CLUSTER 'cluster' FORMAT TSV") +def test_create_view(test_cluster): + instance = test_cluster.instances['ch3'] + test_cluster.ddl_check_query(instance, "CREATE VIEW test.super_simple_view ON CLUSTER 'cluster' AS SELECT * FROM system.numbers FORMAT TSV") + test_cluster.ddl_check_query(instance, "CREATE MATERIALIZED VIEW test.simple_mat_view ON CLUSTER 'cluster' ENGINE = Memory AS SELECT * FROM system.numbers FORMAT TSV") + test_cluster.ddl_check_query(instance, "DROP TABLE test.simple_mat_view ON CLUSTER 'cluster' FORMAT TSV") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.super_simple_view2 ON CLUSTER 'cluster' FORMAT TSV") - ddl_check_query(instance, "CREATE TABLE test.super_simple ON CLUSTER 'cluster' (i Int8) ENGINE = Memory") - ddl_check_query(instance, "RENAME TABLE test.super_simple TO test.super_simple2 ON CLUSTER 'cluster' FORMAT TSV") - ddl_check_query(instance, "DROP TABLE test.super_simple2 ON CLUSTER 'cluster'") + test_cluster.ddl_check_query(instance, "CREATE TABLE test.super_simple ON CLUSTER 'cluster' (i Int8) ENGINE = Memory") + test_cluster.ddl_check_query(instance, "RENAME TABLE test.super_simple TO test.super_simple2 ON CLUSTER 'cluster' FORMAT TSV") + test_cluster.ddl_check_query(instance, "DROP TABLE test.super_simple2 ON CLUSTER 'cluster'") -def test_on_server_fail(started_cluster): - instance = cluster.instances['ch1'] - kill_instance = cluster.instances['ch2'] +def test_on_server_fail(test_cluster): + instance = test_cluster.instances['ch1'] + kill_instance = test_cluster.instances['ch2'] - ddl_check_query(instance, "DROP TABLE IF EXISTS test.test_server_fail ON CLUSTER 'cluster'") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.test_server_fail ON CLUSTER 'cluster'") kill_instance.get_docker_handle().stop() request = instance.get_query_request("CREATE TABLE test.test_server_fail ON CLUSTER 'cluster' (i Int8) ENGINE=Null", timeout=30) kill_instance.get_docker_handle().start() - ddl_check_query(instance, "DROP TABLE IF EXISTS test.__nope__ ON CLUSTER 'cluster'") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.__nope__ ON CLUSTER 'cluster'") # Check query itself - check_all_hosts_sucesfully_executed(request.get_answer()) + test_cluster.check_all_hosts_successfully_executed(request.get_answer()) # And check query artefacts contents = instance.query("SELECT hostName() AS h FROM all_tables WHERE database='test' AND name='test_server_fail' ORDER BY h") assert TSV(contents) == TSV("ch1\nch2\nch3\nch4\n") - ddl_check_query(instance, "DROP TABLE test.test_server_fail ON CLUSTER 'cluster'") + test_cluster.ddl_check_query(instance, "DROP TABLE test.test_server_fail ON CLUSTER 'cluster'") -def _test_on_connection_losses(cluster, zk_timeout): - instance = cluster.instances['ch1'] - kill_instance = cluster.instances['ch2'] +def _test_on_connection_losses(test_cluster, zk_timeout): + instance = test_cluster.instances['ch1'] + kill_instance = test_cluster.instances['ch2'] with PartitionManager() as pm: pm.drop_instance_zk_connections(kill_instance) @@ -182,170 +94,111 @@ def _test_on_connection_losses(cluster, zk_timeout): time.sleep(zk_timeout) pm.restore_instance_zk_connections(kill_instance) - check_all_hosts_sucesfully_executed(request.get_answer()) + test_cluster.check_all_hosts_successfully_executed(request.get_answer()) -def test_on_connection_loss(started_cluster): - _test_on_connection_losses(cluster, 1.5) # connection loss will occur only (3 sec ZK timeout in config) +def test_on_connection_loss(test_cluster): + _test_on_connection_losses(test_cluster, 1.5) # connection loss will occur only (3 sec ZK timeout in config) -def test_on_session_expired(started_cluster): - _test_on_connection_losses(cluster, 4) # session should be expired (3 sec ZK timeout in config) +def test_on_session_expired(test_cluster): + _test_on_connection_losses(test_cluster, 4) # session should be expired (3 sec ZK timeout in config) -def test_replicated_alters(started_cluster): - instance = cluster.instances['ch2'] +def test_simple_alters(test_cluster): + instance = test_cluster.instances['ch2'] - ddl_check_query(instance, "DROP TABLE IF EXISTS merge_for_alter ON CLUSTER cluster") - ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_32 ON CLUSTER cluster") - ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_64 ON CLUSTER cluster") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS merge ON CLUSTER cluster_without_replication") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_32 ON CLUSTER cluster_without_replication") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_64 ON CLUSTER cluster_without_replication") - if not TEST_REPLICATED_ALTERS: - return - - # Temporarily disable random ZK packet drops, they might broke creation if ReplicatedMergeTree replicas - firewall_drops_rules = cluster.pm_random_drops.pop_rules() - - ddl_check_query(instance, """ -CREATE TABLE IF NOT EXISTS merge_for_alter ON CLUSTER cluster (p Date, i Int32) -ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/hits', '{replica}', p, p, 1) -""") - - ddl_check_query(instance, """ -CREATE TABLE IF NOT EXISTS all_merge_32 ON CLUSTER cluster (p Date, i Int32) -ENGINE = Distributed(cluster, default, merge_for_alter, i) -""") - ddl_check_query(instance, """ -CREATE TABLE IF NOT EXISTS all_merge_64 ON CLUSTER cluster (p Date, i Int64, s String) -ENGINE = Distributed(cluster, default, merge_for_alter, i) -""") - - for i in xrange(4): - k = (i / 2) * 2 - insert_reliable(cluster.instances['ch{}'.format(i + 1)], "INSERT INTO merge_for_alter (i) VALUES ({})({})".format(k, k+1)) - - assert TSV(instance.query("SELECT i FROM all_merge_32 ORDER BY i")) == TSV(''.join(['{}\n'.format(x) for x in xrange(4)])) - - - ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster MODIFY COLUMN i Int64") - ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster ADD COLUMN s DEFAULT toString(i)") - - assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)])) - - - for i in xrange(4): - k = (i / 2) * 2 + 4 - insert_reliable(cluster.instances['ch{}'.format(i + 1)], "INSERT INTO merge_for_alter (p, i) VALUES (31, {})(31, {})".format(k, k+1)) - - assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(8)])) - - - ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster DETACH PARTITION 197002") - assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)])) - - ddl_check_query(instance, "DROP TABLE merge_for_alter ON CLUSTER cluster") - - # Enable random ZK packet drops - cluster.pm_random_drops.push_rules(firewall_drops_rules) - - ddl_check_query(instance, "DROP TABLE all_merge_32 ON CLUSTER cluster") - ddl_check_query(instance, "DROP TABLE all_merge_64 ON CLUSTER cluster") - - -def test_simple_alters(started_cluster): - instance = cluster.instances['ch2'] - - ddl_check_query(instance, "DROP TABLE IF EXISTS merge ON CLUSTER cluster_without_replication") - ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_32 ON CLUSTER cluster_without_replication") - ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_64 ON CLUSTER cluster_without_replication") - - ddl_check_query(instance, """ + test_cluster.ddl_check_query(instance, """ CREATE TABLE IF NOT EXISTS merge ON CLUSTER cluster_without_replication (p Date, i Int32) ENGINE = MergeTree(p, p, 1) """) - ddl_check_query(instance, """ + test_cluster.ddl_check_query(instance, """ CREATE TABLE IF NOT EXISTS all_merge_32 ON CLUSTER cluster_without_replication (p Date, i Int32) ENGINE = Distributed(cluster_without_replication, default, merge, i) """) - ddl_check_query(instance, """ + test_cluster.ddl_check_query(instance, """ CREATE TABLE IF NOT EXISTS all_merge_64 ON CLUSTER cluster_without_replication (p Date, i Int64, s String) ENGINE = Distributed(cluster_without_replication, default, merge, i) """) for i in xrange(4): k = (i / 2) * 2 - cluster.instances['ch{}'.format(i + 1)].query("INSERT INTO merge (i) VALUES ({})({})".format(k, k+1)) + test_cluster.instances['ch{}'.format(i + 1)].query("INSERT INTO merge (i) VALUES ({})({})".format(k, k+1)) assert TSV(instance.query("SELECT i FROM all_merge_32 ORDER BY i")) == TSV(''.join(['{}\n'.format(x) for x in xrange(4)])) time.sleep(5) - ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER cluster_without_replication MODIFY COLUMN i Int64") + test_cluster.ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER cluster_without_replication MODIFY COLUMN i Int64") time.sleep(5) - ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER cluster_without_replication ADD COLUMN s DEFAULT toString(i) FORMAT TSV") + test_cluster.ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER cluster_without_replication ADD COLUMN s DEFAULT toString(i) FORMAT TSV") assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)])) for i in xrange(4): k = (i / 2) * 2 + 4 - cluster.instances['ch{}'.format(i + 1)].query("INSERT INTO merge (p, i) VALUES (31, {})(31, {})".format(k, k+1)) + test_cluster.instances['ch{}'.format(i + 1)].query("INSERT INTO merge (p, i) VALUES (31, {})(31, {})".format(k, k+1)) assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(8)])) - ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER cluster_without_replication DETACH PARTITION 197002") + test_cluster.ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER cluster_without_replication DETACH PARTITION 197002") assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)])) - ddl_check_query(instance, "DROP TABLE merge ON CLUSTER cluster_without_replication") - ddl_check_query(instance, "DROP TABLE all_merge_32 ON CLUSTER cluster_without_replication") - ddl_check_query(instance, "DROP TABLE all_merge_64 ON CLUSTER cluster_without_replication") + test_cluster.ddl_check_query(instance, "DROP TABLE merge ON CLUSTER cluster_without_replication") + test_cluster.ddl_check_query(instance, "DROP TABLE all_merge_32 ON CLUSTER cluster_without_replication") + test_cluster.ddl_check_query(instance, "DROP TABLE all_merge_64 ON CLUSTER cluster_without_replication") -def test_macro(started_cluster): - instance = cluster.instances['ch2'] - ddl_check_query(instance, "CREATE TABLE tab ON CLUSTER '{cluster}' (value UInt8) ENGINE = Memory") +def test_macro(test_cluster): + instance = test_cluster.instances['ch2'] + test_cluster.ddl_check_query(instance, "CREATE TABLE tab ON CLUSTER '{cluster}' (value UInt8) ENGINE = Memory") for i in xrange(4): - insert_reliable(cluster.instances['ch{}'.format(i + 1)], "INSERT INTO tab VALUES ({})".format(i)) + test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)], "INSERT INTO tab VALUES ({})".format(i)) - ddl_check_query(instance, "CREATE TABLE distr ON CLUSTER '{cluster}' (value UInt8) ENGINE = Distributed('{cluster}', 'default', 'tab', value % 4)") + test_cluster.ddl_check_query(instance, "CREATE TABLE distr ON CLUSTER '{cluster}' (value UInt8) ENGINE = Distributed('{cluster}', 'default', 'tab', value % 4)") assert TSV(instance.query("SELECT value FROM distr ORDER BY value")) == TSV('0\n1\n2\n3\n') - assert TSV( cluster.instances['ch3'].query("SELECT value FROM distr ORDER BY value")) == TSV('0\n1\n2\n3\n') + assert TSV(test_cluster.instances['ch3'].query("SELECT value FROM distr ORDER BY value")) == TSV('0\n1\n2\n3\n') - ddl_check_query(instance, "DROP TABLE IF EXISTS distr ON CLUSTER '{cluster}'") - ddl_check_query(instance, "DROP TABLE IF EXISTS tab ON CLUSTER '{cluster}'") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS distr ON CLUSTER '{cluster}'") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS tab ON CLUSTER '{cluster}'") -def test_implicit_macros(started_cluster): +def test_implicit_macros(test_cluster): # Temporarily disable random ZK packet drops, they might broke creation if ReplicatedMergeTree replicas - firewall_drops_rules = cluster.pm_random_drops.pop_rules() + firewall_drops_rules = test_cluster.pm_random_drops.pop_rules() - instance = cluster.instances['ch2'] + instance = test_cluster.instances['ch2'] - ddl_check_query(instance, "DROP DATABASE IF EXISTS test_db ON CLUSTER '{cluster}'") - ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS test_db ON CLUSTER '{cluster}'") - - ddl_check_query(instance, """ + test_cluster.ddl_check_query(instance, "DROP DATABASE IF EXISTS test_db ON CLUSTER '{cluster}'") + test_cluster.ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS test_db ON CLUSTER '{cluster}'") + + test_cluster.ddl_check_query(instance, """ CREATE TABLE IF NOT EXISTS test_db.test_macro ON CLUSTER '{cluster}' (p Date, i Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/{layer}-{shard}/{table}', '{replica}', p, p, 1) """) # Check that table was created at correct path in zookeeper - assert cluster.get_kazoo_client('zoo1').exists('/clickhouse/tables/test_db/0-1/test_macro') is not None + assert test_cluster.get_kazoo_client('zoo1').exists('/clickhouse/tables/test_db/0-1/test_macro') is not None # Enable random ZK packet drops - cluster.pm_random_drops.push_rules(firewall_drops_rules) + test_cluster.pm_random_drops.push_rules(firewall_drops_rules) -def test_allowed_databases(started_cluster): - instance = cluster.instances['ch2'] +def test_allowed_databases(test_cluster): + instance = test_cluster.instances['ch2'] instance.query("CREATE DATABASE IF NOT EXISTS db1 ON CLUSTER cluster") instance.query("CREATE DATABASE IF NOT EXISTS db2 ON CLUSTER cluster") instance.query("CREATE TABLE db1.t1 ON CLUSTER cluster (i Int8) ENGINE = Memory", settings={"user" : "restricted_user"}) - + with pytest.raises(Exception): instance.query("CREATE TABLE db2.t2 ON CLUSTER cluster (i Int8) ENGINE = Memory", settings={"user" : "restricted_user"}) with pytest.raises(Exception): @@ -355,45 +208,90 @@ def test_allowed_databases(started_cluster): instance.query("DROP DATABASE db1 ON CLUSTER cluster", settings={"user" : "restricted_user"}) -def test_kill_query(started_cluster): - instance = cluster.instances['ch3'] +def test_kill_query(test_cluster): + instance = test_cluster.instances['ch3'] - ddl_check_query(instance, "KILL QUERY ON CLUSTER 'cluster' WHERE NOT elapsed FORMAT TSV") + test_cluster.ddl_check_query(instance, "KILL QUERY ON CLUSTER 'cluster' WHERE NOT elapsed FORMAT TSV") -def test_detach_query(started_cluster): - instance = cluster.instances['ch3'] +def test_detach_query(test_cluster): + instance = test_cluster.instances['ch3'] - ddl_check_query(instance, "DROP TABLE IF EXISTS test_attach ON CLUSTER cluster FORMAT TSV") - ddl_check_query(instance, "CREATE TABLE test_attach ON CLUSTER cluster (i Int8)ENGINE = Log") - ddl_check_query(instance, "DETACH TABLE test_attach ON CLUSTER cluster FORMAT TSV") - ddl_check_query(instance, "ATTACH TABLE test_attach ON CLUSTER cluster") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_attach ON CLUSTER cluster FORMAT TSV") + test_cluster.ddl_check_query(instance, "CREATE TABLE test_attach ON CLUSTER cluster (i Int8)ENGINE = Log") + test_cluster.ddl_check_query(instance, "DETACH TABLE test_attach ON CLUSTER cluster FORMAT TSV") + test_cluster.ddl_check_query(instance, "ATTACH TABLE test_attach ON CLUSTER cluster") -def test_optimize_query(started_cluster): - instance = cluster.instances['ch3'] +def test_optimize_query(test_cluster): + instance = test_cluster.instances['ch3'] - ddl_check_query(instance, "DROP TABLE IF EXISTS test_optimize ON CLUSTER cluster FORMAT TSV") - ddl_check_query(instance, "CREATE TABLE test_optimize ON CLUSTER cluster (p Date, i Int32) ENGINE = MergeTree(p, p, 8192)") - ddl_check_query(instance, "OPTIMIZE TABLE test_optimize ON CLUSTER cluster FORMAT TSV") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_optimize ON CLUSTER cluster FORMAT TSV") + test_cluster.ddl_check_query(instance, "CREATE TABLE test_optimize ON CLUSTER cluster (p Date, i Int32) ENGINE = MergeTree(p, p, 8192)") + test_cluster.ddl_check_query(instance, "OPTIMIZE TABLE test_optimize ON CLUSTER cluster FORMAT TSV") -def test_create_as_select(started_cluster): - instance = cluster.instances['ch2'] - ddl_check_query(instance, "CREATE TABLE test_as_select ON CLUSTER cluster ENGINE = Memory AS (SELECT 1 AS x UNION ALL SELECT 2 AS x)") +def test_create_as_select(test_cluster): + instance = test_cluster.instances['ch2'] + test_cluster.ddl_check_query(instance, "CREATE TABLE test_as_select ON CLUSTER cluster ENGINE = Memory AS (SELECT 1 AS x UNION ALL SELECT 2 AS x)") assert TSV(instance.query("SELECT x FROM test_as_select ORDER BY x")) == TSV("1\n2\n") - ddl_check_query(instance, "DROP TABLE IF EXISTS test_as_select ON CLUSTER cluster") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_as_select ON CLUSTER cluster") -def test_create_reserved(started_cluster): - instance = cluster.instances['ch2'] - ddl_check_query(instance, "CREATE TABLE test_reserved ON CLUSTER cluster (`p` Date, `image` Nullable(String), `index` Nullable(Float64), `invalidate` Nullable(Int64)) ENGINE = MergeTree(`p`, `p`, 8192)") - ddl_check_query(instance, "CREATE TABLE test_as_reserved ON CLUSTER cluster ENGINE = Memory AS (SELECT * from test_reserved)") - ddl_check_query(instance, "DROP TABLE IF EXISTS test_reserved ON CLUSTER cluster") - ddl_check_query(instance, "DROP TABLE IF EXISTS test_as_reserved ON CLUSTER cluster") +def test_create_reserved(test_cluster): + instance = test_cluster.instances['ch2'] + test_cluster.ddl_check_query(instance, "CREATE TABLE test_reserved ON CLUSTER cluster (`p` Date, `image` Nullable(String), `index` Nullable(Float64), `invalidate` Nullable(Int64)) ENGINE = MergeTree(`p`, `p`, 8192)") + test_cluster.ddl_check_query(instance, "CREATE TABLE test_as_reserved ON CLUSTER cluster ENGINE = Memory AS (SELECT * from test_reserved)") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_reserved ON CLUSTER cluster") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_as_reserved ON CLUSTER cluster") + + +def test_rename(test_cluster): + instance = test_cluster.instances['ch1'] + rules = test_cluster.pm_random_drops.pop_rules() + test_cluster.ddl_check_query(instance, "CREATE TABLE rename_shard ON CLUSTER cluster (id Int64, sid String DEFAULT concat('old', toString(id))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/staging/test_shard', '{replica}') ORDER BY (id)") + test_cluster.ddl_check_query(instance, "CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)") + test_cluster.ddl_check_query(instance, "RENAME TABLE rename_new TO rename ON CLUSTER cluster;") + + + for i in range(10): + instance.query("insert into rename (id) values ({})".format(i)) + + # FIXME ddl_check_query doesnt work for replicated DDDL if replace_hostnames_with_ips=True + # because replicas use wrong host name of leader (and wrong path in zk) to check if it has executed query + # so ddl query will always fail on some replicas even if query was actually executed by leader + # Also such inconsistency in cluster configuration may lead to query duplication if leader suddenly changed + # because path of lock in zk contains shard name, which is list of host names of replicas + instance.query("ALTER TABLE rename_shard ON CLUSTER cluster MODIFY COLUMN sid String DEFAULT concat('new', toString(id))", ignore_error=True) + time.sleep(1) + + test_cluster.ddl_check_query(instance, "CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)") + + instance.query("system stop distributed sends rename") + + for i in range(10, 20): + instance.query("insert into rename (id) values ({})".format(i)) + + test_cluster.ddl_check_query(instance, "RENAME TABLE rename TO rename_old, rename_new TO rename ON CLUSTER cluster") + + for i in range(20, 30): + instance.query("insert into rename (id) values ({})".format(i)) + + instance.query("system flush distributed rename") + for name in ['ch1', 'ch2', 'ch3', 'ch4']: + test_cluster.instances[name].query("system sync replica rename_shard") + + # system stop distributed sends does not affect inserts into local shard, + # so some ids in range (10, 20) will be inserted into rename_shard + assert instance.query("select count(id), sum(id) from rename").rstrip() == "25\t360" + #assert instance.query("select count(id), sum(id) from rename").rstrip() == "20\t290" + assert instance.query("select count(id), sum(id) from rename where sid like 'old%'").rstrip() == "15\t115" + #assert instance.query("select count(id), sum(id) from rename where sid like 'old%'").rstrip() == "10\t45" + assert instance.query("select count(id), sum(id) from rename where sid like 'new%'").rstrip() == "10\t245" + test_cluster.pm_random_drops.push_rules(rules) if __name__ == '__main__': - with contextmanager(started_cluster)() as cluster: - for name, instance in cluster.instances.items(): + with contextmanager(test_cluster)() as ctx_cluster: + for name, instance in ctx_cluster.instances.items(): print name, instance.ip_address raw_input("Cluster created, press any key to destroy...") diff --git a/dbms/tests/integration/test_distributed_ddl/test_replicated_alter.py b/dbms/tests/integration/test_distributed_ddl/test_replicated_alter.py new file mode 100644 index 00000000000..8db9182ad0e --- /dev/null +++ b/dbms/tests/integration/test_distributed_ddl/test_replicated_alter.py @@ -0,0 +1,93 @@ +import os +import sys +import time +import pytest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from helpers.test_tools import TSV +from .cluster import ClickHouseClusterWithDDLHelpers + + +@pytest.fixture(scope="module", params=["configs", "configs_secure"]) +def test_cluster(request): + cluster = ClickHouseClusterWithDDLHelpers(__file__, request.param) + + try: + # TODO: Fix ON CLUSTER alters when nodes have different configs. Need to canonicalize node identity. + cluster.prepare(replace_hostnames_with_ips=False) + + yield cluster + + instance = cluster.instances['ch1'] + cluster.ddl_check_query(instance, "DROP DATABASE test ON CLUSTER 'cluster'") + cluster.ddl_check_query(instance, "DROP DATABASE IF EXISTS test2 ON CLUSTER 'cluster'") + + # Check query log to ensure that DDL queries are not executed twice + time.sleep(1.5) + for instance in cluster.instances.values(): + cluster.ddl_check_there_are_no_dublicates(instance) + + cluster.pm_random_drops.heal_all() + + finally: + cluster.shutdown() + + +def test_replicated_alters(test_cluster): + instance = test_cluster.instances['ch2'] + + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS merge_for_alter ON CLUSTER cluster") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_32 ON CLUSTER cluster") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_64 ON CLUSTER cluster") + + # Temporarily disable random ZK packet drops, they might broke creation if ReplicatedMergeTree replicas + firewall_drops_rules = test_cluster.pm_random_drops.pop_rules() + + test_cluster.ddl_check_query(instance, """ +CREATE TABLE IF NOT EXISTS merge_for_alter ON CLUSTER cluster (p Date, i Int32) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/hits', '{replica}', p, p, 1) +""") + + test_cluster.ddl_check_query(instance, """ +CREATE TABLE IF NOT EXISTS all_merge_32 ON CLUSTER cluster (p Date, i Int32) +ENGINE = Distributed(cluster, default, merge_for_alter, i) +""") + test_cluster.ddl_check_query(instance, """ +CREATE TABLE IF NOT EXISTS all_merge_64 ON CLUSTER cluster (p Date, i Int64, s String) +ENGINE = Distributed(cluster, default, merge_for_alter, i) +""") + + for i in xrange(4): + k = (i / 2) * 2 + test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)], "INSERT INTO merge_for_alter (i) VALUES ({})({})".format(k, k+1)) + + test_cluster.sync_replicas("merge_for_alter") + + assert TSV(instance.query("SELECT i FROM all_merge_32 ORDER BY i")) == TSV(''.join(['{}\n'.format(x) for x in xrange(4)])) + + + test_cluster.ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster MODIFY COLUMN i Int64") + test_cluster.ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster ADD COLUMN s DEFAULT toString(i)") + + assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)])) + + + for i in xrange(4): + k = (i / 2) * 2 + 4 + test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)], "INSERT INTO merge_for_alter (p, i) VALUES (31, {})(31, {})".format(k, k+1)) + + test_cluster.sync_replicas("merge_for_alter") + + assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(8)])) + + test_cluster.ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster DETACH PARTITION 197002") + assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)])) + + test_cluster.ddl_check_query(instance, "DROP TABLE merge_for_alter ON CLUSTER cluster") + + # Enable random ZK packet drops + test_cluster.pm_random_drops.push_rules(firewall_drops_rules) + + test_cluster.ddl_check_query(instance, "DROP TABLE all_merge_32 ON CLUSTER cluster") + test_cluster.ddl_check_query(instance, "DROP TABLE all_merge_64 ON CLUSTER cluster") diff --git a/dbms/tests/integration/test_distributed_ddl_secure/test.py b/dbms/tests/integration/test_distributed_ddl_secure/test.py deleted file mode 100755 index 0c8a1b0bf01..00000000000 --- a/dbms/tests/integration/test_distributed_ddl_secure/test.py +++ /dev/null @@ -1,391 +0,0 @@ -import os -import os.path as p -import sys -import time -import datetime -import pytest -from contextlib import contextmanager - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from helpers.cluster import ClickHouseCluster -from helpers.network import PartitionManager, PartitionManagerDisbaler -from helpers.test_tools import TSV - - -def check_all_hosts_sucesfully_executed(tsv_content, num_hosts=None): - if num_hosts is None: - num_hosts = len(cluster.instances) - - M = TSV.toMat(tsv_content) - hosts = [(l[0], l[1]) for l in M] # (host, port) - codes = [l[2] for l in M] - messages = [l[3] for l in M] - - assert len(hosts) == num_hosts and len(set(hosts)) == num_hosts, "\n" + tsv_content - assert len(set(codes)) == 1, "\n" + tsv_content - assert codes[0] == "0", "\n" + tsv_content - - -def ddl_check_query(instance, query, num_hosts=None): - contents = instance.query(query) - check_all_hosts_sucesfully_executed(contents, num_hosts) - return contents - -def ddl_check_there_are_no_dublicates(instance): - rows = instance.query("SELECT max(c), argMax(q, c) FROM (SELECT lower(query) AS q, count() AS c FROM system.query_log WHERE type=2 AND q LIKE '/* ddl_entry=query-%' GROUP BY query)") - assert len(rows) > 0 and rows[0][0] == "1", "dublicates on {} {}, query {}".format(instance.name, instance.ip_address) - -# Make retries in case of UNKNOWN_STATUS_OF_INSERT or zkutil::KeeperException errors -def insert_reliable(instance, query_insert): - for i in xrange(100): - try: - instance.query(query_insert) - return - except Exception as e: - last_exception = e - s = str(e) - if not (s.find('Unknown status, client must retry') >= 0 or s.find('zkutil::KeeperException')): - raise e - - raise last_exception - - -TEST_REPLICATED_ALTERS=False # TODO: Check code and turn on -cluster = ClickHouseCluster(__file__) - - -def replace_domains_to_ip_addresses_in_cluster_config(instances_to_replace): - clusters_config = open(p.join(cluster.base_dir, 'configs/config.d/clusters.xml')).read() - - for inst_name, inst in cluster.instances.items(): - clusters_config = clusters_config.replace(inst_name, str(inst.ip_address)) - - for inst_name in instances_to_replace: - inst = cluster.instances[inst_name] - cluster.instances[inst_name].exec_in_container(['bash', '-c', 'echo "$NEW_CONFIG" > /etc/clickhouse-server/config.d/clusters.xml'], environment={"NEW_CONFIG": clusters_config}, privileged=True) - # print cluster.instances[inst_name].exec_in_container(['cat', "/etc/clickhouse-server/config.d/clusters.xml"]) - - -def init_cluster(cluster): - try: - for i in xrange(4): - cluster.add_instance( - 'ch{}'.format(i+1), - config_dir="configs", - macros={"layer": 0, "shard": i/2 + 1, "replica": i%2 + 1}, - with_zookeeper=True) - - cluster.start() - - # Replace config files for testing ability to set host in DNS and IP formats - replace_domains_to_ip_addresses_in_cluster_config(['ch1', 'ch3']) - - # Select sacrifice instance to test CONNECTION_LOSS and server fail on it - sacrifice = cluster.instances['ch4'] - cluster.pm_random_drops = PartitionManager() - cluster.pm_random_drops._add_rule({'probability': 0.01, 'destination': sacrifice.ip_address, 'source_port': 2181, 'action': 'REJECT --reject-with tcp-reset'}) - cluster.pm_random_drops._add_rule({'probability': 0.01, 'source': sacrifice.ip_address, 'destination_port': 2181, 'action': 'REJECT --reject-with tcp-reset'}) - - # Initialize databases and service tables - instance = cluster.instances['ch1'] - - ddl_check_query(instance, """ -CREATE TABLE IF NOT EXISTS all_tables ON CLUSTER 'cluster_no_replicas' - (database String, name String, engine String, metadata_modification_time DateTime) - ENGINE = Distributed('cluster_no_replicas', 'system', 'tables') - """) - - ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS test ON CLUSTER 'cluster'") - - except Exception as e: - print e - raise - - -@pytest.fixture(scope="module") -def started_cluster(): - try: - init_cluster(cluster) - - yield cluster - - instance = cluster.instances['ch1'] - ddl_check_query(instance, "DROP DATABASE test ON CLUSTER 'cluster'") - ddl_check_query(instance, "DROP DATABASE IF EXISTS test2 ON CLUSTER 'cluster'") - - # Check query log to ensure that DDL queries are not executed twice - time.sleep(1.5) - for instance in cluster.instances.values(): - ddl_check_there_are_no_dublicates(instance) - - cluster.pm_random_drops.heal_all() - - finally: - cluster.shutdown() - - -def test_default_database(started_cluster): - instance = cluster.instances['ch3'] - - ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS test2 ON CLUSTER 'cluster' FORMAT TSV") - ddl_check_query(instance, "DROP TABLE IF EXISTS null ON CLUSTER 'cluster' FORMAT TSV") - ddl_check_query(instance, "CREATE TABLE null ON CLUSTER 'cluster2' (s String DEFAULT 'escape\t\nme') ENGINE = Null") - - contents = instance.query("SELECT hostName() AS h, database FROM all_tables WHERE name = 'null' ORDER BY h") - assert TSV(contents) == TSV("ch1\tdefault\nch2\ttest2\nch3\tdefault\nch4\ttest2\n") - - ddl_check_query(instance, "DROP TABLE IF EXISTS null ON CLUSTER cluster2") - ddl_check_query(instance, "DROP DATABASE IF EXISTS test2 ON CLUSTER 'cluster'") - - -def test_create_view(started_cluster): - instance = cluster.instances['ch3'] - ddl_check_query(instance, "CREATE VIEW test.super_simple_view ON CLUSTER 'cluster' AS SELECT * FROM system.numbers FORMAT TSV") - ddl_check_query(instance, "CREATE MATERIALIZED VIEW test.simple_mat_view ON CLUSTER 'cluster' ENGINE = Memory AS SELECT * FROM system.numbers FORMAT TSV") - ddl_check_query(instance, "DROP TABLE test.simple_mat_view ON CLUSTER 'cluster' FORMAT TSV") - ddl_check_query(instance, "DROP TABLE IF EXISTS test.super_simple_view2 ON CLUSTER 'cluster' FORMAT TSV") - - ddl_check_query(instance, "CREATE TABLE test.super_simple ON CLUSTER 'cluster' (i Int8) ENGINE = Memory") - ddl_check_query(instance, "RENAME TABLE test.super_simple TO test.super_simple2 ON CLUSTER 'cluster' FORMAT TSV") - ddl_check_query(instance, "DROP TABLE test.super_simple2 ON CLUSTER 'cluster'") - - -def test_on_server_fail(started_cluster): - instance = cluster.instances['ch1'] - kill_instance = cluster.instances['ch2'] - - ddl_check_query(instance, "DROP TABLE IF EXISTS test.test_server_fail ON CLUSTER 'cluster'") - - kill_instance.get_docker_handle().stop() - request = instance.get_query_request("CREATE TABLE test.test_server_fail ON CLUSTER 'cluster' (i Int8) ENGINE=Null", timeout=30) - kill_instance.get_docker_handle().start() - - ddl_check_query(instance, "DROP TABLE IF EXISTS test.__nope__ ON CLUSTER 'cluster'") - - # Check query itself - check_all_hosts_sucesfully_executed(request.get_answer()) - - # And check query artefacts - contents = instance.query("SELECT hostName() AS h FROM all_tables WHERE database='test' AND name='test_server_fail' ORDER BY h") - assert TSV(contents) == TSV("ch1\nch2\nch3\nch4\n") - - ddl_check_query(instance, "DROP TABLE test.test_server_fail ON CLUSTER 'cluster'") - - -def _test_on_connection_losses(cluster, zk_timeout): - instance = cluster.instances['ch1'] - kill_instance = cluster.instances['ch2'] - - with PartitionManager() as pm: - pm.drop_instance_zk_connections(kill_instance) - request = instance.get_query_request("DROP TABLE IF EXISTS test.__nope__ ON CLUSTER 'cluster'", timeout=10) - time.sleep(zk_timeout) - pm.restore_instance_zk_connections(kill_instance) - - check_all_hosts_sucesfully_executed(request.get_answer()) - - -def test_on_connection_loss(started_cluster): - _test_on_connection_losses(cluster, 1.5) # connection loss will occur only (3 sec ZK timeout in config) - - -def test_on_session_expired(started_cluster): - _test_on_connection_losses(cluster, 4) # session should be expired (3 sec ZK timeout in config) - - -def test_replicated_alters(started_cluster): - instance = cluster.instances['ch2'] - - ddl_check_query(instance, "DROP TABLE IF EXISTS merge_for_alter ON CLUSTER cluster") - ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_32 ON CLUSTER cluster") - ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_64 ON CLUSTER cluster") - - if not TEST_REPLICATED_ALTERS: - return - - # Temporarily disable random ZK packet drops, they might broke creation if ReplicatedMergeTree replicas - firewall_drops_rules = cluster.pm_random_drops.pop_rules() - - ddl_check_query(instance, """ -CREATE TABLE IF NOT EXISTS merge_for_alter ON CLUSTER cluster (p Date, i Int32) -ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/hits', '{replica}', p, p, 1) -""") - - ddl_check_query(instance, """ -CREATE TABLE IF NOT EXISTS all_merge_32 ON CLUSTER cluster (p Date, i Int32) -ENGINE = Distributed(cluster, default, merge_for_alter, i) -""") - ddl_check_query(instance, """ -CREATE TABLE IF NOT EXISTS all_merge_64 ON CLUSTER cluster (p Date, i Int64, s String) -ENGINE = Distributed(cluster, default, merge_for_alter, i) -""") - - for i in xrange(4): - k = (i / 2) * 2 - insert_reliable(cluster.instances['ch{}'.format(i + 1)], "INSERT INTO merge_for_alter (i) VALUES ({})({})".format(k, k+1)) - - assert TSV(instance.query("SELECT i FROM all_merge_32 ORDER BY i")) == TSV(''.join(['{}\n'.format(x) for x in xrange(4)])) - - - ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster MODIFY COLUMN i Int64") - ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster ADD COLUMN s DEFAULT toString(i)") - - assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)])) - - - for i in xrange(4): - k = (i / 2) * 2 + 4 - insert_reliable(cluster.instances['ch{}'.format(i + 1)], "INSERT INTO merge_for_alter (p, i) VALUES (31, {})(31, {})".format(k, k+1)) - - assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(8)])) - - - ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster DETACH PARTITION 197002") - assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)])) - - ddl_check_query(instance, "DROP TABLE merge_for_alter ON CLUSTER cluster") - - # Enable random ZK packet drops - cluster.pm_random_drops.push_rules(firewall_drops_rules) - - ddl_check_query(instance, "DROP TABLE all_merge_32 ON CLUSTER cluster") - ddl_check_query(instance, "DROP TABLE all_merge_64 ON CLUSTER cluster") - - -def test_simple_alters(started_cluster): - instance = cluster.instances['ch2'] - - ddl_check_query(instance, "DROP TABLE IF EXISTS merge ON CLUSTER cluster_without_replication") - ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_32 ON CLUSTER cluster_without_replication") - ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_64 ON CLUSTER cluster_without_replication") - - ddl_check_query(instance, """ -CREATE TABLE IF NOT EXISTS merge ON CLUSTER cluster_without_replication (p Date, i Int32) -ENGINE = MergeTree(p, p, 1) -""") - ddl_check_query(instance, """ -CREATE TABLE IF NOT EXISTS all_merge_32 ON CLUSTER cluster_without_replication (p Date, i Int32) -ENGINE = Distributed(cluster_without_replication, default, merge, i) -""") - ddl_check_query(instance, """ -CREATE TABLE IF NOT EXISTS all_merge_64 ON CLUSTER cluster_without_replication (p Date, i Int64, s String) -ENGINE = Distributed(cluster_without_replication, default, merge, i) -""") - - for i in xrange(4): - k = (i / 2) * 2 - cluster.instances['ch{}'.format(i + 1)].query("INSERT INTO merge (i) VALUES ({})({})".format(k, k+1)) - - assert TSV(instance.query("SELECT i FROM all_merge_32 ORDER BY i")) == TSV(''.join(['{}\n'.format(x) for x in xrange(4)])) - - - time.sleep(5) - ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER cluster_without_replication MODIFY COLUMN i Int64") - time.sleep(5) - ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER cluster_without_replication ADD COLUMN s DEFAULT toString(i) FORMAT TSV") - - assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)])) - - - for i in xrange(4): - k = (i / 2) * 2 + 4 - cluster.instances['ch{}'.format(i + 1)].query("INSERT INTO merge (p, i) VALUES (31, {})(31, {})".format(k, k+1)) - - assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(8)])) - - - ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER cluster_without_replication DETACH PARTITION 197002") - assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)])) - - ddl_check_query(instance, "DROP TABLE merge ON CLUSTER cluster_without_replication") - ddl_check_query(instance, "DROP TABLE all_merge_32 ON CLUSTER cluster_without_replication") - ddl_check_query(instance, "DROP TABLE all_merge_64 ON CLUSTER cluster_without_replication") - - -def test_macro(started_cluster): - instance = cluster.instances['ch2'] - ddl_check_query(instance, "CREATE TABLE tab ON CLUSTER '{cluster}' (value UInt8) ENGINE = Memory") - - for i in xrange(4): - insert_reliable(cluster.instances['ch{}'.format(i + 1)], "INSERT INTO tab VALUES ({})".format(i)) - - ddl_check_query(instance, "CREATE TABLE distr ON CLUSTER '{cluster}' (value UInt8) ENGINE = Distributed('{cluster}', 'default', 'tab', value % 4)") - - assert TSV(instance.query("SELECT value FROM distr ORDER BY value")) == TSV('0\n1\n2\n3\n') - assert TSV( cluster.instances['ch3'].query("SELECT value FROM distr ORDER BY value")) == TSV('0\n1\n2\n3\n') - - ddl_check_query(instance, "DROP TABLE IF EXISTS distr ON CLUSTER '{cluster}'") - ddl_check_query(instance, "DROP TABLE IF EXISTS tab ON CLUSTER '{cluster}'") - - -def test_implicit_macros(started_cluster): - # Temporarily disable random ZK packet drops, they might broke creation if ReplicatedMergeTree replicas - firewall_drops_rules = cluster.pm_random_drops.pop_rules() - - instance = cluster.instances['ch2'] - - ddl_check_query(instance, "DROP DATABASE IF EXISTS test_db ON CLUSTER '{cluster}'") - ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS test_db ON CLUSTER '{cluster}'") - - ddl_check_query(instance, """ -CREATE TABLE IF NOT EXISTS test_db.test_macro ON CLUSTER '{cluster}' (p Date, i Int32) -ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/{layer}-{shard}/{table}', '{replica}', p, p, 1) -""") - - # Check that table was created at correct path in zookeeper - assert cluster.get_kazoo_client('zoo1').exists('/clickhouse/tables/test_db/0-1/test_macro') is not None - - # Enable random ZK packet drops - cluster.pm_random_drops.push_rules(firewall_drops_rules) - - -def test_allowed_databases(started_cluster): - instance = cluster.instances['ch2'] - instance.query("CREATE DATABASE IF NOT EXISTS db1 ON CLUSTER cluster") - instance.query("CREATE DATABASE IF NOT EXISTS db2 ON CLUSTER cluster") - - instance.query("CREATE TABLE db1.t1 ON CLUSTER cluster (i Int8) ENGINE = Memory", settings={"user" : "restricted_user"}) - - with pytest.raises(Exception): - instance.query("CREATE TABLE db2.t2 ON CLUSTER cluster (i Int8) ENGINE = Memory", settings={"user" : "restricted_user"}) - with pytest.raises(Exception): - instance.query("CREATE TABLE t3 ON CLUSTER cluster (i Int8) ENGINE = Memory", settings={"user" : "restricted_user"}) - with pytest.raises(Exception): - instance.query("DROP DATABASE db2 ON CLUSTER cluster", settings={"user" : "restricted_user"}) - - instance.query("DROP DATABASE db1 ON CLUSTER cluster", settings={"user" : "restricted_user"}) - -def test_kill_query(started_cluster): - instance = cluster.instances['ch3'] - - ddl_check_query(instance, "KILL QUERY ON CLUSTER 'cluster' WHERE NOT elapsed FORMAT TSV") - -def test_detach_query(started_cluster): - instance = cluster.instances['ch3'] - - ddl_check_query(instance, "DROP TABLE IF EXISTS test_attach ON CLUSTER cluster FORMAT TSV") - ddl_check_query(instance, "CREATE TABLE test_attach ON CLUSTER cluster (i Int8)ENGINE = Log") - ddl_check_query(instance, "DETACH TABLE test_attach ON CLUSTER cluster FORMAT TSV") - ddl_check_query(instance, "ATTACH TABLE test_attach ON CLUSTER cluster") - - -def test_optimize_query(started_cluster): - instance = cluster.instances['ch3'] - - ddl_check_query(instance, "DROP TABLE IF EXISTS test_optimize ON CLUSTER cluster FORMAT TSV") - ddl_check_query(instance, "CREATE TABLE test_optimize ON CLUSTER cluster (p Date, i Int32) ENGINE = MergeTree(p, p, 8192)") - ddl_check_query(instance, "OPTIMIZE TABLE test_optimize ON CLUSTER cluster FORMAT TSV") - - -def test_create_as_select(started_cluster): - instance = cluster.instances['ch2'] - ddl_check_query(instance, "CREATE TABLE test_as_select ON CLUSTER cluster ENGINE = Memory AS (SELECT 1 AS x UNION ALL SELECT 2 AS x)") - assert TSV(instance.query("SELECT x FROM test_as_select ORDER BY x")) == TSV("1\n2\n") - ddl_check_query(instance, "DROP TABLE IF EXISTS test_as_select ON CLUSTER cluster") - - -if __name__ == '__main__': - with contextmanager(started_cluster)() as cluster: - for name, instance in cluster.instances.items(): - print name, instance.ip_address - raw_input("Cluster created, press any key to destroy...") diff --git a/dbms/tests/integration/test_filesystem_layout/__init__.py b/dbms/tests/integration/test_filesystem_layout/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_filesystem_layout/test.py b/dbms/tests/integration/test_filesystem_layout/test.py new file mode 100644 index 00000000000..16d63cbf2b7 --- /dev/null +++ b/dbms/tests/integration/test_filesystem_layout/test.py @@ -0,0 +1,28 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance("node") + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_file_path_escaping(started_cluster): + node.query('CREATE DATABASE IF NOT EXISTS test') + node.query(''' + CREATE TABLE test.`T.a_b,l-e!` (`~Id` UInt32) + ENGINE = MergeTree() PARTITION BY `~Id` ORDER BY `~Id`; + ''') + node.query('''INSERT INTO test.`T.a_b,l-e!` VALUES (1);''') + node.query('''ALTER TABLE test.`T.a_b,l-e!` FREEZE;''') + + node.exec_in_container(["bash", "-c", "test -f /var/lib/clickhouse/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"]) + node.exec_in_container(["bash", "-c", "test -f /var/lib/clickhouse/shadow/1/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"]) diff --git a/dbms/tests/integration/test_globs_in_filepath/test.py b/dbms/tests/integration/test_globs_in_filepath/test.py index db794c35d47..1ca3d003f1d 100644 --- a/dbms/tests/integration/test_globs_in_filepath/test.py +++ b/dbms/tests/integration/test_globs_in_filepath/test.py @@ -10,7 +10,12 @@ path_to_userfiles_from_defaut_config = "/var/lib/clickhouse/user_files/" # sho def start_cluster(): try: cluster.start() + yield cluster + + except Exception as ex: + print(ex) + raise ex finally: cluster.shutdown() @@ -57,6 +62,7 @@ def test_linear_structure(start_cluster): test_requests = [("file{0..9}", "10"), ("file?", "10"), + ("nothing*", "0"), ("file{0..9}{0..9}{0..9}", "10"), ("file???", "10"), ("file*", "20"), @@ -92,7 +98,7 @@ def test_deep_structure(start_cluster): for i in range(10): for j in range(10): for k in range(10): - files.append("directory1/big_dir/file"+str(i)+str(j)+str(k)) + files.append("directory1/big_dir/file" + str(i) + str(j) + str(k)) for dir in dirs: files.append(dir+"file") @@ -112,4 +118,12 @@ def test_deep_structure(start_cluster): '''.format(pattern)) == '{}\n'.format(value) assert node.query(''' select count(*) from file('{}{}', 'TSV', 'text String, number Float64') - '''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value) \ No newline at end of file + '''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value) + +def test_table_function(start_cluster): + node.exec_in_container(['bash', '-c', 'mkdir -p {}some/path/to/'.format(path_to_userfiles_from_defaut_config)]) + node.exec_in_container(['bash', '-c', 'touch {}some/path/to/data.CSV'.format(path_to_userfiles_from_defaut_config)]) + node.query("insert into table function file('some/path/to/data.CSV', CSV, 'n UInt8, s String') select number, concat('str_', toString(number)) from numbers(100000)") + assert node.query("select count() from file('some/path/to/data.CSV', CSV, 'n UInt8, s String')").rstrip() == '100000' + node.query("insert into table function file('nonexist.csv', 'CSV', 'val1 UInt32') values (1)") + assert node.query("select * from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip()== '1' diff --git a/dbms/tests/integration/test_https_replication/test.py b/dbms/tests/integration/test_https_replication/test.py index d9f9b1f23f4..a34c5faeccc 100644 --- a/dbms/tests/integration/test_https_replication/test.py +++ b/dbms/tests/integration/test_https_replication/test.py @@ -4,6 +4,9 @@ import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry +from helpers.network import PartitionManager +from multiprocessing.dummy import Pool +import random """ Both ssl_conf.xml and no_ssl_conf.xml have the same port @@ -46,6 +49,35 @@ def test_both_https(both_https_cluster): assert_eq_with_retry(node1, "SELECT id FROM test_table order by id", '111\n222') assert_eq_with_retry(node2, "SELECT id FROM test_table order by id", '111\n222') + +def test_replication_after_partition(both_https_cluster): + node1.query("truncate table test_table") + node2.query("truncate table test_table") + + manager = PartitionManager() + + def close(num): + manager.partition_instances(node1, node2, port=9010) + time.sleep(1) + manager.heal_all() + + def insert_data_and_check(num): + node1.query("insert into test_table values('2019-10-15', {}, 888)".format(num)) + time.sleep(0.5) + + closing_pool = Pool(1) + inserting_pool = Pool(5) + cres = closing_pool.map_async(close, [random.randint(1, 3) for _ in range(10)]) + ires = inserting_pool.map_async(insert_data_and_check, range(100)) + + cres.wait() + ires.wait() + + assert_eq_with_retry(node1, "SELECT count() FROM test_table", '100') + assert_eq_with_retry(node2, "SELECT count() FROM test_table", '100') + + + node3 = cluster.add_instance('node3', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], with_zookeeper=True) node4 = cluster.add_instance('node4', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], with_zookeeper=True) diff --git a/dbms/tests/integration/test_insert_into_distributed/test.py b/dbms/tests/integration/test_insert_into_distributed/test.py index 622fb01eff6..8d656cef3ea 100644 --- a/dbms/tests/integration/test_insert_into_distributed/test.py +++ b/dbms/tests/integration/test_insert_into_distributed/test.py @@ -68,6 +68,14 @@ CREATE TABLE low_cardinality (d Date, x UInt32, s LowCardinality(String)) ENGINE shard1.query(''' CREATE TABLE low_cardinality_all (d Date, x UInt32, s LowCardinality(String)) ENGINE = Distributed('shard_with_low_cardinality', 'default', 'low_cardinality', sipHash64(s))''') + node1.query(''' +CREATE TABLE table_function (n UInt8, s String) ENGINE = MergeTree() ORDER BY n''') + + node2.query(''' +CREATE TABLE table_function (n UInt8, s String) ENGINE = MergeTree() ORDER BY n''') + + + yield cluster finally: @@ -189,3 +197,7 @@ def test_inserts_low_cardinality(started_cluster): instance.query("INSERT INTO low_cardinality_all (d,x,s) VALUES ('2018-11-12',1,'123')") time.sleep(0.5) assert instance.query("SELECT count(*) FROM low_cardinality_all").strip() == '1' + +def test_table_function(started_cluster): + node1.query("insert into table function cluster('shard_with_local_replica', 'default', 'table_function') select number, concat('str_', toString(number)) from numbers(100000)") + assert node1.query("select count() from cluster('shard_with_local_replica', 'default', 'table_function')").rstrip() == '100000' diff --git a/dbms/tests/integration/test_multiple_disks/configs/config.d/storage_configuration.xml b/dbms/tests/integration/test_multiple_disks/configs/config.d/storage_configuration.xml index 2e6a1f80a6d..f2e4eb01534 100644 --- a/dbms/tests/integration/test_multiple_disks/configs/config.d/storage_configuration.xml +++ b/dbms/tests/integration/test_multiple_disks/configs/config.d/storage_configuration.xml @@ -45,7 +45,7 @@ - +
diff --git a/dbms/tests/integration/test_multiple_disks/test.py b/dbms/tests/integration/test_multiple_disks/test.py index 446eca88142..91ecf5a8b8d 100644 --- a/dbms/tests/integration/test_multiple_disks/test.py +++ b/dbms/tests/integration/test_multiple_disks/test.py @@ -3,6 +3,7 @@ import pytest import random import re import string +import threading import time from multiprocessing.dummy import Pool from helpers.client import QueryRuntimeException @@ -15,6 +16,7 @@ node1 = cluster.add_instance('node1', config_dir='configs', main_configs=['configs/logs_config.xml'], with_zookeeper=True, + stay_alive=True, tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/external:size=200M'], macros={"shard": 0, "replica": 1} ) @@ -22,6 +24,7 @@ node2 = cluster.add_instance('node2', config_dir='configs', main_configs=['configs/logs_config.xml'], with_zookeeper=True, + stay_alive=True, tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/external:size=200M'], macros={"shard": 0, "replica": 2} ) @@ -768,12 +771,42 @@ def test_concurrent_alter_move_and_drop(start_cluster, name, engine): node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) +@pytest.mark.parametrize("name,engine", [ + ("detach_attach_mt","MergeTree()"), + ("replicated_detach_attach_mt","ReplicatedMergeTree('/clickhouse/replicated_detach_attach_mt', '1')",), +]) +def test_detach_attach(start_cluster, name, engine): + try: + node1.query(""" + CREATE TABLE {name} ( + s1 String + ) ENGINE = {engine} + ORDER BY tuple() + SETTINGS storage_policy='moving_jbod_with_external' + """.format(name=name, engine=engine)) + + data = [] # 5MB in total + for i in range(5): + data.append(get_random_string(1024 * 1024)) # 1MB row + node1.query("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) + + node1.query("ALTER TABLE {} DETACH PARTITION tuple()".format(name)) + assert node1.query("SELECT count() FROM {}".format(name)).strip() == "0" + + assert node1.query("SELECT disk FROM system.detached_parts WHERE table = '{}'".format(name)).strip() == "jbod1" + + node1.query("ALTER TABLE {} ATTACH PARTITION tuple()".format(name)) + assert node1.query("SELECT count() FROM {}".format(name)).strip() == "5" + + finally: + node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + + @pytest.mark.parametrize("name,engine", [ ("mutating_mt","MergeTree()"), ("replicated_mutating_mt","ReplicatedMergeTree('/clickhouse/replicated_mutating_mt', '1')",), ]) def test_mutate_to_another_disk(start_cluster, name, engine): - try: node1.query(""" CREATE TABLE {name} ( @@ -998,6 +1031,7 @@ def test_rename(start_cluster): node1.query("DROP TABLE IF EXISTS default.renaming_table1") node1.query("DROP TABLE IF EXISTS test.renaming_table2") + def test_freeze(start_cluster): try: node1.query(""" @@ -1027,6 +1061,109 @@ def test_freeze(start_cluster): node1.exec_in_container(["bash", "-c", "find /jbod1/shadow -name '*.mrk2' | grep '.*'"]) node1.exec_in_container(["bash", "-c", "find /external/shadow -name '*.mrk2' | grep '.*'"]) - finally: node1.query("DROP TABLE IF EXISTS default.freezing_table") + + +def test_kill_while_insert(start_cluster): + try: + name = "test_kill_while_insert" + + node1.query(""" + CREATE TABLE {name} ( + s String + ) ENGINE = MergeTree + ORDER BY tuple() + SETTINGS storage_policy='small_jbod_with_external' + """.format(name=name)) + + data = [] + dates = [] + for i in range(10): + data.append(get_random_string(1024 * 1024)) # 1MB value + node1.query("INSERT INTO {name} VALUES {}".format(','.join(["('" + s + "')" for s in data]), name=name)) + + disks = get_used_disks_for_table(node1, name) + assert set(disks) == {"jbod1"} + + start_time = time.time() + long_select = threading.Thread(target=node1.query, args=("SELECT sleep(3) FROM {name}".format(name=name),)) + long_select.start() + + time.sleep(0.5) + + node1.query("ALTER TABLE {name} MOVE PARTITION tuple() TO DISK 'external'".format(name=name)) + assert time.time() - start_time < 2 + node1.restart_clickhouse(kill=True) + + try: + long_select.join() + except: + """""" + + time.sleep(0.5) + assert node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == ["10"] + + finally: + try: + node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + except: + """ClickHouse may be inactive at this moment and we don't want to mask a meaningful exception.""" + + +def test_move_while_merge(start_cluster): + try: + name = "test_move_while_merge" + + node1.query(""" + CREATE TABLE {name} ( + n Int64 + ) ENGINE = MergeTree + ORDER BY sleep(2) + SETTINGS storage_policy='small_jbod_with_external' + """.format(name=name)) + + node1.query("INSERT INTO {name} VALUES (1)".format(name=name)) + node1.query("INSERT INTO {name} VALUES (2)".format(name=name)) + + parts = node1.query("SELECT name FROM system.parts WHERE table = '{name}' AND active = 1".format(name=name)).splitlines() + assert len(parts) == 2 + + def optimize(): + node1.query("OPTIMIZE TABLE {name}".format(name=name)) + + optimize = threading.Thread(target=optimize) + optimize.start() + + time.sleep(0.5) + + with pytest.raises(QueryRuntimeException): + node1.query("ALTER TABLE {name} MOVE PART '{part}' TO DISK 'external'".format(name=name, part=parts[0])) + + exiting = False + no_exception = {} + + def alter(): + while not exiting: + try: + node1.query("ALTER TABLE {name} MOVE PART '{part}' TO DISK 'external'".format(name=name, part=parts[0])) + no_exception['missing'] = 'exception' + break + except QueryRuntimeException: + """""" + + alter_thread = threading.Thread(target=alter) + alter_thread.start() + + optimize.join() + + time.sleep(0.5) + + exiting = True + alter_thread.join() + assert len(no_exception) == 0 + + assert node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == ["2"] + + finally: + node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) diff --git a/dbms/tests/integration/test_mysql_database_engine/test.py b/dbms/tests/integration/test_mysql_database_engine/test.py index 430083ccc82..9e776b65f32 100644 --- a/dbms/tests/integration/test_mysql_database_engine/test.py +++ b/dbms/tests/integration/test_mysql_database_engine/test.py @@ -1,138 +1,101 @@ -from contextlib import contextmanager - import time -import pytest +import contextlib -## sudo -H pip install PyMySQL import pymysql.cursors +import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) - -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_mysql = True) -create_table_normal_sql_template = """ - CREATE TABLE `clickhouse`.`{}` ( - `id` int(11) NOT NULL, - `name` varchar(50) NOT NULL, - `age` int NOT NULL default 0, - `money` int NOT NULL default 0, - PRIMARY KEY (`id`) - ) ENGINE=InnoDB; - """ - -create_table_mysql_style_sql_template = """ - CREATE TABLE `clickhouse`.`{}` ( - `id` int(11) NOT NULL, - `float` float NOT NULL, - `Float32` float NOT NULL, - `test``name` varchar(50) NOT NULL, - PRIMARY KEY (`id`) - ) ENGINE=InnoDB; - """ - -drop_table_sql_template = "DROP TABLE `clickhouse`.`{}`" - -add_column_sql_template = "ALTER TABLE `clickhouse`.`{}` ADD COLUMN `pid` int(11)" -del_column_sql_template = "ALTER TABLE `clickhouse`.`{}` DROP COLUMN `pid`" +clickhouse_node = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_mysql=True) @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - - conn = get_mysql_conn() - ## create mysql db and table - create_mysql_db(conn, 'clickhouse') - node1.query("CREATE DATABASE clickhouse_mysql ENGINE = MySQL('mysql1:3306', 'clickhouse', 'root', 'clickhouse')") yield cluster - finally: cluster.shutdown() -def test_sync_tables_list_between_clickhouse_and_mysql(started_cluster): - mysql_connection = get_mysql_conn() - assert node1.query('SHOW TABLES FROM clickhouse_mysql FORMAT TSV').rstrip() == '' +class MySQLNodeInstance: + def __init__(self, user='root', password='clickhouse', hostname='127.0.0.1', port=3308): + self.user = user + self.port = port + self.hostname = hostname + self.password = password + self.mysql_connection = None # lazy init - create_normal_mysql_table(mysql_connection, 'first_mysql_table') - assert node1.query("SHOW TABLES FROM clickhouse_mysql LIKE 'first_mysql_table' FORMAT TSV").rstrip() == 'first_mysql_table' + def query(self, execution_query): + if self.mysql_connection is None: + self.mysql_connection = pymysql.connect(user=self.user, password=self.password, host=self.hostname, port=self.port) + with self.mysql_connection.cursor() as cursor: + cursor.execute(execution_query) - create_normal_mysql_table(mysql_connection, 'second_mysql_table') - assert node1.query("SHOW TABLES FROM clickhouse_mysql LIKE 'second_mysql_table' FORMAT TSV").rstrip() == 'second_mysql_table' + def close(self): + if self.mysql_connection is not None: + self.mysql_connection.close() - drop_mysql_table(mysql_connection, 'second_mysql_table') - assert node1.query("SHOW TABLES FROM clickhouse_mysql LIKE 'second_mysql_table' FORMAT TSV").rstrip() == '' - mysql_connection.close() +def test_mysql_ddl_for_mysql_database(started_cluster): + with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', '127.0.0.1', port=3308)) as mysql_node: + mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") -def test_sync_tables_structure_between_clickhouse_and_mysql(started_cluster): - mysql_connection = get_mysql_conn() + clickhouse_node.query("CREATE DATABASE test_database ENGINE = MySQL('mysql1:3306', 'test_database', 'root', 'clickhouse')") + assert 'test_database' in clickhouse_node.query('SHOW DATABASES') - create_normal_mysql_table(mysql_connection, 'test_sync_column') + mysql_node.query('CREATE TABLE `test_database`.`test_table` ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;') + assert 'test_table' in clickhouse_node.query('SHOW TABLES FROM test_database') - assert node1.query( - "SELECT name FROM system.columns WHERE table = 'test_sync_column' AND database = 'clickhouse_mysql' AND name = 'pid' ").rstrip() == '' + time.sleep(3) # Because the unit of MySQL modification time is seconds, modifications made in the same second cannot be obtained + mysql_node.query('ALTER TABLE `test_database`.`test_table` ADD COLUMN `add_column` int(11)') + assert 'add_column' in clickhouse_node.query("SELECT name FROM system.columns WHERE table = 'test_table' AND database = 'test_database'") - time.sleep(3) - add_mysql_table_column(mysql_connection, "test_sync_column") + time.sleep(3) # Because the unit of MySQL modification time is seconds, modifications made in the same second cannot be obtained + mysql_node.query('ALTER TABLE `test_database`.`test_table` DROP COLUMN `add_column`') + assert 'add_column' not in clickhouse_node.query("SELECT name FROM system.columns WHERE table = 'test_table' AND database = 'test_database'") - assert node1.query( - "SELECT name FROM system.columns WHERE table = 'test_sync_column' AND database = 'clickhouse_mysql' AND name = 'pid' ").rstrip() == 'pid' + mysql_node.query('DROP TABLE `test_database`.`test_table`;') + assert 'test_table' not in clickhouse_node.query('SHOW TABLES FROM test_database') - time.sleep(3) - drop_mysql_table_column(mysql_connection, "test_sync_column") - assert node1.query( - "SELECT name FROM system.columns WHERE table = 'test_sync_column' AND database = 'clickhouse_mysql' AND name = 'pid' ").rstrip() == '' + clickhouse_node.query("DROP DATABASE test_database") + assert 'test_database' not in clickhouse_node.query('SHOW DATABASES') - mysql_connection.close() + mysql_node.query("DROP DATABASE test_database") -def test_insert_select(started_cluster): - mysql_connection = get_mysql_conn() - create_normal_mysql_table(mysql_connection, 'test_insert_select') - assert node1.query("SELECT count() FROM `clickhouse_mysql`.{}".format('test_insert_select')).rstrip() == '0' - node1.query("INSERT INTO `clickhouse_mysql`.{}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format('test_insert_select')) - assert node1.query("SELECT count() FROM `clickhouse_mysql`.{}".format('test_insert_select')).rstrip() == '10000' - assert node1.query("SELECT sum(money) FROM `clickhouse_mysql`.{}".format('test_insert_select')).rstrip() == '30000' - mysql_connection.close() +def test_clickhouse_ddl_for_mysql_database(started_cluster): + with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', '127.0.0.1', port=3308)) as mysql_node: + mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") + mysql_node.query('CREATE TABLE `test_database`.`test_table` ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;') -def test_insert_select_with_mysql_style_table(started_cluster): - mysql_connection = get_mysql_conn() - create_mysql_style_mysql_table(mysql_connection, 'test_mysql``_style_table') + clickhouse_node.query("CREATE DATABASE test_database ENGINE = MySQL('mysql1:3306', 'test_database', 'root', 'clickhouse')") - assert node1.query("SELECT count() FROM `clickhouse_mysql`.`{}`".format('test_mysql\`_style_table')).rstrip() == '0' - node1.query("INSERT INTO `clickhouse_mysql`.`{}`(id, `float`, `Float32`, `test\`name`) select number, 3, 3, 'name' from numbers(10000) ".format('test_mysql\`_style_table')) - assert node1.query("SELECT count() FROM `clickhouse_mysql`.`{}`".format('test_mysql\`_style_table')).rstrip() == '10000' - assert node1.query("SELECT sum(`float`) FROM `clickhouse_mysql`.`{}`".format('test_mysql\`_style_table')).rstrip() == '30000' - mysql_connection.close() + assert 'test_table' in clickhouse_node.query('SHOW TABLES FROM test_database') + clickhouse_node.query("DROP TABLE test_database.test_table") + assert 'test_table' not in clickhouse_node.query('SHOW TABLES FROM test_database') + clickhouse_node.query("ATTACH TABLE test_database.test_table") + assert 'test_table' in clickhouse_node.query('SHOW TABLES FROM test_database') + clickhouse_node.query("DETACH TABLE test_database.test_table") + assert 'test_table' not in clickhouse_node.query('SHOW TABLES FROM test_database') + clickhouse_node.query("ATTACH TABLE test_database.test_table") + assert 'test_table' in clickhouse_node.query('SHOW TABLES FROM test_database') -def get_mysql_conn(): - conn = pymysql.connect(user='root', password='clickhouse', host='127.0.0.1', port=3308) - return conn + clickhouse_node.query("DROP DATABASE test_database") + assert 'test_database' not in clickhouse_node.query('SHOW DATABASES') -def create_mysql_db(conn, name): - with conn.cursor() as cursor: - cursor.execute( - "CREATE DATABASE {} DEFAULT CHARACTER SET 'utf8'".format(name)) + mysql_node.query("DROP DATABASE test_database") -def create_normal_mysql_table(conn, table_name): - with conn.cursor() as cursor: - cursor.execute(create_table_normal_sql_template.format(table_name)) -def create_mysql_style_mysql_table(conn, table_name): - with conn.cursor() as cursor: - cursor.execute(create_table_mysql_style_sql_template.format(table_name)) +def test_clickhouse_dml_for_mysql_database(started_cluster): + with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', '127.0.0.1', port=3308)) as mysql_node: + mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") + mysql_node.query('CREATE TABLE `test_database`.`test_table` ( `i``d` int(11) NOT NULL, PRIMARY KEY (`i``d`)) ENGINE=InnoDB;') + clickhouse_node.query("CREATE DATABASE test_database ENGINE = MySQL('mysql1:3306', 'test_database', 'root', 'clickhouse')") -def drop_mysql_table(conn, table_name): - with conn.cursor() as cursor: - cursor.execute(drop_table_sql_template.format(table_name)) + assert clickhouse_node.query("SELECT count() FROM `test_database`.`test_table`").rstrip() == '0' + clickhouse_node.query("INSERT INTO `test_database`.`test_table`(`i\`d`) select number from numbers(10000)") + assert clickhouse_node.query("SELECT count() FROM `test_database`.`test_table`").rstrip() == '10000' -def add_mysql_table_column(conn, table_name): - with conn.cursor() as cursor: - cursor.execute(add_column_sql_template.format(table_name)) - -def drop_mysql_table_column(conn, table_name): - with conn.cursor() as cursor: - cursor.execute(del_column_sql_template.format(table_name)) + mysql_node.query("DROP DATABASE test_database") diff --git a/dbms/tests/integration/test_mysql_protocol/clients/golang/0.reference b/dbms/tests/integration/test_mysql_protocol/clients/golang/0.reference index a151cc2592e..5bfb8b0d1cb 100644 --- a/dbms/tests/integration/test_mysql_protocol/clients/golang/0.reference +++ b/dbms/tests/integration/test_mysql_protocol/clients/golang/0.reference @@ -1,7 +1,7 @@ Columns: a Column types: -a BINARY +a BIGINT Result: 0 1 @@ -10,7 +10,7 @@ name a Column types: name BINARY -a BINARY +a TINYINT Result: tables 1 Columns: @@ -18,6 +18,6 @@ a b Column types: a BINARY -b BINARY +b TINYINT Result: тест 1 diff --git a/dbms/tests/integration/test_mysql_protocol/clients/mysql/docker_compose.yml b/dbms/tests/integration/test_mysql_protocol/clients/mysql/docker_compose.yml index 777e2bad2e3..752e59a2d08 100644 --- a/dbms/tests/integration/test_mysql_protocol/clients/mysql/docker_compose.yml +++ b/dbms/tests/integration/test_mysql_protocol/clients/mysql/docker_compose.yml @@ -2,5 +2,12 @@ version: '2.2' services: mysql1: image: mysql:5.7 - # rewriting default command, because starting server is unnecessary - command: sleep infinity + restart: always + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: 1 + command: --federated --socket /var/run/mysqld/mysqld.sock + healthcheck: + test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] + interval: 1s + timeout: 2s + retries: 100 diff --git a/dbms/tests/integration/test_mysql_protocol/configs/users.xml b/dbms/tests/integration/test_mysql_protocol/configs/users.xml index ebcd1a297e1..b88dfbada37 100644 --- a/dbms/tests/integration/test_mysql_protocol/configs/users.xml +++ b/dbms/tests/integration/test_mysql_protocol/configs/users.xml @@ -15,6 +15,16 @@ default + + + 65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5 + + ::/0 + + default + default + + e395796d6546b1b65db9d665cd43f0e858dd4303 diff --git a/dbms/tests/integration/test_mysql_protocol/test.py b/dbms/tests/integration/test_mysql_protocol/test.py index f8d79cb2e32..3f4f4e2a2f8 100644 --- a/dbms/tests/integration/test_mysql_protocol/test.py +++ b/dbms/tests/integration/test_mysql_protocol/test.py @@ -1,9 +1,12 @@ # coding: utf-8 -import os import docker +import datetime +import math +import os import pytest import subprocess +import time import pymysql.connections from docker.models.containers import Container @@ -36,6 +39,25 @@ def mysql_client(): yield docker.from_env().containers.get(cluster.project_name + '_mysql1_1') +@pytest.fixture(scope='module') +def mysql_server(mysql_client): + """Return MySQL container when it is healthy. + + :type mysql_client: Container + :rtype: Container + """ + retries = 30 + for i in range(retries): + info = mysql_client.client.api.inspect_container(mysql_client.name) + if info['State']['Health']['Status'] == 'healthy': + break + time.sleep(1) + else: + raise Exception('Mysql server has not started in %d seconds.' % retries) + + return mysql_client + + @pytest.fixture(scope='module') def golang_container(): docker_compose = os.path.join(SCRIPT_DIR, 'clients', 'golang', 'docker_compose.yml') @@ -109,7 +131,51 @@ def test_mysql_client(mysql_client, server_address): assert stdout == '\n'.join(['column', '0', '0', '1', '1', '5', '5', 'tmp_column', '0', '1', '']) +def test_mysql_federated(mysql_server, server_address): + node.query('''DROP DATABASE IF EXISTS mysql_federated''', settings={"password": "123"}) + node.query('''CREATE DATABASE mysql_federated''', settings={"password": "123"}) + node.query('''CREATE TABLE mysql_federated.test (col UInt32) ENGINE = Log''', settings={"password": "123"}) + node.query('''INSERT INTO mysql_federated.test VALUES (0), (1), (5)''', settings={"password": "123"}) + + code, (_, stderr) = mysql_server.exec_run(''' + mysql + -e "DROP SERVER IF EXISTS clickhouse;" + -e "CREATE SERVER clickhouse FOREIGN DATA WRAPPER mysql OPTIONS (USER 'default', PASSWORD '123', HOST '{host}', PORT {port}, DATABASE 'mysql_federated');" + -e "DROP DATABASE IF EXISTS mysql_federated;" + -e "CREATE DATABASE mysql_federated;" + '''.format(host=server_address, port=server_port), demux=True) + + assert code == 0 + + code, (stdout, stderr) = mysql_server.exec_run(''' + mysql + -e "CREATE TABLE mysql_federated.test(`col` int UNSIGNED) ENGINE=FEDERATED CONNECTION='clickhouse';" + -e "SELECT * FROM mysql_federated.test ORDER BY col;" + '''.format(host=server_address, port=server_port), demux=True) + + assert stdout == '\n'.join(['col', '0', '1', '5', '']) + + code, (stdout, stderr) = mysql_server.exec_run(''' + mysql + -e "INSERT INTO mysql_federated.test VALUES (0), (1), (5);" + -e "SELECT * FROM mysql_federated.test ORDER BY col;" + '''.format(host=server_address, port=server_port), demux=True) + + assert stdout == '\n'.join(['col', '0', '0', '1', '1', '5', '5', '']) + + def test_python_client(server_address): + client = pymysql.connections.Connection(host=server_address, user='user_with_double_sha1', password='abacaba', database='default', port=server_port) + + with pytest.raises(pymysql.InternalError) as exc_info: + client.query('select name from tables') + + assert exc_info.value.args == (60, "Table default.tables doesn't exist.") + + cursor = client.cursor(pymysql.cursors.DictCursor) + cursor.execute("select 1 as a, 'тест' as b") + assert cursor.fetchall() == [{'a': 1, 'b': 'тест'}] + with pytest.raises(pymysql.InternalError) as exc_info: pymysql.connections.Connection(host=server_address, user='default', password='abacab', database='default', port=server_port) @@ -124,7 +190,7 @@ def test_python_client(server_address): cursor = client.cursor(pymysql.cursors.DictCursor) cursor.execute("select 1 as a, 'тест' as b") - assert cursor.fetchall() == [{'a': '1', 'b': 'тест'}] + assert cursor.fetchall() == [{'a': 1, 'b': 'тест'}] client.select_db('system') @@ -140,11 +206,14 @@ def test_python_client(server_address): cursor.execute("INSERT INTO table1 VALUES (1), (3)") cursor.execute("INSERT INTO table1 VALUES (1), (4)") cursor.execute("SELECT * FROM table1 ORDER BY a") - assert cursor.fetchall() == [{'a': '1'}, {'a': '1'}, {'a': '3'}, {'a': '4'}] + assert cursor.fetchall() == [{'a': 1}, {'a': 1}, {'a': 3}, {'a': 4}] def test_golang_client(server_address, golang_container): # type: (str, Container) -> None + with open(os.path.join(SCRIPT_DIR, 'clients', 'golang', '0.reference')) as fp: + reference = fp.read() + code, (stdout, stderr) = golang_container.exec_run('./main --host {host} --port {port} --user default --password 123 --database ' 'abc'.format(host=server_address, port=server_port), demux=True) @@ -155,10 +224,12 @@ def test_golang_client(server_address, golang_container): 'default'.format(host=server_address, port=server_port), demux=True) assert code == 0 + assert stdout == reference - with open(os.path.join(SCRIPT_DIR, 'clients', 'golang', '0.reference')) as fp: - reference = fp.read() - assert stdout == reference + code, (stdout, stderr) = golang_container.exec_run('./main --host {host} --port {port} --user user_with_double_sha1 --password abacaba --database ' + 'default'.format(host=server_address, port=server_port), demux=True) + assert code == 0 + assert stdout == reference def test_php_client(server_address, php_container): @@ -171,18 +242,80 @@ def test_php_client(server_address, php_container): assert code == 0 assert stdout == 'tables\n' + code, (stdout, stderr) = php_container.exec_run('php -f test.php {host} {port} user_with_double_sha1 abacaba'.format(host=server_address, port=server_port), demux=True) + assert code == 0 + assert stdout == 'tables\n' + + code, (stdout, stderr) = php_container.exec_run('php -f test_ssl.php {host} {port} user_with_double_sha1 abacaba'.format(host=server_address, port=server_port), demux=True) + assert code == 0 + assert stdout == 'tables\n' + def test_mysqljs_client(server_address, nodejs_container): - code, (_, stderr) = nodejs_container.exec_run('node test.js {host} {port} default 123'.format(host=server_address, port=server_port), demux=True) + code, (_, stderr) = nodejs_container.exec_run('node test.js {host} {port} user_with_sha256 abacaba'.format(host=server_address, port=server_port), demux=True) assert code == 1 assert 'MySQL is requesting the sha256_password authentication method, which is not supported.' in stderr code, (_, stderr) = nodejs_container.exec_run('node test.js {host} {port} user_with_empty_password ""'.format(host=server_address, port=server_port), demux=True) - assert code == 1 - assert 'MySQL is requesting the sha256_password authentication method, which is not supported.' in stderr + assert code == 0 code, (_, _) = nodejs_container.exec_run('node test.js {host} {port} user_with_double_sha1 abacaba'.format(host=server_address, port=server_port), demux=True) assert code == 0 code, (_, _) = nodejs_container.exec_run('node test.js {host} {port} user_with_empty_password 123'.format(host=server_address, port=server_port), demux=True) assert code == 1 + + +def test_types(server_address): + client = pymysql.connections.Connection(host=server_address, user='default', password='123', database='default', port=server_port) + + cursor = client.cursor(pymysql.cursors.DictCursor) + cursor.execute( + "select " + "toInt8(-pow(2, 7)) as Int8_column, " + "toUInt8(pow(2, 8) - 1) as UInt8_column, " + "toInt16(-pow(2, 15)) as Int16_column, " + "toUInt16(pow(2, 16) - 1) as UInt16_column, " + "toInt32(-pow(2, 31)) as Int32_column, " + "toUInt32(pow(2, 32) - 1) as UInt32_column, " + "toInt64('-9223372036854775808') as Int64_column, " # -2^63 + "toUInt64('18446744073709551615') as UInt64_column, " # 2^64 - 1 + "'тест' as String_column, " + "toFixedString('тест', 8) as FixedString_column, " + "toFloat32(1.5) as Float32_column, " + "toFloat64(1.5) as Float64_column, " + "toFloat32(NaN) as Float32_NaN_column, " + "-Inf as Float64_Inf_column, " + "toDate('2019-12-08') as Date_column, " + "toDate('1970-01-01') as Date_min_column, " + "toDate('1970-01-02') as Date_after_min_column, " + "toDateTime('2019-12-08 08:24:03') as DateTime_column" + ) + + result = cursor.fetchall()[0] + expected = [ + ('Int8_column', -2 ** 7), + ('UInt8_column', 2 ** 8 - 1), + ('Int16_column', -2 ** 15), + ('UInt16_column', 2 ** 16 - 1), + ('Int32_column', -2 ** 31), + ('UInt32_column', 2 ** 32 - 1), + ('Int64_column', -2 ** 63), + ('UInt64_column', 2 ** 64 - 1), + ('String_column', 'тест'), + ('FixedString_column', 'тест'), + ('Float32_column', 1.5), + ('Float64_column', 1.5), + ('Float32_NaN_column', float('nan')), + ('Float64_Inf_column', float('-inf')), + ('Date_column', datetime.date(2019, 12, 8)), + ('Date_min_column', '0000-00-00'), + ('Date_after_min_column', datetime.date(1970, 1, 2)), + ('DateTime_column', datetime.datetime(2019, 12, 8, 8, 24, 3)), + ] + + for key, value in expected: + if isinstance(value, float) and math.isnan(value): + assert math.isnan(result[key]) + else: + assert result[key] == value diff --git a/dbms/tests/integration/test_odbc_interaction/test.py b/dbms/tests/integration/test_odbc_interaction/test.py index d8e9cc3fb1a..f4af04b5fbc 100644 --- a/dbms/tests/integration/test_odbc_interaction/test.py +++ b/dbms/tests/integration/test_odbc_interaction/test.py @@ -91,7 +91,8 @@ def test_mysql_simple_select_works(started_cluster): with conn.cursor() as cursor: cursor.execute("INSERT INTO clickhouse.{} VALUES(50, 'null-guy', 127, 255, NULL), (100, 'non-null-guy', 127, 255, 511);".format(table_name)) conn.commit() - assert node1.query("SELECT column_x FROM odbc('DSN={}', '{}')".format(mysql_setup["DSN"], table_name)) == '\\N\n511\n' + assert node1.query("SELECT column_x FROM odbc('DSN={}', '{}') SETTINGS external_table_functions_use_nulls=1".format(mysql_setup["DSN"], table_name)) == '\\N\n511\n' + assert node1.query("SELECT column_x FROM odbc('DSN={}', '{}') SETTINGS external_table_functions_use_nulls=0".format(mysql_setup["DSN"], table_name)) == '0\n511\n' node1.query(''' CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32, column_x Nullable(UInt32)) ENGINE = MySQL('mysql1:3306', 'clickhouse', '{}', 'root', 'clickhouse'); diff --git a/dbms/tests/integration/test_old_versions/__init__.py b/dbms/tests/integration/test_old_versions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_old_versions/configs/config.d/test_cluster.xml b/dbms/tests/integration/test_old_versions/configs/config.d/test_cluster.xml new file mode 100644 index 00000000000..56d53d63d78 --- /dev/null +++ b/dbms/tests/integration/test_old_versions/configs/config.d/test_cluster.xml @@ -0,0 +1,13 @@ + + + + + 1 + + node_new + 9000 + + + + + diff --git a/dbms/tests/integration/test_old_versions/test.py b/dbms/tests/integration/test_old_versions/test.py new file mode 100644 index 00000000000..d77b4af016a --- /dev/null +++ b/dbms/tests/integration/test_old_versions/test.py @@ -0,0 +1,73 @@ +import time +import os +import pytest + +from helpers.cluster import ClickHouseCluster +from multiprocessing.dummy import Pool +from helpers.client import QueryRuntimeException, QueryTimeoutExceedException +from helpers.test_tools import assert_eq_with_retry + + +cluster = ClickHouseCluster(__file__) +node18_14 = cluster.add_instance('node18_14', image='yandex/clickhouse-server:18.14.19', with_installed_binary=True, config_dir="configs") +node19_1 = cluster.add_instance('node19_1', image='yandex/clickhouse-server:19.1.16', with_installed_binary=True, config_dir="configs") +node19_4 = cluster.add_instance('node19_4', image='yandex/clickhouse-server:19.4.5.35', with_installed_binary=True, config_dir="configs") +node19_8 = cluster.add_instance('node19_8', image='yandex/clickhouse-server:19.8.3.8', with_installed_binary=True, config_dir="configs") +node19_11 = cluster.add_instance('node19_11', image='yandex/clickhouse-server:19.11.13.74', with_installed_binary=True, config_dir="configs") +node19_13 = cluster.add_instance('node19_13', image='yandex/clickhouse-server:19.13.7.57', with_installed_binary=True, config_dir="configs") +node19_16 = cluster.add_instance('node19_16', image='yandex/clickhouse-server:19.16.2.2', with_installed_binary=True, config_dir="configs") +old_nodes = [node18_14, node19_1, node19_4, node19_8, node19_11, node19_13, node19_16] +new_node = cluster.add_instance('node_new') + + +def query_from_one_node_to_another(client_node, server_node, query): + client_node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host {} --query {!r}".format(server_node.name, query)]) + + +@pytest.fixture(scope="module") +def setup_nodes(): + try: + cluster.start() + + for n in old_nodes + [new_node]: + n.query('''CREATE TABLE test_table (id UInt32, value UInt64) ENGINE = MergeTree() ORDER BY tuple()''') + + for n in old_nodes: + n.query('''CREATE TABLE dist_table AS test_table ENGINE = Distributed('test_cluster', 'default', 'test_table')''') + + yield cluster + finally: + cluster.shutdown() + + +def test_client_is_older_than_server(setup_nodes): + server = new_node + for i, client in enumerate(old_nodes): + query_from_one_node_to_another(client, server, "INSERT INTO test_table VALUES (1, {})".format(i)) + + for client in old_nodes: + query_from_one_node_to_another(client, server, "SELECT COUNT() FROM test_table") + + assert server.query("SELECT COUNT() FROM test_table WHERE id=1") == str(len(old_nodes)) + "\n" + + +def test_server_is_older_than_client(setup_nodes): + client = new_node + for i, server in enumerate(old_nodes): + query_from_one_node_to_another(client, server, "INSERT INTO test_table VALUES (2, {})".format(i)) + + for server in old_nodes: + query_from_one_node_to_another(client, server, "SELECT COUNT() FROM test_table") + + for server in old_nodes: + assert server.query("SELECT COUNT() FROM test_table WHERE id=2") == "1\n" + + +def test_distributed_query_initiator_is_older_than_shard(setup_nodes): + distributed_query_initiator_old_nodes = [node18_14, node19_13, node19_16] + shard = new_node + for i, initiator in enumerate(distributed_query_initiator_old_nodes): + initiator.query("INSERT INTO dist_table VALUES (3, {})".format(i)) + + assert_eq_with_retry(shard, "SELECT COUNT() FROM test_table WHERE id=3", str(len(distributed_query_initiator_old_nodes))) + assert_eq_with_retry(initiator, "SELECT COUNT() FROM dist_table WHERE id=3", str(len(distributed_query_initiator_old_nodes))) diff --git a/dbms/tests/integration/test_old_versions_client/test.py b/dbms/tests/integration/test_old_versions_client/test.py deleted file mode 100644 index 59a4aaca31c..00000000000 --- a/dbms/tests/integration/test_old_versions_client/test.py +++ /dev/null @@ -1,51 +0,0 @@ -import time -import pytest - -from helpers.cluster import ClickHouseCluster -from multiprocessing.dummy import Pool -from helpers.client import QueryRuntimeException, QueryTimeoutExceedException - -from helpers.test_tools import assert_eq_with_retry -cluster = ClickHouseCluster(__file__) -node18_14 = cluster.add_instance('node18_14', image='yandex/clickhouse-server:18.14.19', with_installed_binary=True) -node19_1 = cluster.add_instance('node19_1', image='yandex/clickhouse-server:19.1.16', with_installed_binary=True) -node19_4 = cluster.add_instance('node19_4', image='yandex/clickhouse-server:19.4.5.35', with_installed_binary=True) -node19_6 = cluster.add_instance('node19_6', image='yandex/clickhouse-server:19.6.3.18', with_installed_binary=True) -node19_8 = cluster.add_instance('node19_8', image='yandex/clickhouse-server:19.8.3.8', with_installed_binary=True) -node_new = cluster.add_instance('node_new') - -@pytest.fixture(scope="module") -def setup_nodes(): - try: - cluster.start() - for n in (node18_14, node19_1, node19_4, node19_6, node19_8, node_new): - n.query('''CREATE TABLE test_table (id UInt32, value UInt64) ENGINE = MergeTree() ORDER BY tuple()''') - - yield cluster - finally: - cluster.shutdown() - - -def query_from_one_node_to_another(client_node, server_node, query): - client_node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host {} --query '{}'".format(server_node.name, query)]) - -def test_client_from_different_versions(setup_nodes): - old_nodes = (node18_14, node19_1, node19_4, node19_6, node19_8) - # from new to old - for n in old_nodes: - query_from_one_node_to_another(node_new, n, "INSERT INTO test_table VALUES (1, 1)") - - for n in old_nodes: - query_from_one_node_to_another(node_new, n, "SELECT COUNT() FROM test_table") - - for n in old_nodes: - assert n.query("SELECT COUNT() FROM test_table") == "1\n" - - # from old to new - for i, n in enumerate(old_nodes): - query_from_one_node_to_another(n, node_new, "INSERT INTO test_table VALUES ({i}, {i})".format(i=i)) - - for n in old_nodes: - query_from_one_node_to_another(n, node_new, "SELECT COUNT() FROM test_table") - - assert node_new.query("SELECT COUNT() FROM test_table") == str(len(old_nodes)) + "\n" diff --git a/dbms/tests/integration/test_part_log_table/__init__.py b/dbms/tests/integration/test_part_log_table/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_part_log_table/configs/config_with_non_standard_part_log.xml b/dbms/tests/integration/test_part_log_table/configs/config_with_non_standard_part_log.xml new file mode 100644 index 00000000000..2a8655de830 --- /dev/null +++ b/dbms/tests/integration/test_part_log_table/configs/config_with_non_standard_part_log.xml @@ -0,0 +1,6 @@ + + + database_name +
table_name
+ + diff --git a/dbms/tests/integration/test_part_log_table/configs/config_with_standard_part_log.xml b/dbms/tests/integration/test_part_log_table/configs/config_with_standard_part_log.xml new file mode 100644 index 00000000000..1e640a9e0b0 --- /dev/null +++ b/dbms/tests/integration/test_part_log_table/configs/config_with_standard_part_log.xml @@ -0,0 +1,4 @@ + + + + diff --git a/dbms/tests/integration/test_part_log_table/test.py b/dbms/tests/integration/test_part_log_table/test.py new file mode 100644 index 00000000000..5d4c005714f --- /dev/null +++ b/dbms/tests/integration/test_part_log_table/test.py @@ -0,0 +1,43 @@ +import time +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance("node1") +node2 = cluster.add_instance("node2", main_configs=["configs/config_with_standard_part_log.xml"]) +node3 = cluster.add_instance("node3", main_configs=["configs/config_with_non_standard_part_log.xml"]) + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + +def test_config_without_part_log(start_cluster): + assert "Table system.part_log doesn't exist" in node1.query_and_get_error("SELECT * FROM system.part_log") + node1.query("CREATE TABLE test_table(word String, value UInt64) ENGINE=MergeTree() ORDER BY value") + assert "Table system.part_log doesn't exist" in node1.query_and_get_error("SELECT * FROM system.part_log") + node1.query("INSERT INTO test_table VALUES ('name', 1)") + time.sleep(10) + assert "Table system.part_log doesn't exist" in node1.query_and_get_error("SELECT * FROM system.part_log") + +def test_config_with_standard_part_log(start_cluster): + assert "Table system.part_log doesn't exist" in node2.query_and_get_error("SELECT * FROM system.part_log") + node2.query("CREATE TABLE test_table(word String, value UInt64) ENGINE=MergeTree() Order by value") + assert "Table system.part_log doesn't exist" in node2.query_and_get_error("SELECT * FROM system.part_log") + node2.query("INSERT INTO test_table VALUES ('name', 1)") + time.sleep(10) + assert node2.query("SELECT * FROM system.part_log") != "" + +def test_config_with_non_standard_part_log(start_cluster): + node3.query("CREATE DATABASE database_name") + assert "table_name" not in node3.query("SHOW TABLES FROM database_name") + node3.query("CREATE TABLE test_table(word String, value UInt64) ENGINE=MergeTree() ORDER BY value") + assert "table_name" not in node3.query("SHOW TABLES FROM database_name") + node3.query("INSERT INTO test_table VALUES ('name', 1)") + time.sleep(10) + assert "table_name" in node3.query("SHOW TABLES FROM database_name") + diff --git a/dbms/tests/integration/test_partition/test.py b/dbms/tests/integration/test_partition/test.py index 3365343b6fb..659b9a89069 100644 --- a/dbms/tests/integration/test_partition/test.py +++ b/dbms/tests/integration/test_partition/test.py @@ -241,4 +241,3 @@ def test_drop_detached_parts(drop_detached_parts_table): q("ALTER TABLE test.drop_detached DROP DETACHED PARTITION 1", settings=s) detached = q("SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='test' ORDER BY name") assert TSV(detached) == TSV('0_3_3_0\nattaching_0_6_6_0\ndeleting_0_7_7_0') - diff --git a/dbms/tests/integration/test_prometheus_endpoint/__init__.py b/dbms/tests/integration/test_prometheus_endpoint/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_prometheus_endpoint/configs/prom_conf.xml b/dbms/tests/integration/test_prometheus_endpoint/configs/prom_conf.xml new file mode 100644 index 00000000000..6e31324eac2 --- /dev/null +++ b/dbms/tests/integration/test_prometheus_endpoint/configs/prom_conf.xml @@ -0,0 +1,13 @@ + + 8123 + 9000 + + + /metrics + 8001 + + true + true + true + + diff --git a/dbms/tests/integration/test_prometheus_endpoint/test.py b/dbms/tests/integration/test_prometheus_endpoint/test.py new file mode 100644 index 00000000000..10f49c23072 --- /dev/null +++ b/dbms/tests/integration/test_prometheus_endpoint/test.py @@ -0,0 +1,67 @@ +from __future__ import print_function +import pytest + +import re +import requests + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance('node', main_configs=['configs/prom_conf.xml']) + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def parse_response_line(line): + allowed_prefixes = [ + "ClickHouse", + "# HELP", + "# TYPE", + ] + assert any(line.startswith(prefix) for prefix in allowed_prefixes), msg + + if line.startswith("#"): + return {} + match = re.match('^([a-zA-Z_:][a-zA-Z0-9_:]+)(\{.*\})? (\d)', line) + assert match, line + name, _, val = match.groups() + return {name: int(val)} + + +def get_and_check_metrics(): + response = requests.get("http://{host}:{port}/metrics".format( + host=node.ip_address, port=8001), allow_redirects=False) + + if response.status_code != 200: + response.raise_for_status() + + assert response.headers['content-type'].startswith('text/plain') + + results = {} + for resp_line in response.text.split('\n'): + resp_line = resp_line.rstrip() + if not resp_line: + continue + res = parse_response_line(resp_line) + results.update(res) + return results + + +def test_prometheus_endpoint(start_cluster): + + metrics_dict = get_and_check_metrics() + assert metrics_dict['ClickHouseProfileEventsQuery'] >= 0 + prev_query_count = metrics_dict['ClickHouseProfileEventsQuery'] + + resp = node.query("SELECT 1") + resp = node.query("SELECT 2") + resp = node.query("SELECT 3") + + metrics_dict = get_and_check_metrics() + assert metrics_dict['ClickHouseProfileEventsQuery'] >= prev_query_count + 3 diff --git a/dbms/tests/integration/test_quota/__init__.py b/dbms/tests/integration/test_quota/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_quota/configs/users.d/quota.xml b/dbms/tests/integration/test_quota/configs/users.d/quota.xml new file mode 120000 index 00000000000..9b12dec9c53 --- /dev/null +++ b/dbms/tests/integration/test_quota/configs/users.d/quota.xml @@ -0,0 +1 @@ +../../normal_limits.xml \ No newline at end of file diff --git a/dbms/tests/integration/test_quota/configs/users.xml b/dbms/tests/integration/test_quota/configs/users.xml new file mode 100644 index 00000000000..15a5364449b --- /dev/null +++ b/dbms/tests/integration/test_quota/configs/users.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + ::/0 + + default + myQuota + true + + + diff --git a/dbms/tests/integration/test_quota/no_quotas.xml b/dbms/tests/integration/test_quota/no_quotas.xml new file mode 100644 index 00000000000..9aba4ac0914 --- /dev/null +++ b/dbms/tests/integration/test_quota/no_quotas.xml @@ -0,0 +1,3 @@ + + + diff --git a/dbms/tests/integration/test_quota/normal_limits.xml b/dbms/tests/integration/test_quota/normal_limits.xml new file mode 100644 index 00000000000..b7c3a67b5cc --- /dev/null +++ b/dbms/tests/integration/test_quota/normal_limits.xml @@ -0,0 +1,17 @@ + + + + + + + 31556952 + + + 1000 + 0 + 1000 + 0 + + + + diff --git a/dbms/tests/integration/test_quota/simpliest.xml b/dbms/tests/integration/test_quota/simpliest.xml new file mode 100644 index 00000000000..6d51d68d8d9 --- /dev/null +++ b/dbms/tests/integration/test_quota/simpliest.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/dbms/tests/integration/test_quota/test.py b/dbms/tests/integration/test_quota/test.py new file mode 100644 index 00000000000..e7caaf5cd06 --- /dev/null +++ b/dbms/tests/integration/test_quota/test.py @@ -0,0 +1,251 @@ +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry +import os +import re +import time + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance('instance', + config_dir="configs") + +query_from_system_quotas = "SELECT * FROM system.quotas ORDER BY name"; + +query_from_system_quota_usage = "SELECT id, key, duration, "\ + "queries, errors, result_rows, result_bytes, read_rows, read_bytes "\ + "FROM system.quota_usage ORDER BY id, key, duration"; + +def system_quotas(): + return instance.query(query_from_system_quotas).rstrip('\n') + +def system_quota_usage(): + return instance.query(query_from_system_quota_usage).rstrip('\n') + + +def copy_quota_xml(local_file_name, reload_immediately = True): + script_dir = os.path.dirname(os.path.realpath(__file__)) + instance.copy_file_to_container(os.path.join(script_dir, local_file_name), '/etc/clickhouse-server/users.d/quota.xml') + if reload_immediately: + instance.query("SYSTEM RELOAD CONFIG") + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + + instance.query("CREATE TABLE test_table(x UInt32) ENGINE = MergeTree ORDER BY tuple()") + instance.query("INSERT INTO test_table SELECT number FROM numbers(50)") + + yield cluster + + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def reset_quotas_and_usage_info(): + try: + yield + finally: + instance.query("DROP QUOTA IF EXISTS qA, qB") + copy_quota_xml('simpliest.xml') # To reset usage info. + copy_quota_xml('normal_limits.xml') + + +def test_quota_from_users_xml(): + assert instance.query("SELECT currentQuota()") == "myQuota\n" + assert instance.query("SELECT currentQuotaID()") == "e651da9c-a748-8703-061a-7e5e5096dae7\n" + assert instance.query("SELECT currentQuotaKey()") == "default\n" + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + + instance.query("SELECT * from test_table") + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t0\t50\t200\t50\t200" + + instance.query("SELECT COUNT() from test_table") + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t2\t0\t51\t208\t50\t200" + + +def test_simpliest_quota(): + # Simpliest quota doesn't even track usage. + copy_quota_xml('simpliest.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N" + + instance.query("SELECT * from test_table") + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N" + + +def test_tracking_quota(): + # Now we're tracking usage. + copy_quota_xml('tracking.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[0]\t[0]\t[0]\t[0]\t[0]\t[0]\t[0]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + + instance.query("SELECT * from test_table") + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t0\t50\t200\t50\t200" + + instance.query("SELECT COUNT() from test_table") + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t2\t0\t51\t208\t50\t200" + + +def test_exceed_quota(): + # Change quota, now the limits are tiny so we will exceed the quota. + copy_quota_xml('tiny_limits.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1]\t[1]\t[1]\t[0]\t[1]\t[0]\t[0]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + + assert re.search("Quota.*has\ been\ exceeded", instance.query_and_get_error("SELECT * from test_table")) + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t1\t0\t0\t50\t0" + + # Change quota, now the limits are enough to execute queries. + copy_quota_xml('normal_limits.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t1\t0\t0\t50\t0" + + instance.query("SELECT * from test_table") + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t2\t1\t50\t200\t100\t200" + + +def test_add_remove_interval(): + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + + # Add interval. + copy_quota_xml('two_intervals.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952,63113904]\t[0,1]\t[1000,0]\t[0,0]\t[0,0]\t[0,30000]\t[1000,0]\t[0,20000]\t[0,120]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0\n"\ + "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t63113904\t0\t0\t0\t0\t0\t0" + + instance.query("SELECT * from test_table") + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t0\t50\t200\t50\t200\n"\ + "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t63113904\t1\t0\t50\t200\t50\t200" + + # Remove interval. + copy_quota_xml('normal_limits.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t0\t50\t200\t50\t200" + + instance.query("SELECT * from test_table") + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t2\t0\t100\t400\t100\t400" + + # Remove all intervals. + copy_quota_xml('simpliest.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N" + + instance.query("SELECT * from test_table") + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N" + + # Add one interval back. + copy_quota_xml('normal_limits.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + + +def test_add_remove_quota(): + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + + # Add quota. + copy_quota_xml('two_quotas.xml') + assert system_quotas() ==\ + "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]\n"\ + "myQuota2\t4590510c-4d13-bf21-ec8a-c2187b092e73\tusers.xml\tclient key or user name\t[]\t0\t[]\t[3600,2629746]\t[1,0]\t[0,0]\t[0,0]\t[4000,0]\t[400000,0]\t[4000,0]\t[400000,0]\t[60,1800]" + + # Drop quota. + copy_quota_xml('normal_limits.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + + # Drop all quotas. + copy_quota_xml('no_quotas.xml') + assert system_quotas() == "" + assert system_quota_usage() == "" + + # Add one quota back. + copy_quota_xml('normal_limits.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + + +def test_reload_users_xml_by_timer(): + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + + time.sleep(1) # The modification time of the 'quota.xml' file should be different, + # because config files are reload by timer only when the modification time is changed. + copy_quota_xml('tiny_limits.xml', reload_immediately=False) + assert_eq_with_retry(instance, query_from_system_quotas, "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1]\t[1]\t[1]\t[0]\t[1]\t[0]\t[0]") + + +def test_dcl_introspection(): + assert instance.query("SHOW QUOTAS") == "myQuota\n" + assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES = 1000, MAX READ ROWS = 1000 TO default\n" + expected_usage = "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=0/1000 errors=0 result_rows=0 result_bytes=0 read_rows=0/1000 read_bytes=0 execution_time=0" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE CURRENT")) + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE ALL")) + + instance.query("SELECT * from test_table") + expected_usage = "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=1/1000 errors=0 result_rows=50 result_bytes=200 read_rows=50/1000 read_bytes=200 execution_time=.*" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + + # Add interval. + copy_quota_xml('two_intervals.xml') + assert instance.query("SHOW QUOTAS") == "myQuota\n" + assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES = 1000, MAX READ ROWS = 1000, FOR RANDOMIZED INTERVAL 2 YEAR MAX RESULT BYTES = 30000, MAX READ BYTES = 20000, MAX EXECUTION TIME = 120 TO default\n" + expected_usage = "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=1/1000 errors=0 result_rows=50 result_bytes=200 read_rows=50/1000 read_bytes=200 execution_time=.*\n"\ + "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=0 errors=0 result_rows=0 result_bytes=0/30000 read_rows=0 read_bytes=0/20000 execution_time=0/120" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + + # Drop interval, add quota. + copy_quota_xml('two_quotas.xml') + assert instance.query("SHOW QUOTAS") == "myQuota\nmyQuota2\n" + assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES = 1000, MAX READ ROWS = 1000 TO default\n" + assert instance.query("SHOW CREATE QUOTA myQuota2") == "CREATE QUOTA myQuota2 KEYED BY \\'client key or user name\\' FOR RANDOMIZED INTERVAL 1 HOUR MAX RESULT ROWS = 4000, MAX RESULT BYTES = 400000, MAX READ ROWS = 4000, MAX READ BYTES = 400000, MAX EXECUTION TIME = 60, FOR INTERVAL 1 MONTH MAX EXECUTION TIME = 1800\n" + expected_usage = "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=1/1000 errors=0 result_rows=50 result_bytes=200 read_rows=50/1000 read_bytes=200 execution_time=.*" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + + +def test_dcl_management(): + copy_quota_xml('no_quotas.xml') + assert instance.query("SHOW QUOTAS") == "" + assert instance.query("SHOW QUOTA USAGE") == "" + + instance.query("CREATE QUOTA qA FOR INTERVAL 15 MONTH SET MAX QUERIES = 123 TO CURRENT_USER") + assert instance.query("SHOW QUOTAS") == "qA\n" + assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR INTERVAL 5 QUARTER MAX QUERIES = 123 TO default\n" + expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=0/123 errors=0 result_rows=0 result_bytes=0 read_rows=0 read_bytes=0 execution_time=.*" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + + instance.query("SELECT * from test_table") + expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=1/123 errors=0 result_rows=50 result_bytes=200 read_rows=50 read_bytes=200 execution_time=.*" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + + instance.query("ALTER QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES = 321, MAX ERRORS = 10, FOR INTERVAL 0.5 HOUR MAX EXECUTION TIME = 0.5") + assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR INTERVAL 30 MINUTE MAX EXECUTION TIME = 0.5, FOR INTERVAL 5 QUARTER MAX QUERIES = 321, MAX ERRORS = 10 TO default\n" + expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=0 errors=0 result_rows=0 result_bytes=0 read_rows=0 read_bytes=0 execution_time=.*/0.5\n"\ + "qA key=\\\\'\\\\' interval=\[.*\] queries=1/321 errors=0/10 result_rows=50 result_bytes=200 read_rows=50 read_bytes=200 execution_time=.*" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + + instance.query("ALTER QUOTA qA FOR INTERVAL 15 MONTH UNSET TRACKING, FOR RANDOMIZED INTERVAL 16 MONTH SET TRACKING, FOR INTERVAL 1800 SECOND UNSET TRACKING") + assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR RANDOMIZED INTERVAL 16 MONTH TRACKING TO default\n" + expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=0 errors=0 result_rows=0 result_bytes=0 read_rows=0 read_bytes=0 execution_time=.*" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + + instance.query("SELECT * from test_table") + expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=1 errors=0 result_rows=50 result_bytes=200 read_rows=50 read_bytes=200 execution_time=.*" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + + instance.query("ALTER QUOTA qA RENAME TO qB") + assert instance.query("SHOW CREATE QUOTA qB") == "CREATE QUOTA qB KEYED BY \\'none\\' FOR RANDOMIZED INTERVAL 16 MONTH TRACKING TO default\n" + expected_usage = "qB key=\\\\'\\\\' interval=\[.*\] queries=1 errors=0 result_rows=50 result_bytes=200 read_rows=50 read_bytes=200 execution_time=.*" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + + instance.query("DROP QUOTA qB") + assert instance.query("SHOW QUOTAS") == "" + assert instance.query("SHOW QUOTA USAGE") == "" + + +def test_users_xml_is_readonly(): + assert re.search("storage is readonly", instance.query_and_get_error("DROP QUOTA myQuota")) diff --git a/dbms/tests/integration/test_quota/tiny_limits.xml b/dbms/tests/integration/test_quota/tiny_limits.xml new file mode 100644 index 00000000000..3ab8858738a --- /dev/null +++ b/dbms/tests/integration/test_quota/tiny_limits.xml @@ -0,0 +1,17 @@ + + + + + + + 31556952 + + + 1 + 1 + 1 + 1 + + + + diff --git a/dbms/tests/integration/test_quota/tracking.xml b/dbms/tests/integration/test_quota/tracking.xml new file mode 100644 index 00000000000..47e12bf8005 --- /dev/null +++ b/dbms/tests/integration/test_quota/tracking.xml @@ -0,0 +1,17 @@ + + + + + + + 31556952 + + + 0 + 0 + 0 + 0 + + + + diff --git a/dbms/tests/integration/test_quota/two_intervals.xml b/dbms/tests/integration/test_quota/two_intervals.xml new file mode 100644 index 00000000000..d0de605b895 --- /dev/null +++ b/dbms/tests/integration/test_quota/two_intervals.xml @@ -0,0 +1,20 @@ + + + + + + 31556952 + 1000 + 1000 + + + + true + 63113904 + 20000 + 30000 + 120 + + + + diff --git a/dbms/tests/integration/test_quota/two_quotas.xml b/dbms/tests/integration/test_quota/two_quotas.xml new file mode 100644 index 00000000000..c08cc82aca7 --- /dev/null +++ b/dbms/tests/integration/test_quota/two_quotas.xml @@ -0,0 +1,29 @@ + + + + + + 31556952 + 1000 + 1000 + + + + + + + true + 3600 + 4000 + 4000 + 400000 + 400000 + 60 + + + 2629746 + 1800 + + + + diff --git a/dbms/tests/integration/test_read_temporary_tables_on_failure/__init__.py b/dbms/tests/integration/test_read_temporary_tables_on_failure/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_read_temporary_tables_on_failure/test.py b/dbms/tests/integration/test_read_temporary_tables_on_failure/test.py new file mode 100644 index 00000000000..ad1a41b8979 --- /dev/null +++ b/dbms/tests/integration/test_read_temporary_tables_on_failure/test.py @@ -0,0 +1,26 @@ +import pytest +import time + +from helpers.cluster import ClickHouseCluster +from helpers.client import QueryTimeoutExceedException, QueryRuntimeException + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance('node') + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + + yield cluster + finally: + cluster.shutdown() + +def test_different_versions(start_cluster): + with pytest.raises(QueryTimeoutExceedException): + node.query("SELECT sleep(3)", timeout=1) + with pytest.raises(QueryRuntimeException): + node.query("SELECT 1", settings={'max_concurrent_queries_for_user': 1}) + assert node.contains_in_log('Too many simultaneous queries for user') + assert not node.contains_in_log('Unknown packet') diff --git a/dbms/tests/integration/test_relative_filepath/__init__.py b/dbms/tests/integration/test_relative_filepath/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_relative_filepath/configs/config.xml b/dbms/tests/integration/test_relative_filepath/configs/config.xml new file mode 100644 index 00000000000..dbc339c4741 --- /dev/null +++ b/dbms/tests/integration/test_relative_filepath/configs/config.xml @@ -0,0 +1,4 @@ + + + user_files + diff --git a/dbms/tests/integration/test_relative_filepath/test.py b/dbms/tests/integration/test_relative_filepath/test.py new file mode 100644 index 00000000000..a8e2341a3cd --- /dev/null +++ b/dbms/tests/integration/test_relative_filepath/test.py @@ -0,0 +1,36 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance('node', main_configs=['configs/config.xml']) +path_to_userfiles_from_defaut_config = "user_files" + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + +def test_filepath(start_cluster): + # 2 rows data + some_data = "Test\t111.222\nData\t333.444" + + node.exec_in_container(['bash', '-c', 'mkdir -p {}'.format( + path_to_userfiles_from_defaut_config + )], privileged=True, user='root') + + node.exec_in_container(['bash', '-c', 'echo "{}" > {}'.format( + some_data, + path_to_userfiles_from_defaut_config + "/relative_user_file_test" + )], privileged=True, user='root') + + test_requests = [("relative_user_file_test", "2"), + ("../" + path_to_userfiles_from_defaut_config + "/relative_user_file_test", "2")] + + for pattern, value in test_requests: + assert node.query(''' + select count() from file('{}', 'TSV', 'text String, number Float64') + '''.format(pattern)) == '{}\n'.format(value) diff --git a/dbms/tests/integration/test_reload_max_table_size_to_drop/__init__.py b/dbms/tests/integration/test_reload_max_table_size_to_drop/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_reload_max_table_size_to_drop/configs/config.xml b/dbms/tests/integration/test_reload_max_table_size_to_drop/configs/config.xml new file mode 100644 index 00000000000..b5e5495c096 --- /dev/null +++ b/dbms/tests/integration/test_reload_max_table_size_to_drop/configs/config.xml @@ -0,0 +1,31 @@ + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + 9000 + 127.0.0.1 + + + + true + none + + AcceptCertificateHandler + + + + + 500 + 5368709120 + ./clickhouse/ + users.xml + + 1 + 1 + diff --git a/dbms/tests/integration/test_reload_max_table_size_to_drop/configs/users.xml b/dbms/tests/integration/test_reload_max_table_size_to_drop/configs/users.xml new file mode 100644 index 00000000000..6061af8e33d --- /dev/null +++ b/dbms/tests/integration/test_reload_max_table_size_to_drop/configs/users.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + ::/0 + + default + default + + + + + + + + diff --git a/dbms/tests/integration/test_reload_max_table_size_to_drop/test.py b/dbms/tests/integration/test_reload_max_table_size_to_drop/test.py new file mode 100644 index 00000000000..3959b383fc5 --- /dev/null +++ b/dbms/tests/integration/test_reload_max_table_size_to_drop/test.py @@ -0,0 +1,49 @@ +import time +import pytest +import os + +from helpers.cluster import ClickHouseCluster + + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance('node', config_dir="configs") + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +CONFIG_PATH = os.path.join(SCRIPT_DIR, './_instances/node/configs/config.xml') + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + node.query("CREATE TABLE test(date Date, id UInt32) ENGINE = MergeTree() PARTITION BY date ORDER BY id") + yield cluster + finally: + cluster.shutdown() + + +def test_reload_max_table_size_to_drop(start_cluster): + node.query("INSERT INTO test VALUES (now(), 0)") + + time.sleep(5) # wait for data part commit + + drop = node.get_query_request("DROP TABLE test") + out, err = drop.get_answer_and_error() + assert out == "" + assert err != "" + + config = open(CONFIG_PATH, 'r') + config_lines = config.readlines() + config.close() + config_lines = map(lambda line: line.replace("1", "1000000"), + config_lines) + config = open(CONFIG_PATH, 'w') + config.writelines(config_lines) + config.close() + + node.query("SYSTEM RELOAD CONFIG") + + drop = node.get_query_request("DROP TABLE test") + out, err = drop.get_answer_and_error() + assert out == "" + assert err == "" diff --git a/dbms/tests/integration/test_replicating_constants/__init__.py b/dbms/tests/integration/test_replicating_constants/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_replicating_constants/test.py b/dbms/tests/integration/test_replicating_constants/test.py new file mode 100644 index 00000000000..f340817b584 --- /dev/null +++ b/dbms/tests/integration/test_replicating_constants/test.py @@ -0,0 +1,21 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance('node1', with_zookeeper=True) +node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server:19.1.14', with_installed_binary=True) + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + + yield cluster + finally: + cluster.shutdown() + +def test_different_versions(start_cluster): + + assert node1.query("SELECT uniqExact(x) FROM (SELECT version() as x from remote('node{1,2}', system.one))") == "2\n" diff --git a/dbms/tests/integration/test_server_initialization/clickhouse_path/data/default/should_be_restored/data.CSV b/dbms/tests/integration/test_server_initialization/clickhouse_path/data/default/should_be_restored/data.CSV new file mode 100644 index 00000000000..01e79c32a8c --- /dev/null +++ b/dbms/tests/integration/test_server_initialization/clickhouse_path/data/default/should_be_restored/data.CSV @@ -0,0 +1,3 @@ +1 +2 +3 diff --git a/dbms/tests/integration/test_server_initialization/clickhouse_path/metadata/default/should_be_dropped.sql.tmp_drop b/dbms/tests/integration/test_server_initialization/clickhouse_path/metadata/default/should_be_dropped.sql.tmp_drop new file mode 100644 index 00000000000..d6e1364dc3e --- /dev/null +++ b/dbms/tests/integration/test_server_initialization/clickhouse_path/metadata/default/should_be_dropped.sql.tmp_drop @@ -0,0 +1 @@ +attach table should_be_dropped (n UInt8) engine = File(CSV) diff --git a/dbms/tests/integration/test_server_initialization/clickhouse_path/metadata/default/should_be_restored.sql.tmp_drop b/dbms/tests/integration/test_server_initialization/clickhouse_path/metadata/default/should_be_restored.sql.tmp_drop new file mode 100644 index 00000000000..0452819cac0 --- /dev/null +++ b/dbms/tests/integration/test_server_initialization/clickhouse_path/metadata/default/should_be_restored.sql.tmp_drop @@ -0,0 +1 @@ +attach table should_be_restored (n UInt8) engine = File(CSV) diff --git a/dbms/tests/integration/test_server_initialization/test.py b/dbms/tests/integration/test_server_initialization/test.py index 212037b0c93..64ed3181118 100644 --- a/dbms/tests/integration/test_server_initialization/test.py +++ b/dbms/tests/integration/test_server_initialization/test.py @@ -26,3 +26,11 @@ def test_sophisticated_default(started_cluster): instance.query("INSERT INTO sophisticated_default (c) VALUES (0)") assert instance.query("SELECT a, b, c FROM sophisticated_default") == "3\t9\t0\n" + +def test_partially_dropped_tables(started_cluster): + instance = started_cluster.instances['dummy'] + assert instance.exec_in_container(['bash', '-c', 'cd / && find -name *.sql* | sort'], privileged=True, user='root') \ + == "./var/lib/clickhouse/metadata/default/should_be_restored.sql\n" \ + "./var/lib/clickhouse/metadata/default/sophisticated_default.sql\n" + assert instance.query("SELECT n FROM should_be_restored") == "1\n2\n3\n" + assert instance.query("SELECT count() FROM system.tables WHERE name='should_be_dropped'") == "0\n" diff --git a/dbms/tests/integration/test_settings_constraints/configs/users.xml b/dbms/tests/integration/test_settings_constraints/configs/users.xml index 8bbe18d7c6c..b29b66d827f 100644 --- a/dbms/tests/integration/test_settings_constraints/configs/users.xml +++ b/dbms/tests/integration/test_settings_constraints/configs/users.xml @@ -14,6 +14,14 @@ + + + 1 + + + + 0 + @@ -25,6 +33,22 @@ default default + + + + ::/0 + + readonly_profile + default + + + + + ::/0 + + no_dll_profile + default + diff --git a/dbms/tests/integration/test_settings_constraints/test.py b/dbms/tests/integration/test_settings_constraints/test.py index be296b51827..625aab82e12 100644 --- a/dbms/tests/integration/test_settings_constraints/test.py +++ b/dbms/tests/integration/test_settings_constraints/test.py @@ -18,68 +18,123 @@ def started_cluster(): cluster.shutdown() +def test_system_settings(started_cluster): + assert instance.query("SELECT name, value, min, max, readonly from system.settings WHERE name = 'force_index_by_date'") ==\ + "force_index_by_date\t0\t\\N\t\\N\t1\n" + + assert instance.query("SELECT name, value, min, max, readonly from system.settings WHERE name = 'max_memory_usage'") ==\ + "max_memory_usage\t10000000000\t5000000000\t20000000000\t0\n" + + assert instance.query("SELECT name, value, min, max, readonly from system.settings WHERE name = 'readonly'") ==\ + "readonly\t0\t\\N\t\\N\t0\n" + + +def test_system_constraints(started_cluster): + assert_query_settings(instance, "SELECT 1", + settings={'readonly': 0}, + exception="Cannot modify 'readonly'", + user="readonly_user") + + assert_query_settings(instance, "SELECT 1", + settings={'allow_ddl': 1}, + exception="Cannot modify 'allow_ddl'", + user="no_dll_user") + + def test_read_only_constraint(started_cluster): - # Change a setting for session with SET. - assert instance.query("SELECT value FROM system.settings WHERE name='force_index_by_date'") ==\ - "0\n" + # Default value + assert_query_settings(instance, "SELECT value FROM system.settings WHERE name='force_index_by_date'", + settings={}, + result="0") - expected_error = "Setting force_index_by_date should not be changed" - assert expected_error in instance.query_and_get_error("SET force_index_by_date=1") - - # Change a setting for query with SETTINGS. - assert instance.query("SELECT value FROM system.settings WHERE name='force_index_by_date'") ==\ - "0\n" - - assert expected_error in instance.query_and_get_error( - "SELECT value FROM system.settings WHERE name='force_index_by_date' " - "SETTINGS force_index_by_date=1") + # Invalid value + assert_query_settings(instance, "SELECT value FROM system.settings WHERE name='force_index_by_date'", + settings={'force_index_by_date': 1}, + result=None, + exception="Setting force_index_by_date should not be changed") def test_min_constraint(started_cluster): - # Change a setting for session with SET. - assert instance.query("SELECT value FROM system.settings WHERE name='max_memory_usage'") ==\ - "10000000000\n" + # Default value + assert_query_settings(instance, "SELECT value FROM system.settings WHERE name='max_memory_usage'", + {}, + result="10000000000") - assert instance.query("SET max_memory_usage=5000000000;\n" - "SELECT value FROM system.settings WHERE name='max_memory_usage'") ==\ - "5000000000\n" + # Valid value + assert_query_settings(instance, "SELECT value FROM system.settings WHERE name='max_memory_usage'", + settings={'max_memory_usage': 5000000000}, + result="5000000000") - expected_error = "Setting max_memory_usage shouldn't be less than 5000000000" - assert expected_error in instance.query_and_get_error("SET max_memory_usage=4999999999") - - # Change a setting for query with SETTINGS. - assert instance.query("SELECT value FROM system.settings WHERE name='max_memory_usage'") ==\ - "10000000000\n" - - assert instance.query("SET max_memory_usage=5000000001;\n" - "SELECT value FROM system.settings WHERE name='max_memory_usage'") ==\ - "5000000001\n" - - assert expected_error in instance.query_and_get_error( - "SELECT value FROM system.settings WHERE name='max_memory_usage' " - "SETTINGS max_memory_usage=4999999999") + # Invalid value + assert_query_settings(instance, "SELECT value FROM system.settings WHERE name='max_memory_usage'", + settings={'max_memory_usage': 4999999999}, + result=None, + exception="Setting max_memory_usage shouldn't be less than 5000000000") def test_max_constraint(started_cluster): - # Change a setting for session with SET. - assert instance.query("SELECT value FROM system.settings WHERE name='max_memory_usage'") ==\ - "10000000000\n" + # Default value + assert_query_settings(instance, "SELECT value FROM system.settings WHERE name='max_memory_usage'", + {}, + result="10000000000") - assert instance.query("SET max_memory_usage=20000000000;\n" - "SELECT value FROM system.settings WHERE name='max_memory_usage'") ==\ - "20000000000\n" + # Valid value + assert_query_settings(instance, "SELECT value FROM system.settings WHERE name='max_memory_usage'", + settings={'max_memory_usage': 20000000000}, + result="20000000000") - expected_error = "Setting max_memory_usage shouldn't be greater than 20000000000" - assert expected_error in instance.query_and_get_error("SET max_memory_usage=20000000001") + # Invalid value + assert_query_settings(instance, "SELECT value FROM system.settings WHERE name='max_memory_usage'", + settings={'max_memory_usage': 20000000001}, + result=None, + exception="Setting max_memory_usage shouldn't be greater than 20000000000") - # Change a setting for query with SETTINGS. - assert instance.query("SELECT value FROM system.settings WHERE name='max_memory_usage'") ==\ - "10000000000\n" - assert instance.query("SELECT value FROM system.settings WHERE name='max_memory_usage' " - "SETTINGS max_memory_usage=19999999999") == "19999999999\n" +def assert_query_settings(instance, query, settings, result=None, exception=None, user=None): + """ + Try and send the query with custom settings via all available methods: + 1. TCP Protocol with settings packet + 2. HTTP Protocol with settings params + 3. TCP Protocol with session level settings + 4. TCP Protocol with query level settings + """ - assert expected_error in instance.query_and_get_error( - "SELECT value FROM system.settings WHERE name='max_memory_usage' " - "SETTINGS max_memory_usage=20000000001") - \ No newline at end of file + if not settings: + settings = {} + + # tcp level settings + if exception: + assert exception in instance.query_and_get_error(query, settings=settings, user=user) + else: + assert instance.query(query, settings=settings, user=user).strip() == result + + # http level settings + if exception: + assert exception in instance.http_query(query, params=settings, user=user) + else: + assert instance.http_query(query, params=settings, user=user).strip() == result + + # session level settings + queries = "" + + for k, v in settings.items(): + queries += "SET {}={};\n".format(k, v) + + queries += query + + if exception: + assert exception in instance.query_and_get_error(queries, user=user) + else: + assert instance.query(queries, user=user).strip() == result + + if settings: + query += " SETTINGS " + for ix, (k, v) in enumerate(settings.items()): + query += "{} = {}".format(k, v) + if ix != len(settings) - 1: + query += ", " + + if exception: + assert exception in instance.query_and_get_error(queries, user=user) + else: + assert instance.query(queries, user=user).strip() == result diff --git a/dbms/tests/integration/test_storage_hdfs/test.py b/dbms/tests/integration/test_storage_hdfs/test.py index cf4205115ff..575b7593ca0 100644 --- a/dbms/tests/integration/test_storage_hdfs/test.py +++ b/dbms/tests/integration/test_storage_hdfs/test.py @@ -133,4 +133,56 @@ def test_globs_in_read_table(started_cluster): ("?", 0)] for pattern, value in test_requests: - assert node1.query("select * from hdfs('hdfs://hdfs1:9000" + globs_dir + pattern + "', 'TSV', 'id UInt64, text String, number Float64')") == value * some_data \ No newline at end of file + assert node1.query("select * from hdfs('hdfs://hdfs1:9000" + globs_dir + pattern + "', 'TSV', 'id UInt64, text String, number Float64')") == value * some_data + +def test_read_write_gzip_table(started_cluster): + hdfs_api = HDFSApi("root") + data = "1\tHello Jessica\t555.222\n2\tI rolled a joint\t777.333\n" + hdfs_api.write_gzip_data("/simple_table_function.gz", data) + + assert hdfs_api.read_gzip_data("/simple_table_function.gz") == data + + assert node1.query("select * from hdfs('hdfs://hdfs1:9000/simple_table_function.gz', 'TSV', 'id UInt64, text String, number Float64')") == data + +def test_read_write_gzip_table_with_parameter_gzip(started_cluster): + hdfs_api = HDFSApi("root") + data = "1\tHello Jessica\t555.222\n2\tI rolled a joint\t777.333\n" + hdfs_api.write_gzip_data("/simple_table_function", data) + + assert hdfs_api.read_gzip_data("/simple_table_function") == data + + assert node1.query("select * from hdfs('hdfs://hdfs1:9000/simple_table_function', 'TSV', 'id UInt64, text String, number Float64', 'gzip')") == data + +def test_read_write_table_with_parameter_none(started_cluster): + hdfs_api = HDFSApi("root") + data = "1\tHello Jessica\t555.222\n2\tI rolled a joint\t777.333\n" + hdfs_api.write_data("/simple_table_function.gz", data) + + assert hdfs_api.read_data("/simple_table_function.gz") == data + + assert node1.query("select * from hdfs('hdfs://hdfs1:9000/simple_table_function.gz', 'TSV', 'id UInt64, text String, number Float64', 'none')") == data + +def test_read_write_gzip_table_with_parameter_auto_gz(started_cluster): + hdfs_api = HDFSApi("root") + data = "1\tHello Jessica\t555.222\n2\tI rolled a joint\t777.333\n" + hdfs_api.write_gzip_data("/simple_table_function.gz", data) + + assert hdfs_api.read_gzip_data("/simple_table_function.gz") == data + + assert node1.query("select * from hdfs('hdfs://hdfs1:9000/simple_table_function.gz', 'TSV', 'id UInt64, text String, number Float64', 'auto')") == data + +def test_write_gz_storage(started_cluster): + hdfs_api = HDFSApi("root") + + node1.query("create table GZHDFSStorage (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage.gz', 'TSV')") + node1.query("insert into GZHDFSStorage values (1, 'Mark', 72.53)") + assert hdfs_api.read_gzip_data("/storage.gz") == "1\tMark\t72.53\n" + assert node1.query("select * from GZHDFSStorage") == "1\tMark\t72.53\n" + +def test_write_gzip_storage(started_cluster): + hdfs_api = HDFSApi("root") + + node1.query("create table GZIPHDFSStorage (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/gzip_storage', 'TSV', 'gzip')") + node1.query("insert into GZIPHDFSStorage values (1, 'Mark', 72.53)") + assert hdfs_api.read_gzip_data("/gzip_storage") == "1\tMark\t72.53\n" + assert node1.query("select * from GZIPHDFSStorage") == "1\tMark\t72.53\n" diff --git a/dbms/tests/integration/test_storage_kafka/configs/log_conf.xml b/dbms/tests/integration/test_storage_kafka/configs/log_conf.xml new file mode 100644 index 00000000000..95466269afe --- /dev/null +++ b/dbms/tests/integration/test_storage_kafka/configs/log_conf.xml @@ -0,0 +1,11 @@ + + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + \ No newline at end of file diff --git a/dbms/tests/integration/test_storage_kafka/test.py b/dbms/tests/integration/test_storage_kafka/test.py index a2e1511537a..cf438bf3c55 100644 --- a/dbms/tests/integration/test_storage_kafka/test.py +++ b/dbms/tests/integration/test_storage_kafka/test.py @@ -31,7 +31,7 @@ import kafka_pb2 cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance', config_dir='configs', - main_configs=['configs/kafka.xml'], + main_configs=['configs/kafka.xml', 'configs/log_conf.xml' ], with_kafka=True, clickhouse_path_dir='clickhouse_path') kafka_id = '' @@ -355,6 +355,43 @@ def test_kafka_materialized_view(kafka_cluster): kafka_check_result(result, True) +@pytest.mark.timeout(180) +def test_kafka_materialized_view_with_subquery(kafka_cluster): + instance.query(''' + DROP TABLE IF EXISTS test.view; + DROP TABLE IF EXISTS test.consumer; + CREATE TABLE test.kafka (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'mvsq', + kafka_group_name = 'mvsq', + kafka_format = 'JSONEachRow', + kafka_row_delimiter = '\\n'; + CREATE TABLE test.view (key UInt64, value UInt64) + ENGINE = MergeTree() + ORDER BY key; + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM (SELECT * FROM test.kafka); + ''') + + messages = [] + for i in range(50): + messages.append(json.dumps({'key': i, 'value': i})) + kafka_produce('mvsq', messages) + + while True: + result = instance.query('SELECT * FROM test.view') + if kafka_check_result(result): + break + + instance.query(''' + DROP TABLE test.consumer; + DROP TABLE test.view; + ''') + + kafka_check_result(result, True) + + @pytest.mark.timeout(180) def test_kafka_many_materialized_views(kafka_cluster): instance.query(''' @@ -557,7 +594,7 @@ def test_kafka_insert(kafka_cluster): kafka_check_result(result, True) -@pytest.mark.timeout(180) +@pytest.mark.timeout(240) def test_kafka_produce_consume(kafka_cluster): instance.query(''' DROP TABLE IF EXISTS test.view; diff --git a/dbms/tests/integration/test_storage_mysql/test.py b/dbms/tests/integration/test_storage_mysql/test.py index dab1110d1eb..452e56c7ab5 100644 --- a/dbms/tests/integration/test_storage_mysql/test.py +++ b/dbms/tests/integration/test_storage_mysql/test.py @@ -9,8 +9,8 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_mysql = True) -create_table_sql_template = """ +node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_mysql=True) +create_table_sql_template = """ CREATE TABLE `clickhouse`.`{}` ( `id` int(11) NOT NULL, `name` varchar(50) NOT NULL, @@ -19,6 +19,7 @@ create_table_sql_template = """ PRIMARY KEY (`id`)) ENGINE=InnoDB; """ + @pytest.fixture(scope="module") def started_cluster(): try: @@ -76,6 +77,7 @@ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL assert node1.query("SELECT sum(money) FROM {}".format(table_name)).rstrip() == '60000' conn.close() + def test_where(started_cluster): table_name = 'test_where' conn = get_mysql_conn() @@ -92,15 +94,37 @@ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL assert node1.query("SELECT count() FROM {} WHERE name LIKE concat('name_', toString(1))".format(table_name)).rstrip() == '1' conn.close() + +def test_table_function(started_cluster): + conn = get_mysql_conn() + create_mysql_table(conn, 'table_function') + table_function = "mysql('mysql1:3306', 'clickhouse', '{}', 'root', 'clickhouse')".format('table_function') + assert node1.query("SELECT count() FROM {}".format(table_function)).rstrip() == '0' + node1.query("INSERT INTO {} (id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000)".format( + 'TABLE FUNCTION ' + table_function)) + assert node1.query("SELECT count() FROM {}".format(table_function)).rstrip() == '10000' + assert node1.query("SELECT sum(c) FROM (" + "SELECT count() as c FROM {} WHERE id % 3 == 0" + " UNION ALL SELECT count() as c FROM {} WHERE id % 3 == 1" + " UNION ALL SELECT count() as c FROM {} WHERE id % 3 == 2)".format(table_function, table_function, + table_function)).rstrip() == '10000' + assert node1.query("SELECT sum(`money`) FROM {}".format(table_function)).rstrip() == '30000' + node1.query("INSERT INTO {} SELECT id + 100000, name, age, money FROM {}".format('TABLE FUNCTION ' + table_function, table_function)) + assert node1.query("SELECT sum(`money`) FROM {}".format(table_function)).rstrip() == '60000' + conn.close() + + def get_mysql_conn(): conn = pymysql.connect(user='root', password='clickhouse', host='127.0.0.1', port=3308) return conn + def create_mysql_db(conn, name): with conn.cursor() as cursor: cursor.execute( "CREATE DATABASE {} DEFAULT CHARACTER SET 'utf8'".format(name)) + def create_mysql_table(conn, tableName): with conn.cursor() as cursor: cursor.execute(create_table_sql_template.format(tableName)) diff --git a/dbms/tests/integration/test_storage_s3/configs/config_for_test_remote_host_filter.xml b/dbms/tests/integration/test_storage_s3/configs/config_for_test_remote_host_filter.xml new file mode 100644 index 00000000000..98a9d547705 --- /dev/null +++ b/dbms/tests/integration/test_storage_s3/configs/config_for_test_remote_host_filter.xml @@ -0,0 +1,5 @@ + + + ^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|$)){4}$ + + diff --git a/dbms/tests/integration/test_storage_s3/test.py b/dbms/tests/integration/test_storage_s3/test.py index c5e7d2a7cf1..a651df9eb70 100644 --- a/dbms/tests/integration/test_storage_s3/test.py +++ b/dbms/tests/integration/test_storage_s3/test.py @@ -1,159 +1,264 @@ -import httplib import json import logging -import os -import time -import traceback import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, ClickHouseInstance + +import helpers.client logging.getLogger().setLevel(logging.INFO) logging.getLogger().addHandler(logging.StreamHandler()) -def get_communication_data(started_cluster): - conn = httplib.HTTPConnection(started_cluster.instances["dummy"].ip_address, started_cluster.communication_port) - conn.request("GET", "/") - r = conn.getresponse() - raw_data = r.read() - conn.close() - return json.loads(raw_data) +# Creates S3 bucket for tests and allows anonymous read-write access to it. +def prepare_s3_bucket(cluster): + minio_client = cluster.minio_client + + if minio_client.bucket_exists(cluster.minio_bucket): + minio_client.remove_bucket(cluster.minio_bucket) + + minio_client.make_bucket(cluster.minio_bucket) + + # Allows read-write access for bucket without authorization. + bucket_read_write_policy = {"Version": "2012-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": "s3:GetBucketLocation", + "Resource": "arn:aws:s3:::root" + }, + { + "Sid": "", + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::root" + }, + { + "Sid": "", + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::root/*" + }, + { + "Sid": "", + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::root/*" + } + ]} + + minio_client.set_bucket_policy(cluster.minio_bucket, json.dumps(bucket_read_write_policy)) + + cluster.minio_restricted_bucket = "{}-with-auth".format(cluster.minio_bucket) + if minio_client.bucket_exists(cluster.minio_restricted_bucket): + minio_client.remove_bucket(cluster.minio_restricted_bucket) + + minio_client.make_bucket(cluster.minio_restricted_bucket) -def put_communication_data(started_cluster, body): - conn = httplib.HTTPConnection(started_cluster.instances["dummy"].ip_address, started_cluster.communication_port) - conn.request("PUT", "/", body) - r = conn.getresponse() - conn.close() +# Returns content of given S3 file as string. +def get_s3_file_content(cluster, bucket, filename): + # type: (ClickHouseCluster, str) -> str + + data = cluster.minio_client.get_object(bucket, filename) + data_str = "" + for chunk in data.stream(): + data_str += chunk + return data_str + + +# Returns nginx access log lines. +def get_nginx_access_logs(): + handle = open("/nginx/access.log", "r") + data = handle.readlines() + handle.close() + return data @pytest.fixture(scope="module") -def started_cluster(): +def cluster(): try: cluster = ClickHouseCluster(__file__) - instance = cluster.add_instance("dummy") + cluster.add_instance("restricted_dummy", main_configs=["configs/config_for_test_remote_host_filter.xml"], with_minio=True) + cluster.add_instance("dummy", with_minio=True) + logging.info("Starting cluster...") cluster.start() + logging.info("Cluster started") - cluster.communication_port = 10000 - instance.copy_file_to_container(os.path.join(os.path.dirname(__file__), "test_server.py"), "test_server.py") - cluster.bucket = "abc" - instance.exec_in_container(["python", "test_server.py", str(cluster.communication_port), cluster.bucket], detach=True) - cluster.mock_host = instance.ip_address - - for i in range(10): - try: - data = get_communication_data(cluster) - cluster.redirecting_to_http_port = data["redirecting_to_http_port"] - cluster.preserving_data_port = data["preserving_data_port"] - cluster.multipart_preserving_data_port = data["multipart_preserving_data_port"] - cluster.redirecting_preserving_data_port = data["redirecting_preserving_data_port"] - except: - logging.error(traceback.format_exc()) - time.sleep(0.5) - else: - break - else: - assert False, "Could not initialize mock server" + prepare_s3_bucket(cluster) + logging.info("S3 bucket created") yield cluster - finally: cluster.shutdown() def run_query(instance, query, stdin=None, settings=None): + # type: (ClickHouseInstance, str, object, dict) -> str + logging.info("Running query '{}'...".format(query)) result = instance.query(query, stdin=stdin, settings=settings) logging.info("Query finished") + return result -def test_get_with_redirect(started_cluster): - instance = started_cluster.instances["dummy"] - format = "column1 UInt32, column2 UInt32, column3 UInt32" +# Test simple put. +@pytest.mark.parametrize("maybe_auth,positive", [ + ("", True), + ("'minio','minio123',", True), + ("'wrongid','wrongkey',", False) +]) +def test_put(cluster, maybe_auth, positive): + # type: (ClickHouseCluster) -> None - put_communication_data(started_cluster, "=== Get with redirect test ===") - query = "select *, column1*column2*column3 from s3('http://{}:{}/', 'CSV', '{}')".format(started_cluster.mock_host, started_cluster.redirecting_to_http_port, format) - stdout = run_query(instance, query) - data = get_communication_data(started_cluster) - expected = [ [str(row[0]), str(row[1]), str(row[2]), str(row[0]*row[1]*row[2])] for row in data["redirect_csv_data"] ] - assert list(map(str.split, stdout.splitlines())) == expected - - -def test_put(started_cluster): - instance = started_cluster.instances["dummy"] - format = "column1 UInt32, column2 UInt32, column3 UInt32" - - logging.info("Phase 3") - put_communication_data(started_cluster, "=== Put test ===") + bucket = cluster.minio_bucket if not maybe_auth else cluster.minio_restricted_bucket + instance = cluster.instances["dummy"] # type: ClickHouseInstance + table_format = "column1 UInt32, column2 UInt32, column3 UInt32" values = "(1, 2, 3), (3, 2, 1), (78, 43, 45)" - put_query = "insert into table function s3('http://{}:{}/{}/test.csv', 'CSV', '{}') values {}".format(started_cluster.mock_host, started_cluster.preserving_data_port, started_cluster.bucket, format, values) - run_query(instance, put_query) - data = get_communication_data(started_cluster) - received_data_completed = data["received_data_completed"] - received_data = data["received_data"] - finalize_data = data["finalize_data"] - finalize_data_query = data["finalize_data_query"] - assert received_data[-1].decode() == "1,2,3\n3,2,1\n78,43,45\n" - assert received_data_completed - assert finalize_data == "1hello-etag" - assert finalize_data_query == "uploadId=TEST" + values_csv = "1,2,3\n3,2,1\n78,43,45\n" + filename = "test.csv" + put_query = "insert into table function s3('http://{}:{}/{}/{}', {}'CSV', '{}') values {}".format( + cluster.minio_host, cluster.minio_port, bucket, filename, maybe_auth, table_format, values) - -def test_put_csv(started_cluster): - instance = started_cluster.instances["dummy"] - format = "column1 UInt32, column2 UInt32, column3 UInt32" + try: + run_query(instance, put_query) + except helpers.client.QueryRuntimeException: + if positive: + raise + else: + assert positive + assert values_csv == get_s3_file_content(cluster, bucket, filename) - put_communication_data(started_cluster, "=== Put test CSV ===") - put_query = "insert into table function s3('http://{}:{}/{}/test.csv', 'CSV', '{}') format CSV".format(started_cluster.mock_host, started_cluster.preserving_data_port, started_cluster.bucket, format) + +# Test put values in CSV format. +@pytest.mark.parametrize("maybe_auth,positive", [ + ("", True), + ("'minio','minio123',", True), + ("'wrongid','wrongkey',", False) +]) +def test_put_csv(cluster, maybe_auth, positive): + # type: (ClickHouseCluster) -> None + + bucket = cluster.minio_bucket if not maybe_auth else cluster.minio_restricted_bucket + instance = cluster.instances["dummy"] # type: ClickHouseInstance + table_format = "column1 UInt32, column2 UInt32, column3 UInt32" + filename = "test.csv" + put_query = "insert into table function s3('http://{}:{}/{}/{}', {}'CSV', '{}') format CSV".format( + cluster.minio_host, cluster.minio_port, bucket, filename, maybe_auth, table_format) csv_data = "8,9,16\n11,18,13\n22,14,2\n" - run_query(instance, put_query, stdin=csv_data) - data = get_communication_data(started_cluster) - received_data_completed = data["received_data_completed"] - received_data = data["received_data"] - finalize_data = data["finalize_data"] - finalize_data_query = data["finalize_data_query"] - assert received_data[-1].decode() == csv_data - assert received_data_completed - assert finalize_data == "1hello-etag" - assert finalize_data_query == "uploadId=TEST" - -def test_put_with_redirect(started_cluster): - instance = started_cluster.instances["dummy"] - format = "column1 UInt32, column2 UInt32, column3 UInt32" + try: + run_query(instance, put_query, stdin=csv_data) + except helpers.client.QueryRuntimeException: + if positive: + raise + else: + assert positive + assert csv_data == get_s3_file_content(cluster, bucket, filename) - put_communication_data(started_cluster, "=== Put with redirect test ===") - other_values = "(1, 1, 1), (1, 1, 1), (11, 11, 11)" - query = "insert into table function s3('http://{}:{}/{}/test.csv', 'CSV', '{}') values {}".format(started_cluster.mock_host, started_cluster.redirecting_preserving_data_port, started_cluster.bucket, format, other_values) + +# Test put and get with S3 server redirect. +def test_put_get_with_redirect(cluster): + # type: (ClickHouseCluster) -> None + + bucket = cluster.minio_bucket + instance = cluster.instances["dummy"] # type: ClickHouseInstance + table_format = "column1 UInt32, column2 UInt32, column3 UInt32" + values = "(1, 1, 1), (1, 1, 1), (11, 11, 11)" + values_csv = "1,1,1\n1,1,1\n11,11,11\n" + filename = "test.csv" + query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values {}".format( + cluster.minio_redirect_host, cluster.minio_redirect_port, bucket, filename, table_format, values) run_query(instance, query) - query = "select *, column1*column2*column3 from s3('http://{}:{}/{}/test.csv', 'CSV', '{}')".format(started_cluster.mock_host, started_cluster.preserving_data_port, started_cluster.bucket, format) + assert values_csv == get_s3_file_content(cluster, bucket, filename) + + query = "select *, column1*column2*column3 from s3('http://{}:{}/{}/{}', 'CSV', '{}')".format( + cluster.minio_redirect_host, cluster.minio_redirect_port, bucket, filename, table_format) stdout = run_query(instance, query) + assert list(map(str.split, stdout.splitlines())) == [ ["1", "1", "1", "1"], ["1", "1", "1", "1"], ["11", "11", "11", "1331"], ] - data = get_communication_data(started_cluster) - received_data = data["received_data"] - assert received_data[-1].decode() == "1,1,1\n1,1,1\n11,11,11\n" -def test_multipart_put(started_cluster): - instance = started_cluster.instances["dummy"] +# Test multipart put. +@pytest.mark.parametrize("maybe_auth,positive", [ + ("", True), + # ("'minio','minio123',",True), Redirect with credentials not working with nginx. + ("'wrongid','wrongkey',", False) +]) +def test_multipart_put(cluster, maybe_auth, positive): + # type: (ClickHouseCluster) -> None + + bucket = cluster.minio_bucket if not maybe_auth else cluster.minio_restricted_bucket + instance = cluster.instances["dummy"] # type: ClickHouseInstance + table_format = "column1 UInt32, column2 UInt32, column3 UInt32" + + # Minimum size of part is 5 Mb for Minio. + # See: https://github.com/minio/minio/blob/master/docs/minio-limits.md + min_part_size_bytes = 5 * 1024 * 1024 + csv_size_bytes = int(min_part_size_bytes * 1.5) # To have 2 parts. + + one_line_length = 6 # 3 digits, 2 commas, 1 line separator. + + # Generate data having size more than one part + int_data = [[1, 2, 3] for i in range(csv_size_bytes / one_line_length)] + csv_data = "".join(["{},{},{}\n".format(x, y, z) for x, y, z in int_data]) + + assert len(csv_data) > min_part_size_bytes + + filename = "test_multipart.csv" + put_query = "insert into table function s3('http://{}:{}/{}/{}', {}'CSV', '{}') format CSV".format( + cluster.minio_redirect_host, cluster.minio_redirect_port, bucket, filename, maybe_auth, table_format) + + try: + run_query(instance, put_query, stdin=csv_data, settings={'s3_min_upload_part_size': min_part_size_bytes}) + except helpers.client.QueryRuntimeException: + if positive: + raise + else: + assert positive + + # Use Nginx access logs to count number of parts uploaded to Minio. + nginx_logs = get_nginx_access_logs() + uploaded_parts = filter(lambda log_line: log_line.find(filename) >= 0 and log_line.find("PUT") >= 0, nginx_logs) + assert len(uploaded_parts) > 1 + + assert csv_data == get_s3_file_content(cluster, bucket, filename) + + +def test_remote_host_filter(cluster): + instance = cluster.instances["restricted_dummy"] format = "column1 UInt32, column2 UInt32, column3 UInt32" - put_communication_data(started_cluster, "=== Multipart test ===") - long_data = [[i, i+1, i+2] for i in range(100000)] - long_values = "".join([ "{},{},{}\n".format(x,y,z) for x, y, z in long_data ]) - put_query = "insert into table function s3('http://{}:{}/{}/test.csv', 'CSV', '{}') format CSV".format(started_cluster.mock_host, started_cluster.multipart_preserving_data_port, started_cluster.bucket, format) - run_query(instance, put_query, stdin=long_values, settings={'s3_min_upload_part_size': 1000000}) - data = get_communication_data(started_cluster) - assert "multipart_received_data" in data - received_data = data["multipart_received_data"] - assert received_data[-1].decode() == "".join([ "{},{},{}\n".format(x, y, z) for x, y, z in long_data ]) - assert 1 < data["multipart_parts"] < 10000 + query = "select *, column1*column2*column3 from s3('http://{}:{}/{}/test.csv', 'CSV', '{}')".format( + "invalid_host", cluster.minio_port, cluster.minio_bucket, format) + assert "not allowed in config.xml" in instance.query_and_get_error(query) + + other_values = "(1, 1, 1), (1, 1, 1), (11, 11, 11)" + query = "insert into table function s3('http://{}:{}/{}/test.csv', 'CSV', '{}') values {}".format( + "invalid_host", cluster.minio_port, cluster.minio_bucket, format, other_values) + assert "not allowed in config.xml" in instance.query_and_get_error(query) + + +@pytest.mark.parametrize("s3_storage_args", [ + "''", # 1 arguments + "'','','','','',''" # 6 arguments +]) +def test_wrong_s3_syntax(cluster, s3_storage_args): + instance = cluster.instances["dummy"] # type: ClickHouseInstance + expected_err_msg = "Code: 42" # NUMBER_OF_ARGUMENTS_DOESNT_MATCH + + query = "create table test_table_s3_syntax (id UInt32) ENGINE = S3({})".format(s3_storage_args) + assert expected_err_msg in instance.query_and_get_error(query) diff --git a/dbms/tests/integration/test_storage_s3/test_server.py b/dbms/tests/integration/test_storage_s3/test_server.py deleted file mode 100644 index 08a1904d1f2..00000000000 --- a/dbms/tests/integration/test_storage_s3/test_server.py +++ /dev/null @@ -1,367 +0,0 @@ -try: - from BaseHTTPServer import BaseHTTPRequestHandler -except ImportError: - from http.server import BaseHTTPRequestHandler - -try: - from BaseHTTPServer import HTTPServer -except ImportError: - from http.server import HTTPServer - -try: - import urllib.parse as urlparse -except ImportError: - import urlparse - -import json -import logging -import os -import socket -import sys -import threading -import time -import uuid -import xml.etree.ElementTree - - -logging.getLogger().setLevel(logging.INFO) -file_handler = logging.FileHandler("/var/log/clickhouse-server/test-server.log", "a", encoding="utf-8") -file_handler.setFormatter(logging.Formatter("%(asctime)s %(message)s")) -logging.getLogger().addHandler(file_handler) -logging.getLogger().addHandler(logging.StreamHandler()) - -communication_port = int(sys.argv[1]) -bucket = sys.argv[2] - - -def GetFreeTCPPortsAndIP(n): - result = [] - sockets = [] - for i in range(n): - tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - tcp.bind((socket.gethostname(), 0)) - addr, port = tcp.getsockname() - result.append(port) - sockets.append(tcp) - [ s.close() for s in sockets ] - return result, addr - -( - redirecting_to_http_port, - simple_server_port, - preserving_data_port, - multipart_preserving_data_port, - redirecting_preserving_data_port -), localhost = GetFreeTCPPortsAndIP(5) - - -data = { - "redirecting_to_http_port": redirecting_to_http_port, - "preserving_data_port": preserving_data_port, - "multipart_preserving_data_port": multipart_preserving_data_port, - "redirecting_preserving_data_port": redirecting_preserving_data_port, -} - - -class SimpleHTTPServerHandler(BaseHTTPRequestHandler): - def do_GET(self): - logging.info("GET {}".format(self.path)) - if self.path == "/milovidov/test.csv": - self.send_response(200) - self.send_header("Content-type", "text/plain") - self.end_headers() - data["redirect_csv_data"] = [[42, 87, 44], [55, 33, 81], [1, 0, 9]] - self.wfile.write("".join([ "{},{},{}\n".format(*row) for row in data["redirect_csv_data"]])) - else: - self.send_response(404) - self.end_headers() - self.finish() - - -class RedirectingToHTTPHandler(BaseHTTPRequestHandler): - def do_GET(self): - self.send_response(307) - self.send_header("Content-type", "text/xml") - self.send_header("Location", "http://{}:{}/milovidov/test.csv".format(localhost, simple_server_port)) - self.end_headers() - self.wfile.write(r""" - - TemporaryRedirect - Please re-send this request to the specified temporary endpoint. - Continue to use the original request endpoint for future requests. - storage.yandexcloud.net -""".encode()) - self.finish() - - -class PreservingDataHandler(BaseHTTPRequestHandler): - protocol_version = "HTTP/1.1" - - def parse_request(self): - result = BaseHTTPRequestHandler.parse_request(self) - # Adaptation to Python 3. - if sys.version_info.major == 2 and result == True: - expect = self.headers.get("Expect", "") - if (expect.lower() == "100-continue" and self.protocol_version >= "HTTP/1.1" and self.request_version >= "HTTP/1.1"): - if not self.handle_expect_100(): - return False - return result - - def send_response_only(self, code, message=None): - if message is None: - if code in self.responses: - message = self.responses[code][0] - else: - message = "" - if self.request_version != "HTTP/0.9": - self.wfile.write("%s %d %s\r\n" % (self.protocol_version, code, message)) - - def handle_expect_100(self): - logging.info("Received Expect-100") - self.send_response_only(100) - self.end_headers() - return True - - def do_POST(self): - self.send_response(200) - query = urlparse.urlparse(self.path).query - logging.info("PreservingDataHandler POST ?" + query) - if query == "uploads": - post_data = r""" -TEST""".encode() - self.send_header("Content-length", str(len(post_data))) - self.send_header("Content-type", "text/plain") - self.end_headers() - self.wfile.write(post_data) - else: - post_data = self.rfile.read(int(self.headers.get("Content-Length"))) - self.send_header("Content-type", "text/plain") - self.end_headers() - data["received_data_completed"] = True - data["finalize_data"] = post_data - data["finalize_data_query"] = query - self.finish() - - def do_PUT(self): - self.send_response(200) - self.send_header("Content-type", "text/plain") - self.send_header("ETag", "hello-etag") - self.end_headers() - query = urlparse.urlparse(self.path).query - path = urlparse.urlparse(self.path).path - logging.info("Content-Length = " + self.headers.get("Content-Length")) - logging.info("PUT " + query) - assert self.headers.get("Content-Length") - assert self.headers["Expect"] == "100-continue" - put_data = self.rfile.read() - data.setdefault("received_data", []).append(put_data) - logging.info("PUT to {}".format(path)) - self.server.storage[path] = put_data - self.finish() - - def do_GET(self): - path = urlparse.urlparse(self.path).path - if path in self.server.storage: - self.send_response(200) - self.send_header("Content-type", "text/plain") - self.send_header("Content-length", str(len(self.server.storage[path]))) - self.end_headers() - self.wfile.write(self.server.storage[path]) - else: - self.send_response(404) - self.end_headers() - self.finish() - - -class MultipartPreservingDataHandler(BaseHTTPRequestHandler): - protocol_version = "HTTP/1.1" - - def parse_request(self): - result = BaseHTTPRequestHandler.parse_request(self) - # Adaptation to Python 3. - if sys.version_info.major == 2 and result == True: - expect = self.headers.get("Expect", "") - if (expect.lower() == "100-continue" and self.protocol_version >= "HTTP/1.1" and self.request_version >= "HTTP/1.1"): - if not self.handle_expect_100(): - return False - return result - - def send_response_only(self, code, message=None): - if message is None: - if code in self.responses: - message = self.responses[code][0] - else: - message = "" - if self.request_version != "HTTP/0.9": - self.wfile.write("%s %d %s\r\n" % (self.protocol_version, code, message)) - - def handle_expect_100(self): - logging.info("Received Expect-100") - self.send_response_only(100) - self.end_headers() - return True - - def do_POST(self): - query = urlparse.urlparse(self.path).query - logging.info("MultipartPreservingDataHandler POST ?" + query) - if query == "uploads": - self.send_response(200) - post_data = r""" -TEST""".encode() - self.send_header("Content-length", str(len(post_data))) - self.send_header("Content-type", "text/plain") - self.end_headers() - self.wfile.write(post_data) - else: - try: - assert query == "uploadId=TEST" - logging.info("Content-Length = " + self.headers.get("Content-Length")) - post_data = self.rfile.read(int(self.headers.get("Content-Length"))) - root = xml.etree.ElementTree.fromstring(post_data) - assert root.tag == "CompleteMultipartUpload" - assert len(root) > 1 - content = "" - for i, part in enumerate(root): - assert part.tag == "Part" - assert len(part) == 2 - assert part[0].tag == "PartNumber" - assert part[1].tag == "ETag" - assert int(part[0].text) == i + 1 - content += self.server.storage["@"+part[1].text] - data.setdefault("multipart_received_data", []).append(content) - data["multipart_parts"] = len(root) - self.send_response(200) - self.send_header("Content-type", "text/plain") - self.end_headers() - logging.info("Sending 200") - except: - logging.error("Sending 500") - self.send_response(500) - self.finish() - - def do_PUT(self): - uid = uuid.uuid4() - self.send_response(200) - self.send_header("Content-type", "text/plain") - self.send_header("ETag", str(uid)) - self.end_headers() - query = urlparse.urlparse(self.path).query - path = urlparse.urlparse(self.path).path - logging.info("Content-Length = " + self.headers.get("Content-Length")) - logging.info("PUT " + query) - assert self.headers.get("Content-Length") - assert self.headers["Expect"] == "100-continue" - put_data = self.rfile.read() - data.setdefault("received_data", []).append(put_data) - logging.info("PUT to {}".format(path)) - self.server.storage["@"+str(uid)] = put_data - self.finish() - - def do_GET(self): - path = urlparse.urlparse(self.path).path - if path in self.server.storage: - self.send_response(200) - self.send_header("Content-type", "text/plain") - self.send_header("Content-length", str(len(self.server.storage[path]))) - self.end_headers() - self.wfile.write(self.server.storage[path]) - else: - self.send_response(404) - self.end_headers() - self.finish() - - -class RedirectingPreservingDataHandler(BaseHTTPRequestHandler): - protocol_version = "HTTP/1.1" - - def parse_request(self): - result = BaseHTTPRequestHandler.parse_request(self) - # Adaptation to Python 3. - if sys.version_info.major == 2 and result == True: - expect = self.headers.get("Expect", "") - if (expect.lower() == "100-continue" and self.protocol_version >= "HTTP/1.1" and self.request_version >= "HTTP/1.1"): - if not self.handle_expect_100(): - return False - return result - - def send_response_only(self, code, message=None): - if message is None: - if code in self.responses: - message = self.responses[code][0] - else: - message = "" - if self.request_version != "HTTP/0.9": - self.wfile.write("%s %d %s\r\n" % (self.protocol_version, code, message)) - - def handle_expect_100(self): - logging.info("Received Expect-100") - return True - - def do_POST(self): - query = urlparse.urlparse(self.path).query - if query: - query = "?{}".format(query) - self.send_response(307) - self.send_header("Content-type", "text/xml") - self.send_header("Location", "http://{host}:{port}/{bucket}/test.csv{query}".format(host=localhost, port=preserving_data_port, bucket=bucket, query=query)) - self.end_headers() - self.wfile.write(r""" - - TemporaryRedirect - Please re-send this request to the specified temporary endpoint. - Continue to use the original request endpoint for future requests. - {host}:{port} -""".format(host=localhost, port=preserving_data_port).encode()) - self.finish() - - def do_PUT(self): - query = urlparse.urlparse(self.path).query - if query: - query = "?{}".format(query) - self.send_response(307) - self.send_header("Content-type", "text/xml") - self.send_header("Location", "http://{host}:{port}/{bucket}/test.csv{query}".format(host=localhost, port=preserving_data_port, bucket=bucket, query=query)) - self.end_headers() - self.wfile.write(r""" - - TemporaryRedirect - Please re-send this request to the specified temporary endpoint. - Continue to use the original request endpoint for future requests. - {host}:{port} -""".format(host=localhost, port=preserving_data_port).encode()) - self.finish() - - -class CommunicationServerHandler(BaseHTTPRequestHandler): - def do_GET(self): - self.send_response(200) - self.end_headers() - self.wfile.write(json.dumps(data)) - self.finish() - - def do_PUT(self): - self.send_response(200) - self.end_headers() - logging.info(self.rfile.read()) - self.finish() - - -servers = [] -servers.append(HTTPServer((localhost, communication_port), CommunicationServerHandler)) -servers.append(HTTPServer((localhost, redirecting_to_http_port), RedirectingToHTTPHandler)) -servers.append(HTTPServer((localhost, preserving_data_port), PreservingDataHandler)) -servers[-1].storage = {} -servers.append(HTTPServer((localhost, multipart_preserving_data_port), MultipartPreservingDataHandler)) -servers[-1].storage = {} -servers.append(HTTPServer((localhost, simple_server_port), SimpleHTTPServerHandler)) -servers.append(HTTPServer((localhost, redirecting_preserving_data_port), RedirectingPreservingDataHandler)) -jobs = [ threading.Thread(target=server.serve_forever) for server in servers ] -[ job.start() for job in jobs ] - -time.sleep(60) # Timeout - -logging.info("Shutting down") -[ server.shutdown() for server in servers ] -logging.info("Joining threads") -[ job.join() for job in jobs ] -logging.info("Done") diff --git a/dbms/tests/integration/test_system_merges/__init__.py b/dbms/tests/integration/test_system_merges/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_system_merges/configs/config.d/cluster.xml b/dbms/tests/integration/test_system_merges/configs/config.d/cluster.xml new file mode 100644 index 00000000000..ec7c9b8e4f8 --- /dev/null +++ b/dbms/tests/integration/test_system_merges/configs/config.d/cluster.xml @@ -0,0 +1,16 @@ + + + + + + node1 + 9000 + + + node2 + 9000 + + + + + \ No newline at end of file diff --git a/dbms/tests/integration/test_system_merges/configs/logs_config.xml b/dbms/tests/integration/test_system_merges/configs/logs_config.xml new file mode 100644 index 00000000000..bdf1bbc11c1 --- /dev/null +++ b/dbms/tests/integration/test_system_merges/configs/logs_config.xml @@ -0,0 +1,17 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + + system + part_log
+ 500 + + diff --git a/dbms/tests/integration/test_system_merges/test.py b/dbms/tests/integration/test_system_merges/test.py new file mode 100644 index 00000000000..7b638ce05c7 --- /dev/null +++ b/dbms/tests/integration/test_system_merges/test.py @@ -0,0 +1,160 @@ +import pytest +import threading +import time +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance('node1', + config_dir='configs', + main_configs=['configs/logs_config.xml'], + with_zookeeper=True, + macros={"shard": 0, "replica": 1} ) + +node2 = cluster.add_instance('node2', + config_dir='configs', + main_configs=['configs/logs_config.xml'], + with_zookeeper=True, + macros={"shard": 0, "replica": 2} ) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def split_tsv(data): + return [ x.split("\t") for x in data.splitlines() ] + + +@pytest.mark.parametrize("replicated", [ + "", + "replicated" +]) +def test_merge_simple(started_cluster, replicated): + try: + clickhouse_path = "/var/lib/clickhouse" + name = "test_merge_simple" + nodes = [node1, node2] if replicated else [node1] + engine = "ReplicatedMergeTree('/clickhouse/test_merge_simple', '{replica}')" if replicated else "MergeTree()" + node_check = nodes[-1] + starting_block = 0 if replicated else 1 + + for node in nodes: + node.query(""" + CREATE TABLE {name} + ( + `a` Int64 + ) + ENGINE = {engine} + ORDER BY sleep(2) + """.format(engine=engine, name=name)) + + node1.query("INSERT INTO {name} VALUES (1)".format(name=name)) + node1.query("INSERT INTO {name} VALUES (2)".format(name=name)) + node1.query("INSERT INTO {name} VALUES (3)".format(name=name)) + + parts = ["all_{}_{}_0".format(x, x) for x in range(starting_block, starting_block+3)] + result_part = "all_{}_{}_1".format(starting_block, starting_block+2) + + def optimize(): + node1.query("OPTIMIZE TABLE {name}".format(name=name)) + + wait = threading.Thread(target=time.sleep, args=(5,)) + wait.start() + t = threading.Thread(target=optimize) + t.start() + + time.sleep(1) + assert split_tsv(node_check.query(""" + SELECT database, table, num_parts, source_part_names, source_part_paths, result_part_name, result_part_path, partition_id, is_mutation + FROM system.merges + WHERE table = '{name}' + """.format(name=name))) == [ + [ + "default", + name, + "3", + "['{}','{}','{}']".format(*parts), + "['{clickhouse}/data/default/{name}/{}/','{clickhouse}/data/default/{name}/{}/','{clickhouse}/data/default/{name}/{}/']".format(*parts, clickhouse=clickhouse_path, name=name), + result_part, + "{clickhouse}/data/default/{name}/{}/".format(result_part, clickhouse=clickhouse_path, name=name), + "all", + "0" + ] + ] + t.join() + wait.join() + + assert node_check.query("SELECT * FROM system.merges WHERE table = '{name}'".format(name=name)) == "" + + finally: + for node in nodes: + node.query("DROP TABLE {name}".format(name=name)) + + +@pytest.mark.parametrize("replicated", [ + "", + "replicated" +]) +def test_mutation_simple(started_cluster, replicated): + try: + clickhouse_path = "/var/lib/clickhouse" + name = "test_mutation_simple" + nodes = [node1, node2] if replicated else [node1] + engine = "ReplicatedMergeTree('/clickhouse/test_mutation_simple', '{replica}')" if replicated else "MergeTree()" + node_check = nodes[-1] + starting_block = 0 if replicated else 1 + + for node in nodes: + node.query(""" + CREATE TABLE {name} + ( + `a` Int64 + ) + ENGINE = {engine} + ORDER BY tuple() + """.format(engine=engine, name=name)) + + node1.query("INSERT INTO {name} VALUES (1)".format(name=name)) + part = "all_{}_{}_0".format(starting_block, starting_block) + result_part = "all_{}_{}_0_{}".format(starting_block, starting_block, starting_block+1) + + def alter(): + node1.query("ALTER TABLE {name} UPDATE a = 42 WHERE sleep(2) OR 1".format(name=name)) + + t = threading.Thread(target=alter) + t.start() + + time.sleep(1) + assert split_tsv(node_check.query(""" + SELECT database, table, num_parts, source_part_names, source_part_paths, result_part_name, result_part_path, partition_id, is_mutation + FROM system.merges + WHERE table = '{name}' + """.format(name=name))) == [ + [ + "default", + name, + "1", + "['{}']".format(part), + "['{clickhouse}/data/default/{name}/{}/']".format(part, clickhouse=clickhouse_path, name=name), + result_part, + "{clickhouse}/data/default/{name}/{}/".format(result_part, clickhouse=clickhouse_path, name=name), + "all", + "1" + ], + ] + t.join() + + time.sleep(1.5) + + assert node_check.query("SELECT * FROM system.merges WHERE table = '{name}'".format(name=name)) == "" + + finally: + for node in nodes: + node.query("DROP TABLE {name}".format(name=name)) diff --git a/dbms/tests/integration/test_ttl_move/__init__.py b/dbms/tests/integration/test_ttl_move/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_ttl_move/configs/config.d/cluster.xml b/dbms/tests/integration/test_ttl_move/configs/config.d/cluster.xml new file mode 100644 index 00000000000..ec7c9b8e4f8 --- /dev/null +++ b/dbms/tests/integration/test_ttl_move/configs/config.d/cluster.xml @@ -0,0 +1,16 @@ + + + + + + node1 + 9000 + + + node2 + 9000 + + + + + \ No newline at end of file diff --git a/dbms/tests/integration/test_ttl_move/configs/config.d/instant_moves.xml b/dbms/tests/integration/test_ttl_move/configs/config.d/instant_moves.xml new file mode 100644 index 00000000000..7b68c6946ca --- /dev/null +++ b/dbms/tests/integration/test_ttl_move/configs/config.d/instant_moves.xml @@ -0,0 +1,4 @@ + + 0.5 + 0.5 + diff --git a/dbms/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml b/dbms/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml new file mode 100644 index 00000000000..b48de85007a --- /dev/null +++ b/dbms/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml @@ -0,0 +1,76 @@ + + + + + + + + /jbod1/ + + + /jbod2/ + + + /external/ + + + + + + + + external + +
+ jbod1 + jbod2 +
+ + + + + +
+ jbod1 + jbod2 +
+ + external + + + + + + +
+ jbod1 +
+ + external + + + + + + +
+ jbod1 +
+ + jbod2 + + + + + + +
+ jbod2 +
+ + + + + + + diff --git a/dbms/tests/integration/test_ttl_move/configs/logs_config.xml b/dbms/tests/integration/test_ttl_move/configs/logs_config.xml new file mode 100644 index 00000000000..bdf1bbc11c1 --- /dev/null +++ b/dbms/tests/integration/test_ttl_move/configs/logs_config.xml @@ -0,0 +1,17 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + + system + part_log
+ 500 + + diff --git a/dbms/tests/integration/test_ttl_move/test.py b/dbms/tests/integration/test_ttl_move/test.py new file mode 100644 index 00000000000..00eb50febea --- /dev/null +++ b/dbms/tests/integration/test_ttl_move/test.py @@ -0,0 +1,442 @@ +import json +import pytest +import random +import re +import string +import threading +import time +from multiprocessing.dummy import Pool +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV + + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance('node1', + config_dir='configs', + main_configs=['configs/logs_config.xml'], + with_zookeeper=True, + tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/external:size=200M'], + macros={"shard": 0, "replica": 1} ) + +node2 = cluster.add_instance('node2', + config_dir='configs', + main_configs=['configs/logs_config.xml'], + with_zookeeper=True, + tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/external:size=200M'], + macros={"shard": 0, "replica": 2} ) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def get_random_string(length): + symbols = bytes(string.ascii_uppercase + string.digits) + result_list = bytearray([0])*length + for i in range(length): + result_list[i] = random.choice(symbols) + return str(result_list) + + +def get_used_disks_for_table(node, table_name): + return node.query("select disk_name from system.parts where table == '{}' and active=1 order by modification_time".format(table_name)).strip().split('\n') + + +@pytest.mark.parametrize("name,engine,positive", [ + ("mt_test_inserts_to_disk_do_not_work","MergeTree()",0), + ("replicated_mt_test_inserts_to_disk_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_disk_do_not_work', '1')",0), + ("mt_test_inserts_to_disk_work","MergeTree()",1), + ("replicated_mt_test_inserts_to_disk_work","ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_disk_work', '1')",1), +]) +def test_inserts_to_disk_work(started_cluster, name, engine, positive): + try: + node1.query(""" + CREATE TABLE {name} ( + s1 String, + d1 DateTime + ) ENGINE = {engine} + ORDER BY tuple() + TTL d1 TO DISK 'external' + SETTINGS storage_policy='small_jbod_with_external' + """.format(name=name, engine=engine)) + + data = [] # 10MB in total + for i in range(10): + data.append(("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time.time()-1 if i > 0 or positive else time.time()+300))) # 1MB row + + node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + used_disks = get_used_disks_for_table(node1, name) + assert set(used_disks) == {"external" if positive else "jbod1"} + + assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + + finally: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + + +@pytest.mark.parametrize("name,engine,positive", [ + ("mt_test_moves_to_disk_do_not_work","MergeTree()",0), + ("replicated_mt_test_moves_to_disk_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_disk_do_not_work', '1')",0), + ("mt_test_moves_to_disk_work","MergeTree()",1), + ("replicated_mt_test_moves_to_disk_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_disk_work', '1')",1), +]) +def test_moves_to_disk_work(started_cluster, name, engine, positive): + try: + node1.query(""" + CREATE TABLE {name} ( + s1 String, + d1 DateTime + ) ENGINE = {engine} + ORDER BY tuple() + TTL d1 TO DISK 'external' + SETTINGS storage_policy='small_jbod_with_external' + """.format(name=name, engine=engine)) + + wait_expire_1 = 6 + wait_expire_2 = 4 + time_1 = time.time() + wait_expire_1 + time_2 = time.time() + wait_expire_1 + wait_expire_2 + + wait_expire_1_thread = threading.Thread(target=time.sleep, args=(wait_expire_1,)) + wait_expire_1_thread.start() + + data = [] # 10MB in total + for i in range(10): + data.append(("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time_1 if i > 0 or positive else time_2))) # 1MB row + + node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + used_disks = get_used_disks_for_table(node1, name) + assert set(used_disks) == {"jbod1"} + + wait_expire_1_thread.join() + time.sleep(wait_expire_2/2) + + used_disks = get_used_disks_for_table(node1, name) + assert set(used_disks) == {"external" if positive else "jbod1"} + + assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + + finally: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + + +@pytest.mark.parametrize("name,engine", [ + ("mt_test_moves_to_volume_work","MergeTree()"), + ("replicated_mt_test_moves_to_volume_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_volume_work', '1')"), +]) +def test_moves_to_volume_work(started_cluster, name, engine): + try: + node1.query(""" + CREATE TABLE {name} ( + p1 Int64, + s1 String, + d1 DateTime + ) ENGINE = {engine} + ORDER BY tuple() + PARTITION BY p1 + TTL d1 TO VOLUME 'external' + SETTINGS storage_policy='jbods_with_external' + """.format(name=name, engine=engine)) + + wait_expire_1 = 10 + time_1 = time.time() + wait_expire_1 + + wait_expire_1_thread = threading.Thread(target=time.sleep, args=(wait_expire_1,)) + wait_expire_1_thread.start() + + for p in range(2): + data = [] # 10MB in total + for i in range(5): + data.append((str(p), "'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time_1))) # 1MB row + + node1.query("INSERT INTO {} (p1, s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + + used_disks = get_used_disks_for_table(node1, name) + assert set(used_disks) == {'jbod1', 'jbod2'} + + wait_expire_1_thread.join() + time.sleep(1) + + used_disks = get_used_disks_for_table(node1, name) + assert set(used_disks) == {"external"} + + assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + + finally: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + + +@pytest.mark.parametrize("name,engine,positive", [ + ("mt_test_inserts_to_volume_do_not_work","MergeTree()",0), + ("replicated_mt_test_inserts_to_volume_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_volume_do_not_work', '1')",0), + ("mt_test_inserts_to_volume_work","MergeTree()",1), + ("replicated_mt_test_inserts_to_volume_work","ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_volume_work', '1')",1), +]) +def test_inserts_to_volume_work(started_cluster, name, engine, positive): + try: + node1.query(""" + CREATE TABLE {name} ( + p1 Int64, + s1 String, + d1 DateTime + ) ENGINE = {engine} + ORDER BY tuple() + PARTITION BY p1 + TTL d1 TO VOLUME 'external' + SETTINGS storage_policy='small_jbod_with_external' + """.format(name=name, engine=engine)) + + node1.query("SYSTEM STOP MOVES {name}".format(name=name)) + + for p in range(2): + data = [] # 20MB in total + for i in range(10): + data.append((str(p), "'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time.time()-1 if i > 0 or positive else time.time()+300))) # 1MB row + + node1.query("INSERT INTO {} (p1, s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + + used_disks = get_used_disks_for_table(node1, name) + assert set(used_disks) == {"external" if positive else "jbod1"} + + assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "20" + + finally: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + + +@pytest.mark.parametrize("name,engine", [ + ("mt_test_moves_to_disk_eventually_work","MergeTree()"), + ("replicated_mt_test_moves_to_disk_eventually_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_disk_eventually_work', '1')"), +]) +def test_moves_to_disk_eventually_work(started_cluster, name, engine): + try: + name_temp = name + "_temp" + + node1.query(""" + CREATE TABLE {name} ( + s1 String + ) ENGINE = MergeTree() + ORDER BY tuple() + SETTINGS storage_policy='only_jbod2' + """.format(name=name_temp)) + + data = [] # 35MB in total + for i in range(35): + data.append(get_random_string(1024 * 1024)) # 1MB row + + node1.query("INSERT INTO {} VALUES {}".format(name_temp, ",".join(["('" + x + "')" for x in data]))) + used_disks = get_used_disks_for_table(node1, name_temp) + assert set(used_disks) == {"jbod2"} + + node1.query(""" + CREATE TABLE {name} ( + s1 String, + d1 DateTime + ) ENGINE = {engine} + ORDER BY tuple() + TTL d1 TO DISK 'jbod2' + SETTINGS storage_policy='jbod1_with_jbod2' + """.format(name=name, engine=engine)) + + data = [] # 10MB in total + for i in range(10): + data.append(("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time.time()-1))) # 1MB row + + node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + used_disks = get_used_disks_for_table(node1, name) + assert set(used_disks) == {"jbod1"} + + node1.query("DROP TABLE {}".format(name_temp)) + + time.sleep(2) + used_disks = get_used_disks_for_table(node1, name) + assert set(used_disks) == {"jbod2"} + + assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + + finally: + node1.query("DROP TABLE IF EXISTS {}".format(name_temp)) + node1.query("DROP TABLE IF EXISTS {}".format(name)) + + +@pytest.mark.parametrize("name,engine,positive", [ + ("mt_test_merges_to_disk_do_not_work","MergeTree()",0), + ("replicated_mt_test_merges_to_disk_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_merges_to_disk_do_not_work', '1')",0), + ("mt_test_merges_to_disk_work","MergeTree()",1), + ("replicated_mt_test_merges_to_disk_work","ReplicatedMergeTree('/clickhouse/replicated_test_merges_to_disk_work', '1')",1), +]) +def test_merges_to_disk_work(started_cluster, name, engine, positive): + try: + node1.query(""" + CREATE TABLE {name} ( + s1 String, + d1 DateTime + ) ENGINE = {engine} + ORDER BY tuple() + TTL d1 TO DISK 'external' + SETTINGS storage_policy='small_jbod_with_external' + """.format(name=name, engine=engine)) + + node1.query("SYSTEM STOP MERGES {}".format(name)) + node1.query("SYSTEM STOP MOVES {}".format(name)) + + wait_expire_1 = 10 + wait_expire_2 = 4 + time_1 = time.time() + wait_expire_1 + time_2 = time.time() + wait_expire_1 + wait_expire_2 + + wait_expire_1_thread = threading.Thread(target=time.sleep, args=(wait_expire_1,)) + wait_expire_1_thread.start() + + for _ in range(2): + data = [] # 16MB in total + for i in range(8): + data.append(("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time_1 if i > 0 or positive else time_2))) # 1MB row + + node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + + used_disks = get_used_disks_for_table(node1, name) + assert set(used_disks) == {"jbod1"} + assert "2" == node1.query("SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() + + wait_expire_1_thread.join() + time.sleep(wait_expire_2/2) + + node1.query("SYSTEM START MERGES {}".format(name)) + node1.query("OPTIMIZE TABLE {}".format(name)) + + time.sleep(1) + used_disks = get_used_disks_for_table(node1, name) + assert set(used_disks) == {"external" if positive else "jbod1"} + assert "1" == node1.query("SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() + + assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "16" + + finally: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + + +@pytest.mark.parametrize("name,engine", [ + ("mt_test_merges_with_full_disk_work","MergeTree()"), + ("replicated_mt_test_merges_with_full_disk_work","ReplicatedMergeTree('/clickhouse/replicated_test_merges_with_full_disk_work', '1')"), +]) +def test_merges_with_full_disk_work(started_cluster, name, engine): + try: + name_temp = name + "_temp" + + node1.query(""" + CREATE TABLE {name} ( + s1 String + ) ENGINE = MergeTree() + ORDER BY tuple() + SETTINGS storage_policy='only_jbod2' + """.format(name=name_temp)) + + data = [] # 35MB in total + for i in range(35): + data.append(get_random_string(1024 * 1024)) # 1MB row + + node1.query("INSERT INTO {} VALUES {}".format(name_temp, ",".join(["('" + x + "')" for x in data]))) + used_disks = get_used_disks_for_table(node1, name_temp) + assert set(used_disks) == {"jbod2"} + + node1.query(""" + CREATE TABLE {name} ( + s1 String, + d1 DateTime + ) ENGINE = {engine} + ORDER BY tuple() + TTL d1 TO DISK 'jbod2' + SETTINGS storage_policy='jbod1_with_jbod2' + """.format(name=name, engine=engine)) + + wait_expire_1 = 10 + time_1 = time.time() + wait_expire_1 + + wait_expire_1_thread = threading.Thread(target=time.sleep, args=(wait_expire_1,)) + wait_expire_1_thread.start() + + for _ in range(2): + data = [] # 12MB in total + for i in range(6): + data.append(("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time_1))) # 1MB row + node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + + used_disks = get_used_disks_for_table(node1, name) + assert set(used_disks) == {"jbod1"} + assert "2" == node1.query("SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() + + wait_expire_1_thread.join() + + node1.query("OPTIMIZE TABLE {}".format(name)) + time.sleep(1) + + used_disks = get_used_disks_for_table(node1, name) + assert set(used_disks) == {"jbod1"} # Merged to the same disk against the rule. + assert "1" == node1.query("SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() + + assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "12" + + finally: + node1.query("DROP TABLE IF EXISTS {}".format(name_temp)) + node1.query("DROP TABLE IF EXISTS {}".format(name)) + + +@pytest.mark.parametrize("name,engine,positive", [ + ("mt_test_moves_after_merges_do_not_work","MergeTree()",0), + ("replicated_mt_test_moves_after_merges_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_merges_do_not_work', '1')",0), + ("mt_test_moves_after_merges_work","MergeTree()",1), + ("replicated_mt_test_moves_after_merges_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_merges_work', '1')",1), +]) +def test_moves_after_merges_work(started_cluster, name, engine, positive): + try: + node1.query(""" + CREATE TABLE {name} ( + s1 String, + d1 DateTime + ) ENGINE = {engine} + ORDER BY tuple() + TTL d1 TO DISK 'external' + SETTINGS storage_policy='small_jbod_with_external' + """.format(name=name, engine=engine)) + + wait_expire_1 = 10 + wait_expire_2 = 4 + time_1 = time.time() + wait_expire_1 + time_2 = time.time() + wait_expire_1 + wait_expire_2 + + wait_expire_1_thread = threading.Thread(target=time.sleep, args=(wait_expire_1,)) + wait_expire_1_thread.start() + + for _ in range(2): + data = [] # 14MB in total + for i in range(7): + data.append(("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time_1 if i > 0 or positive else time_2))) # 1MB row + + node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + + node1.query("OPTIMIZE TABLE {}".format(name)) + time.sleep(1) + + used_disks = get_used_disks_for_table(node1, name) + assert set(used_disks) == {"jbod1"} + assert "1" == node1.query("SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() + + wait_expire_1_thread.join() + time.sleep(wait_expire_2/2) + + used_disks = get_used_disks_for_table(node1, name) + assert set(used_disks) == {"external" if positive else "jbod1"} + + assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "14" + + finally: + node1.query("DROP TABLE IF EXISTS {}".format(name)) diff --git a/dbms/tests/integration/test_ttl_replicated/test.py b/dbms/tests/integration/test_ttl_replicated/test.py index f028b8fdbe5..ae4fa8404ac 100644 --- a/dbms/tests/integration/test_ttl_replicated/test.py +++ b/dbms/tests/integration/test_ttl_replicated/test.py @@ -1,6 +1,7 @@ import time import pytest +import helpers.client as client from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV @@ -9,7 +10,7 @@ node1 = cluster.add_instance('node1', with_zookeeper=True) node2 = cluster.add_instance('node2', with_zookeeper=True) @pytest.fixture(scope="module") -def start_cluster(): +def started_cluster(): try: cluster.start() @@ -25,7 +26,7 @@ def drop_table(nodes, table_name): for node in nodes: node.query("DROP TABLE IF EXISTS {}".format(table_name)) -def test_ttl_columns(start_cluster): +def test_ttl_columns(started_cluster): drop_table([node1, node2], "test_ttl") for node in [node1, node2]: node.query( @@ -43,8 +44,12 @@ def test_ttl_columns(start_cluster): expected = "1\t0\t0\n2\t0\t0\n" assert TSV(node1.query("SELECT id, a, b FROM test_ttl ORDER BY id")) == TSV(expected) assert TSV(node2.query("SELECT id, a, b FROM test_ttl ORDER BY id")) == TSV(expected) - -def test_ttl_table(start_cluster): + +@pytest.mark.parametrize("delete_suffix", [ + "", + "DELETE", +]) +def test_ttl_table(started_cluster, delete_suffix): drop_table([node1, node2], "test_ttl") for node in [node1, node2]: node.query( @@ -52,8 +57,8 @@ def test_ttl_table(start_cluster): CREATE TABLE test_ttl(date DateTime, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) - TTL date + INTERVAL 1 DAY SETTINGS merge_with_ttl_timeout=0; - '''.format(replica=node.name)) + TTL date + INTERVAL 1 DAY {delete_suffix} SETTINGS merge_with_ttl_timeout=0; + '''.format(replica=node.name, delete_suffix=delete_suffix)) node1.query("INSERT INTO test_ttl VALUES (toDateTime('2000-10-10 00:00:00'), 1)") node1.query("INSERT INTO test_ttl VALUES (toDateTime('2000-10-11 10:00:00'), 2)") @@ -62,4 +67,18 @@ def test_ttl_table(start_cluster): assert TSV(node1.query("SELECT * FROM test_ttl")) == TSV("") assert TSV(node2.query("SELECT * FROM test_ttl")) == TSV("") - + +def test_ttl_double_delete_rule_returns_error(started_cluster): + drop_table([node1, node2], "test_ttl") + try: + node1.query(''' + CREATE TABLE test_ttl(date DateTime, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl', '{replica}') + ORDER BY id PARTITION BY toDayOfMonth(date) + TTL date + INTERVAL 1 DAY, date + INTERVAL 2 DAY SETTINGS merge_with_ttl_timeout=0; + '''.format(replica=node1.name)) + assert False + except client.QueryRuntimeException: + pass + except: + assert False diff --git a/dbms/tests/integration/test_user_zero_database_access/__init__.py b/dbms/tests/integration/test_user_zero_database_access/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_user_zero_database_access/configs/config.xml b/dbms/tests/integration/test_user_zero_database_access/configs/config.xml new file mode 100644 index 00000000000..00ca74de8a2 --- /dev/null +++ b/dbms/tests/integration/test_user_zero_database_access/configs/config.xml @@ -0,0 +1,31 @@ + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + 9000 + 127.0.0.1 + + + + true + none + + AcceptCertificateHandler + + + + + 500 + 5368709120 + ./clickhouse/ + users.xml + + 1 + 1 + \ No newline at end of file diff --git a/dbms/tests/integration/test_user_zero_database_access/configs/users.xml b/dbms/tests/integration/test_user_zero_database_access/configs/users.xml new file mode 100644 index 00000000000..f800318af16 --- /dev/null +++ b/dbms/tests/integration/test_user_zero_database_access/configs/users.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + ::/0 + + default + default + + + + + + ::/0 + + default + default + + + + + + + ::/0 + + default + default + + test + db1 + + + + + + + + + \ No newline at end of file diff --git a/dbms/tests/integration/test_user_zero_database_access/test_user_zero_database_access.py b/dbms/tests/integration/test_user_zero_database_access/test_user_zero_database_access.py new file mode 100644 index 00000000000..f3d57e2e174 --- /dev/null +++ b/dbms/tests/integration/test_user_zero_database_access/test_user_zero_database_access.py @@ -0,0 +1,64 @@ +import time +import pytest + +from helpers.cluster import ClickHouseCluster + + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance('node', config_dir="configs") + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + node.query("CREATE DATABASE test;") + yield cluster + finally: + cluster.shutdown() + + +def test_user_zero_database_access(start_cluster): + try: + node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --user 'no_access' --query 'DROP DATABASE test'"], user='root') + assert False, "user with no access rights dropped database test" + except AssertionError: + raise + except Exception as ex: + print ex + + try: + node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --user 'has_access' --query 'DROP DATABASE test'"], user='root') + except Exception as ex: + assert False, "user with access rights can't drop database test" + + try: + node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --user 'has_access' --query 'CREATE DATABASE test'"], user='root') + except Exception as ex: + assert False, "user with access rights can't create database test" + + try: + node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --user 'no_access' --query 'CREATE DATABASE test2'"], user='root') + assert False, "user with no access rights created database test2" + except AssertionError: + raise + except Exception as ex: + print ex + + try: + node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --user 'has_access' --query 'CREATE DATABASE test2'"], user='root') + assert False, "user with limited access rights created database test2 which is outside of his scope of rights" + except AssertionError: + raise + except Exception as ex: + print ex + + try: + node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --user 'default' --query 'CREATE DATABASE test2'"], user='root') + except Exception as ex: + assert False, "user with full access rights can't create database test2" + + try: + node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --user 'default' --query 'DROP DATABASE test2'"], user='root') + except Exception as ex: + assert False, "user with full access rights can't drop database test2" \ No newline at end of file diff --git a/dbms/tests/msan_suppressions.txt b/dbms/tests/msan_suppressions.txt new file mode 100644 index 00000000000..24d77231c4b --- /dev/null +++ b/dbms/tests/msan_suppressions.txt @@ -0,0 +1,2 @@ +# https://github.com/google/oss-fuzz/issues/1099 +fun:__gxx_personality_* diff --git a/dbms/tests/performance/README.md b/dbms/tests/performance/README.md index 0a78fe481b2..ecda08a80b1 100644 --- a/dbms/tests/performance/README.md +++ b/dbms/tests/performance/README.md @@ -22,6 +22,8 @@ You can use `substitions`, `create`, `fill` and `drop` queries to prepare test. Take into account, that these tests will run in CI which consists of 56-cores and 512 RAM machines. Queries will be executed much faster than on local laptop. +If your test continued more than 10 minutes, please, add tag `long` to have an opportunity to run all tests and skip long ones. + ### How to run performance test You have to run clickhouse-server and after you can start testing: diff --git a/dbms/tests/performance/and_function.xml b/dbms/tests/performance/and_function.xml new file mode 100644 index 00000000000..08fd07ea7e5 --- /dev/null +++ b/dbms/tests/performance/and_function.xml @@ -0,0 +1,22 @@ + + loop + + + + 3 + 10000 + + + 5 + 60000 + + + + + + + + select count() from numbers(10000000) where number != 96594 AND number != 18511 AND number != 98085 AND number != 84177 AND number != 70314 AND number != 28083 AND number != 54202 AND number != 66522 AND number != 66939 AND number != 99469 AND number != 65776 AND number != 22876 AND number != 42151 AND number != 19924 AND number != 66681 AND number != 63022 AND number != 17487 AND number != 83914 AND number != 59754 AND number != 968 AND number != 73334 AND number != 68569 AND number != 49853 AND number != 33155 AND number != 31777 AND number != 99698 AND number != 26708 AND number != 76409 AND number != 42191 AND number != 55397 AND number != 25724 AND number != 39170 AND number != 22728 AND number != 98238 AND number != 86052 AND number != 12756 AND number != 13948 AND number != 57774 AND number != 82511 AND number != 11337 AND number != 23506 AND number != 11875 AND number != 58536 AND number != 56919 AND number != 25986 AND number != 80710 AND number != 61797 AND number != 99244 AND number != 11665 AND number != 15758 AND number != 82899 AND number != 63150 AND number != 7198 AND number != 40071 AND number != 46310 AND number != 78488 AND number != 9273 AND number != 91878 AND number != 57904 AND number != 53941 AND number != 75675 AND number != 12093 AND number != 50090 AND number != 59675 AND number != 41632 AND number != 81448 AND number != 46821 AND number != 51919 AND number != 49028 AND number != 71059 AND number != 15673 AND number != 6132 AND number != 15473 AND number != 32527 AND number != 63842 AND number != 33121 AND number != 53271 AND number != 86033 AND number != 96807 AND number != 4791 AND number != 80089 AND number != 51616 AND number != 46311 AND number != 82844 AND number != 59353 AND number != 63538 AND number != 64857 AND number != 58471 AND number != 29870 AND number != 80209 AND number != 61000 AND number != 75991 AND number != 44506 AND number != 11283 AND number != 6335 AND number != 73502 AND number != 22354 AND number != 72816 AND number != 66399 AND number != 61703 + + select count() from numbers(10000000) where number != 96594 AND number != 18511 AND number != 98085 AND number != 84177 AND number != 70314 AND number != 28083 AND number != 54202 AND number != 66522 AND number != 66939 AND number != 99469 + diff --git a/dbms/tests/performance/array_fill.xml b/dbms/tests/performance/array_fill.xml new file mode 100644 index 00000000000..c4c0955dfc6 --- /dev/null +++ b/dbms/tests/performance/array_fill.xml @@ -0,0 +1,20 @@ + + once + + + + 10000 + + + + + + + + SELECT arraySlice(arrayFill(x -> ((x % 2) >= 0), range(100000000)), 1, 10) + SELECT arraySlice(arrayFill(x -> (((x.1) % 2) >= 0), arrayMap(x -> (x, toString(x)), range(100000000))), 1, 10) + SELECT arraySlice(arrayFill(x -> ((x % 2) >= 2), range(100000000)), 1, 10) + SELECT arraySlice(arrayFill(x -> (((x.1) % 2) >= 2), arrayMap(x -> (x, toString(x)), range(100000000))), 1, 10) + SELECT arraySlice(arrayFill(x -> ((x % 2) = 0), range(100000000)), 1, 10) + SELECT arraySlice(arrayFill(x -> (((x.1) % 2) = 0), arrayMap(x -> (x, toString(x)), range(100000000))), 1, 10) + diff --git a/dbms/tests/performance/collations.xml b/dbms/tests/performance/collations.xml new file mode 100644 index 00000000000..9bc48d76bce --- /dev/null +++ b/dbms/tests/performance/collations.xml @@ -0,0 +1,25 @@ + + loop + + + + 5 + 10000 + + + 100 + 60000 + + + + + + + + + test.hits + + + SELECT Title FROM test.hits ORDER BY Title DESC LIMIT 1000, 10 + SELECT Title FROM test.hits ORDER BY Title DESC COLLATE 'tr' LIMIT 1000, 10 + diff --git a/dbms/tests/performance/date_time_64.xml b/dbms/tests/performance/date_time_64.xml new file mode 100644 index 00000000000..60c77ca22f8 --- /dev/null +++ b/dbms/tests/performance/date_time_64.xml @@ -0,0 +1,52 @@ + + loop + + + default.hits_100m_single + + + CREATE TABLE dt (x DateTime) ENGINE = MergeTree ORDER BY tuple() + CREATE TABLE dt64 (x DateTime64(3)) ENGINE = MergeTree ORDER BY tuple() + INSERT INTO dt SELECT EventTime FROM hits_100m_single + INSERT INTO dt64 SELECT toDateTime64(toDecimal64(toUInt64(EventTime), 3) + toDecimal64((rand() % 1000) / 1000, 3), 3) FROM hits_100m_single + + + + 5 + 10000 + + + 100 + 60000 + + + + + + + + + SELECT count() FROM dt where not ignore(x) + SELECT count() FROM dt64 where not ignore(x) + SELECT max(x) FROM dt + SELECT max(x) FROM dt64 + SELECT count() from dt where not ignore(toString(x)) + SELECT count() from dt64 where not ignore(toString(x)) + SELECT count() from dt where not ignore(toStartOfDay(x)) + SELECT count() from dt64 where not ignore(toStartOfDay(x)) + SELECT count() from dt where not ignore(toStartOfWeek(x)) + SELECT count() from dt64 where not ignore(toStartOfWeek(x)) + SELECT count() from dt where not ignore(toRelativeMinuteNum(x)) + SELECT count() from dt64 where not ignore(toRelativeMinuteNum(x)) + SELECT count() from dt where not ignore(addDays(x, 1)) + SELECT count() from dt64 where not ignore(addDays(x, 1)) + + SELECT sum(x = x) FROM dt + SELECT sum(x = x) FROM dt64 + SELECT sum(toDateTime(toString(x)) != x) FROM dt + SELECT sum(toDateTime64(toString(x), 3) != x) FROM dt64 + + + DROP TABLE IF EXISTS dt + DROP TABLE IF EXISTS dt64 + diff --git a/dbms/tests/performance/decimal_aggregates.xml b/dbms/tests/performance/decimal_aggregates.xml new file mode 100644 index 00000000000..f22cb89de36 --- /dev/null +++ b/dbms/tests/performance/decimal_aggregates.xml @@ -0,0 +1,45 @@ + + loop + + CREATE TABLE t (x UInt64, d32 Decimal32(3), d64 Decimal64(4), d128 Decimal128(5)) ENGINE = Memory + INSERT INTO t SELECT number AS x, x AS d32, x AS d64, x d128 FROM numbers(1000000) + DROP TABLE IF EXISTS t + + + + 10 + + + + + + + + SELECT min(d32), max(d32), argMin(x, d32), argMax(x, d32) FROM t + SELECT min(d64), max(d64), argMin(x, d64), argMax(x, d64) FROM t + SELECT min(d128), max(d128), argMin(x, d128), argMax(x, d128) FROM t + + SELECT avg(d32), sum(d32), sumWithOverflow(d32) FROM t + SELECT avg(d64), sum(d64), sumWithOverflow(d64) FROM t + SELECT avg(d128), sum(d128), sumWithOverflow(d128) FROM t + + SELECT uniq(d32), uniqCombined(d32), uniqExact(d32), uniqHLL12(d32) FROM t + SELECT uniq(d64), uniqCombined(d64), uniqExact(d64), uniqHLL12(d64) FROM t + SELECT uniq(d128), uniqCombined(d128), uniqExact(d128), uniqHLL12(d128) FROM t + + SELECT median(d32), medianExact(d32), medianExactWeighted(d32, 2) FROM t + SELECT median(d64), medianExact(d64), medianExactWeighted(d64, 2) FROM t + SELECT median(d128), medianExact(d128), medianExactWeighted(d128, 2) FROM t + + SELECT quantile(d32), quantileExact(d32), quantileExactWeighted(d32, 2) FROM t + SELECT quantile(d64), quantileExact(d64), quantileExactWeighted(d64, 2) FROM t + SELECT quantile(d128), quantileExact(d128), quantileExactWeighted(d128, 2) FROM t + + SELECT quantilesExact(0.1, 0.9)(d32), quantilesExactWeighted(0.1, 0.9)(d32, 2) FROM t + SELECT quantilesExact(0.1, 0.9)(d64), quantilesExactWeighted(0.1, 0.9)(d64, 2) FROM t + SELECT quantilesExact(0.1, 0.9)(d128), quantilesExactWeighted(0.1, 0.9)(d128, 2) FROM t + + SELECT varPop(d32), varSamp(d32), stddevPop(d32) FROM t + SELECT varPop(d64), varSamp(d64), stddevPop(d64) FROM t + SELECT varPop(d128), varSamp(d128), stddevPop(d128) FROM t + diff --git a/dbms/tests/performance/early_constant_folding.xml b/dbms/tests/performance/early_constant_folding.xml new file mode 100644 index 00000000000..04fb4057d17 --- /dev/null +++ b/dbms/tests/performance/early_constant_folding.xml @@ -0,0 +1,23 @@ + + loop + + + + 30000 + + + 5000 + 60000 + + + + + + + + + default.hits_100m_single + + + SELECT count(JavaEnable) FROM default.hits_100m_single WHERE WatchID = 1 OR Title = 'next' OR URL = 'prev' OR OriginalURL = '???' OR 1 + diff --git a/dbms/tests/performance/general_purpose_hashes_on_UUID.xml b/dbms/tests/performance/general_purpose_hashes_on_UUID.xml new file mode 100644 index 00000000000..23e00909bbe --- /dev/null +++ b/dbms/tests/performance/general_purpose_hashes_on_UUID.xml @@ -0,0 +1,54 @@ + + loop + + + + 3 + 10000 + + + 5 + 60000 + + + + + + + + + + hash_func + + cityHash64 + farmHash64 + metroHash64 + murmurHash2_32 + murmurHash2_64 + murmurHash3_32 + murmurHash3_64 + javaHash + hiveHash + xxHash32 + xxHash64 + + + + value + + toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0') + toDecimal128(number, 23) + generateUUIDv4() + + + + table + + numbers(1000000) + numbers_mt(10000000) + + + + + SELECT count() from {table} where not ignore({hash_func}({value})) + diff --git a/dbms/tests/performance/grear_circle_dist.xml b/dbms/tests/performance/great_circle_dist.xml similarity index 57% rename from dbms/tests/performance/grear_circle_dist.xml rename to dbms/tests/performance/great_circle_dist.xml index bb26605bd89..99382543d60 100644 --- a/dbms/tests/performance/grear_circle_dist.xml +++ b/dbms/tests/performance/great_circle_dist.xml @@ -9,7 +9,8 @@ - SELECT count() FROM system.numbers WHERE NOT ignore(greatCircleDistance((rand() % 360) * 1. - 180, (number % 150) * 1.2 - 90, (number % 360) + toFloat64(rand()) / 4294967296 - 180, (rand() % 180) * 1. - 90)) + SELECT count() FROM system.numbers WHERE NOT ignore(greatCircleDistance((rand(1) % 360) * 1. - 180, (number % 150) * 1.2 - 90, (number % 360) + toFloat64(rand(2)) / 4294967296 - 180, (rand(3) % 180) * 1. - 90)) - SELECT count() FROM system.numbers WHERE NOT ignore(greatCircleDistance(55. + toFloat64(rand()) / 4294967296, 37. + toFloat64(rand()) / 4294967296, 55. + toFloat64(rand()) / 4294967296, 37. + toFloat64(rand()) / 4294967296)) + SELECT count() FROM system.numbers WHERE NOT ignore(greatCircleDistance(55. + toFloat64(rand(1)) / 4294967296, 37. + toFloat64(rand(2)) / 4294967296, 55. + toFloat64(rand(3)) / 4294967296, 37. + toFloat64(rand(4)) / 4294967296)) + diff --git a/dbms/tests/performance/information_value.xml b/dbms/tests/performance/information_value.xml new file mode 100644 index 00000000000..63d61f6a432 --- /dev/null +++ b/dbms/tests/performance/information_value.xml @@ -0,0 +1,24 @@ + + loop + + + test.hits + + + + + 10000 + + + 5000 + 20000 + + + + + + + + SELECT categoricalInformationValue(Age < 15, IsMobile) + SELECT categoricalInformationValue(Age < 15, Age >= 15 and Age < 30, Age >= 30 and Age < 45, Age >= 45 and Age < 60, Age >= 60, IsMobile) + diff --git a/dbms/tests/performance/insert_values_with_expressions.xml b/dbms/tests/performance/insert_values_with_expressions.xml index 6693b84d365..66fe2aef18b 100644 --- a/dbms/tests/performance/insert_values_with_expressions.xml +++ b/dbms/tests/performance/insert_values_with_expressions.xml @@ -2,7 +2,6 @@ loop - 1 1 diff --git a/docs/en/data_types/special_data_types/interval.md b/docs/en/data_types/special_data_types/interval.md new file mode 100644 index 00000000000..b0d152e69bd --- /dev/null +++ b/docs/en/data_types/special_data_types/interval.md @@ -0,0 +1,74 @@ +# Interval {#data-type-interval} + +The family of data types representing time and date intervals. The resulting types of the [INTERVAL](../../query_language/operators.md#operator-interval) operator. + +!!! warning "Warning" + You can't use `Interval` data types for storing values in tables. + +Structure: + +- Time interval as an unsigned integer value. +- Type of an interval. + +Supported interval types: + +- `SECOND` +- `MINUTE` +- `HOUR` +- `DAY` +- `WEEK` +- `MONTH` +- `QUARTER` +- `YEAR` + +For each interval type, there is a separate data type. For example, the `DAY` interval is expressed as the `IntervalDay` data type: + +```sql +SELECT toTypeName(INTERVAL 4 DAY) +``` +```text +┌─toTypeName(toIntervalDay(4))─┐ +│ IntervalDay │ +└──────────────────────────────┘ +``` + +## Usage Remarks {#data-type-interval-usage-remarks} + +You can use `Interval`-type values in arithmetical operations with [Date](../../data_types/date.md) and [DateTime](../../data_types/datetime.md)-type values. For example, you can add 4 days to the current time: + +```sql +SELECT now() as current_date_time, current_date_time + INTERVAL 4 DAY +``` +```text +┌───current_date_time─┬─plus(now(), toIntervalDay(4))─┐ +│ 2019-10-23 10:58:45 │ 2019-10-27 10:58:45 │ +└─────────────────────┴───────────────────────────────┘ +``` + +Intervals with different types can't be combined. You can't use intervals like `4 DAY 1 HOUR`. Express intervals in units that are smaller or equal to the smallest unit of the interval, for example, the interval `1 day and an hour` interval can be expressed as `25 HOUR` or `90000 SECOND`. + +You can't perform arithmetical operations with `Interval`-type values, but you can add intervals of different types consequently to values in `Date` or `DateTime` data types. For example: + +```sql +SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL 3 HOUR +``` +```text +┌───current_date_time─┬─plus(plus(now(), toIntervalDay(4)), toIntervalHour(3))─┐ +│ 2019-10-23 11:16:28 │ 2019-10-27 14:16:28 │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +``` + +The following query causes an exception: + +```sql +select now() AS current_date_time, current_date_time + (INTERVAL 4 DAY + INTERVAL 3 HOUR) +``` +```text +Received exception from server (version 19.14.1): +Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Wrong argument types for function plus: if one argument is Interval, then another must be Date or DateTime.. +``` + +## See Also + +- [INTERVAL](../../query_language/operators.md#operator-interval) operator +- [toInterval](../../query_language/functions/type_conversion_functions.md#function-tointerval) type convertion functions diff --git a/docs/en/database_engines/mysql.md b/docs/en/database_engines/mysql.md index ba2405de0ea..049d784faae 100644 --- a/docs/en/database_engines/mysql.md +++ b/docs/en/database_engines/mysql.md @@ -6,8 +6,6 @@ The `MySQL` database engine translate queries to the MySQL server so you can per You cannot perform the following queries: -- `ATTACH`/`DETACH` -- `DROP` - `RENAME` - `CREATE TABLE` - `ALTER` diff --git a/docs/en/development/build.md b/docs/en/development/build.md index 9bb9a7c6471..2df8e7eeb25 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -24,15 +24,9 @@ $ ./release The following tutorial is based on the Ubuntu Linux system. With appropriate changes, it should also work on any other Linux distribution. -Only x86_64 with SSE 4.2 is supported. Support for AArch64 is experimental. +Supported platforms: x86_64 and AArch64. Support for Power9 is experimental. -To test for SSE 4.2, do - -```bash -$ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" -``` - -## Install Git and CMake +## Install Git, CMake and Ninja ```bash $ sudo apt-get install git cmake ninja-build @@ -67,7 +61,7 @@ $ export CXX=g++-9 ## Install Required Libraries from Packages ```bash -$ sudo apt-get install libicu-dev libreadline-dev gperf +$ sudo apt-get install libreadline-dev ``` ## Checkout ClickHouse Sources @@ -75,7 +69,7 @@ $ sudo apt-get install libicu-dev libreadline-dev gperf ```bash $ git clone --recursive git@github.com:ClickHouse/ClickHouse.git ``` -or +or ```bash $ git clone --recursive https://github.com/ClickHouse/ClickHouse.git $ cd ClickHouse diff --git a/docs/en/development/build_cross_arm.md b/docs/en/development/build_cross_arm.md new file mode 100644 index 00000000000..68a4596559f --- /dev/null +++ b/docs/en/development/build_cross_arm.md @@ -0,0 +1,36 @@ +# How to Build ClickHouse on Linux for AARCH64 (ARM64) architecture + +This is for the case when you have Linux machine and want to use it to build `clickhouse` binary that will run on another Linux machine with AARCH64 CPU architecture. This is intended for continuous integration checks that run on Linux servers. + +The cross-build for AARCH64 is based on the [Build instructions](build.md), follow them first. + +# Install Clang-8 + +Follow the instructions from https://apt.llvm.org/ for your Ubuntu or Debian setup. +For example, in Ubuntu Bionic you can use the following commands: + +```bash +echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" | sudo tee /etc/apt/sources.list.d/llvm.list +sudo apt-get update +sudo apt-get install clang-8 +``` + +# Install Cross-Compilation Toolset + +```bash +cd ClickHouse +mkdir -p build-aarch64/cmake/toolchain/linux-aarch64 +wget 'https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en' -O gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz +tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -C build-aarch64/cmake/toolchain/linux-aarch64 --strip-components=1 +``` + +# Build ClickHouse + +```bash +cd ClickHouse +mkdir build-arm64 +CC=clang-8 CXX=clang++-8 cmake . -Bbuild-arm64 -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-aarch64.cmake +ninja -C build-arm64 +``` + +The resulting binary will run only on Linux with the AARCH64 CPU architecture. diff --git a/docs/en/development/build_cross_osx.md b/docs/en/development/build_cross_osx.md new file mode 100644 index 00000000000..b6baabbb85d --- /dev/null +++ b/docs/en/development/build_cross_osx.md @@ -0,0 +1,57 @@ +# How to Build ClickHouse on Linux for Mac OS X + +This is for the case when you have Linux machine and want to use it to build `clickhouse` binary that will run on OS X. This is intended for continuous integration checks that run on Linux servers. If you want to build ClickHouse directly on Mac OS X, then proceed with another instruction: https://clickhouse.yandex/docs/en/development/build_osx/ + +The cross-build for Mac OS X is based on the [Build instructions](build.md), follow them first. + +# Install Clang-8 + +Follow the instructions from https://apt.llvm.org/ for your Ubuntu or Debian setup. +For example the commands for Bionic are like: + +```bash +sudo echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" >> /etc/apt/sources.list +sudo apt-get install clang-8 +``` + +# Install Cross-Compilation Toolset + +Let's remember the path where we install `cctools` as ${CCTOOLS} + +```bash +mkdir ${CCTOOLS} + +git clone https://github.com/tpoechtrager/apple-libtapi.git +cd apple-libtapi +INSTALLPREFIX=${CCTOOLS} ./build.sh +./install.sh +cd .. + +git clone https://github.com/tpoechtrager/cctools-port.git +cd cctools-port/cctools +./configure --prefix=${CCTOOLS} --with-libtapi=${CCTOOLS} --target=x86_64-apple-darwin +make install +``` + +Also, we need to download MacOS X SDK into the working tree. + +```bash +cd ClickHouse +wget 'https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz' +mkdir -p build-darwin/cmake/toolchain/darwin-x86_64 +tar xJf MacOSX10.14.sdk.tar.xz -C build-darwin/cmake/toolchain/darwin-x86_64 --strip-components=1 +``` + +# Build ClickHouse + +```bash +cd ClickHouse +mkdir build-osx +CC=clang-8 CXX=clang++-8 cmake . -Bbuild-osx -DCMAKE_TOOLCHAIN_FILE=cmake/darwin/toolchain-x86_64.cmake \ + -DCMAKE_AR:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ar \ + -DCMAKE_RANLIB:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ranlib \ + -DLINKER_NAME=${CCTOOLS}/bin/x86_64-apple-darwin-ld +ninja -C build-osx +``` + +The resulting binary will have Mach-O executable format and can't be run on Linux. diff --git a/docs/en/development/build_osx.md b/docs/en/development/build_osx.md index c6a2be20530..23fe52ddb64 100644 --- a/docs/en/development/build_osx.md +++ b/docs/en/development/build_osx.md @@ -11,7 +11,7 @@ $ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/inst ## Install Required Compilers, Tools, and Libraries ```bash -$ brew install cmake ninja gcc icu4c openssl libtool gettext readline gperf +$ brew install cmake ninja libtool gettext readline ``` ## Checkout ClickHouse Sources @@ -33,7 +33,7 @@ For the latest stable version, switch to the `stable` branch. ```bash $ mkdir build $ cd build -$ cmake .. -DCMAKE_CXX_COMPILER=`which g++-8` -DCMAKE_C_COMPILER=`which gcc-8` +$ cmake .. -DCMAKE_CXX_COMPILER=`which clang++` -DCMAKE_C_COMPILER=`which clang` $ ninja $ cd .. ``` diff --git a/docs/en/development/contrib.md b/docs/en/development/contrib.md index 744917449f4..22bb0242cba 100644 --- a/docs/en/development/contrib.md +++ b/docs/en/development/contrib.md @@ -10,6 +10,7 @@ | double-conversion | [BSD 3-Clause License](https://github.com/google/double-conversion/blob/cf2f0f3d547dc73b4612028a155b80536902ba02/LICENSE) | | FastMemcpy | [MIT](https://github.com/yandex/ClickHouse/blob/master/libs/libmemcpy/impl/LICENSE) | | googletest | [BSD 3-Clause License](https://github.com/google/googletest/blob/master/LICENSE) | +| h3 | [Apache License 2.0](https://github.com/uber/h3/blob/master/LICENSE) | hyperscan | [BSD 3-Clause License](https://github.com/intel/hyperscan/blob/master/LICENSE) | | libbtrie | [BSD 2-Clause License](https://github.com/yandex/ClickHouse/blob/master/contrib/libbtrie/LICENSE) | | libcxxabi | [BSD + MIT](https://github.com/yandex/ClickHouse/blob/master/libs/libglibc-compatibility/libcxxabi/LICENSE.TXT) | diff --git a/docs/en/development/developer_instruction.md b/docs/en/development/developer_instruction.md new file mode 100644 index 00000000000..40eb60fc5da --- /dev/null +++ b/docs/en/development/developer_instruction.md @@ -0,0 +1,277 @@ +Building of ClickHouse is supported on Linux, FreeBSD and Mac OS X. + +# If you use Windows + +If you use Windows, you need to create a virtual machine with Ubuntu. To start working with a virtual machine please install VirtualBox. You can download Ubuntu from the website: https://www.ubuntu.com/#download. Please create a virtual machine from the downloaded image (you should reserve at least 4GB of RAM for it). To run a command line terminal in Ubuntu, please locate a program containing the word "terminal" in its name (gnome-terminal, konsole etc.) or just press Ctrl+Alt+T. + + +# Creating a repository on GitHub + +To start working with ClickHouse repository you will need a GitHub account. + +You probably already have one, but if you don't, please register at https://github.com. In case you do not have SSH keys, you should generate them and then upload them on GitHub. It is required for sending over your patches. It is also possible to use the same SSH keys that you use with any other SSH servers - probably you already have those. + +Create a fork of ClickHouse repository. To do that please click on the "fork" button in the upper right corner at https://github.com/ClickHouse/ClickHouse. It will fork your own copy of ClickHouse/ClickHouse to your account. + +Development process consists of first committing the intended changes into your fork of ClickHouse and then creating a "pull request" for these changes to be accepted into the main repository (ClickHouse/ClickHouse). + +To work with git repositories, please install `git`. + +To do that in Ubuntu you would run in the command line terminal: +``` +sudo apt update +sudo apt install git +``` + +A brief manual on using Git can be found here: https://services.github.com/on-demand/downloads/github-git-cheat-sheet.pdf. +For a detailed manual on Git see: https://git-scm.com/book/ru/v2. + + +# Cloning a repository to your development machine + +Next, you need to download the source files onto your working machine. This is called "to clone a repository" because it creates a local copy of the repository on your working machine. + +In the command line terminal run: +``` +git clone --recursive git@guthub.com:your_github_username/ClickHouse.git +cd ClickHouse +``` +Note: please, substitute *your_github_username* with what is appropriate! + +This command will create a directory `ClickHouse` containing the working copy of the project. + +It is important that the path to the working directory contains no whitespaces as it may lead to problems with running the build system. + +Please note that ClickHouse repository uses `submodules`. That is what the references to additional repositories are called (i.e. external libraries on which the project depends). It means that when cloning the repository you need to specify the `--recursive` flag as in the example above. If the repository has been cloned without submodules, to download them you need to run the following: +``` +git submodule init +git submodule update +``` +You can check status with command: `git submodule status`. + +If you get the following error message: +``` +Permission denied (publickey). +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. +``` +It generally means that the SSH keys for connecting to GitHub are missing. These keys are normally located in `~/.ssh`. For SSH keys to be accepted you need to upload them in the settings section of GitHub UI. + +You can also clone the repository via https protocol: +``` +git clone https://github.com/ClickHouse/ClickHouse.git +``` +This however will not let you send your changes to the server. You can still use it temporarily and add the SSH keys later replacing the remote address of the repository with `git remote` command. + +You can also add original ClickHouse repo's address to your local repository to pull updates from there: +``` +git remote add upstream git@github.com:ClickHouse/ClickHouse.git +``` +After successfully running this command you will be able to pull updates from the main ClickHouse repo by running `git pull upstream master`. + + +# Build System + +ClickHouse uses CMake and Ninja for building. + +CMake - a meta-build system that can generate Ninja files (build tasks). +Ninja - a smaller build system with focus on speed used to execute those cmake generated tasks. + +To install on Ubuntu, Debian or Mint run `sudo apt install cmake ninja-build`. + +On CentOS, RedHat run `sudo yum install cmake ninja-build`. + +If you use Arch or Gentoo, you probably know it yourself how to install CMake. + +For installing CMake and Ninja on Mac OS X first install Homebrew and then install everything else via brew: +``` +/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +brew install cmake ninja +``` + +Next, check the version of CMake: `cmake --version`. If it is below 3.3, you should install a newer version from the website: https://cmake.org/download/. + + +# Optional External Libraries + +ClickHouse uses several external libraries for building. Most of them do not need to be installed separately as they are built together with ClickHouse from the sources located in the submodules. You can check the list in `contrib`. + +There is one library that is not built from sources but is supplied by the system: Readline, and thus is recommended to be installed. + +Ubuntu: `sudo apt install libreadline-dev` + +Mac OS X: `brew install readline` + +However, these libraries are optional and ClickHouse can well be built without them. ICU is used for support of `COLLATE` in `ORDER BY` (i.e. for sorting in turkish alphabet). Readline is used for more convenient command input in clickhouse-client. + + +# C++ Compiler + +Compilers GCC starting from version 9 and Clang version 8 or above are supported for building ClickHouse. + +Official Yandex builds currently use GCC because it generates machine code of slightly better performance (yielding a difference of up to several percent according to our benchmarks). And Clang is more convenient for development usually. Though, our continuous integration (CI) platform runs checks for about a dozen of build combinations. + +To install GCC on Ubuntu run: `sudo apt install gcc g++` + +Check the version of gcc: `gcc --version`. If it is below 9, then follow the instruction here: https://clickhouse.yandex/docs/en/development/build/#install-gcc-9. + +Mac OS X build is supported only for Clang. Just run `brew install llvm` + +If you decide to use Clang, you can also install `libc++` and `lld`, if you know what it is. Using `ccache` is also recommended. + + +# The Building process + +Now that you are ready to build ClickHouse we recommend you to create a separate directory `build` inside `ClickHouse` that will contain all of the build artefacts: +``` +mkdir build +cd build +``` +You can have several different directories (build_release, build_debug, etc.) for different types of build. + +While inside the `build` directory, configure your build by running CMake. Before the first run you need to define environment variables that specify compiler (version 9 gcc compiler in this example). + +Linux: +``` +export CC=gcc-9 CXX=g++-9 +cmake .. +``` + +Mac OS X: +``` +export CC=clang CXX=clang++ +cmake .. +``` + +The `CC` variable specifies the compiler for C (short for C Compiler), and `CXX` variable instructs which C++ compiler is to be used for building. + +For a faster build you can resort to the `debug` build type - a build with no optimizations. For that supply the following parameter `-D CMAKE_BUILD_TYPE=Debug`: +``` +cmake -D CMAKE_BUILD_TYPE=Debug .. +``` +You can change the type of build by running this command in the `build` directory. + +Run ninja to build: +``` +ninja clickhouse-server clickhouse-client +``` +Only the required binaries are going to be built in this example. + +If you require to build all the binaries (utilities and tests), you should run ninja with no parameters: +``` +ninja +``` + +Full build requires about 30GB of free disk space or 15GB to build the main binaries. + +When large amount of RAM is available on build machine you should limit the number of build tasks run in parallel with `-j` param: +``` +ninja -j 1 clickhouse-server clickhouse-client +``` +On machines with 4GB of RAM it is recommended to specify 1, for 8GB of RAM `-j 2` is recommended. + +If you get the message: `ninja: error: loading 'build.ninja': No such file or directory`, it means that generating a build configuration has failed and you need to inspect the message above. + +Upon successful start of the building process you'll see the build progress - the number of processed tasks and the total number of tasks. + +While building messages about protobuf files in libhdfs2 library like `libprotobuf WARNING` may show up. They affect nothing and are safe to be ignored. + +Upon successful build you get an executable file `ClickHouse//dbms/programs/clickhouse`: +``` +ls -l dbms/programs/clickhouse +``` + + +# Running the built executable of ClickHouse + +To run the server under the current user you need to navigate to `ClickHouse/dbms/programs/server/` (located outside of `build`) and run: + +``` +../../../build/dbms/programs/clickhouse server +``` + +In this case ClickHouse will use config files located in the current directory. You can run `clickhouse server` from any directory specifying the path to a config file as a command line parameter `--config-file`. + +To connect to ClickHouse with clickhouse-client in another terminal navigate to `ClickHouse/build/dbms/programs/` and run `clickhouse client`. + +If you get `Connection refused` message on Mac OS X or FreeBSD, try specifying host address 127.0.0.1: +``` +clickhouse client --host 127.0.0.1 +``` + +You can replace production version of ClickHouse binary installed in your system with your custom built ClickHouse binary. To do that install ClickHouse on your machine following the instructions from the official website. Next, run the following: +``` +sudo service clickhouse-server stop +sudo cp ClickHouse/build/dbms/programs/clickhouse /usr/bin/ +sudo service clickhouse-server start +``` + +Note that `clickhouse-client`, `clickhouse-server` and others are symlinks to the commonly shared `clickhouse` binary. + +You can also run your custom built ClickHouse binary with the config file from the ClickHouse package installed on your system: +``` +sudo service clickhouse-server stop +sudo -u clickhouse ClickHouse/build/dbms/programs/clickhouse server --config-file /etc/clickhouse-server/config.xml +``` + + +# IDE (Integrated Development Environment) + +If you do not know which IDE to use, we recommend that you use CLion. CLion is a commercial software, but it offers 30 day free trial period. It is also free of charge for students. CLion can be used both on Linux and on Mac OS X. + +KDevelop and QTCreator are another great alternatives of an IDE for developing ClickHouse. KDevelop comes in as a very handy IDE although unstable. If KDevelop crashes after a while upon opening project, you should click "Stop All" button as soon as it has opened the list of project's files. After doing so KDevelop should be fine to work with. + +As simple code editors you can use Sublime Text or Visual Studio Code, or Kate (all of which are available on Linux). + +Just in case, it is worth mentioning that CLion creates `build` path on its own, it also on its own selects `debug` for build type, for configuration it uses a version of CMake that is defined in CLion and not the one installed by you, and finally CLion will use `make` to run build tasks instead of `ninja`. This is a normal behaviour, just keep that in mind to avoid confusion. + + +# Writing Code + +The description of ClickHouse architecture can be found here: https://clickhouse.yandex/docs/en/development/architecture/ + +The Code Style Guide: https://clickhouse.yandex/docs/en/development/style/ + +Writing tests: https://clickhouse.yandex/docs/en/development/tests/ + +List of tasks: https://github.com/yandex/ClickHouse/blob/master/dbms/tests/instructions/easy_tasks_sorted_en.md + + +# Test Data + +Developing ClickHouse often requires loading realistic datasets. It is particularly important for performance testing. We have a specially prepared set of anonymized data from Yandex.Metrica. It requires additionally some 3GB of free disk space. Note that this data is not required to accomplish most of development tasks. + +``` +sudo apt install wget xz-utils + +wget https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz +wget https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz + +xz -v -d hits_v1.tsv.xz +xz -v -d visits_v1.tsv.xz + +clickhouse-client + +CREATE TABLE test.hits ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree PARTITION BY toYYYYMM(EventDate) SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID), EventTime); + +CREATE TABLE test.visits ( CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8, VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32, Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String, EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32, SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32, SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16, UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16, FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8, Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), Params Array(String), `Goals.ID` Array(UInt32), `Goals.Serial` Array(UInt32), `Goals.EventTime` Array(DateTime), `Goals.Price` Array(Int64), `Goals.OrderID` Array(String), `Goals.CurrencyID` Array(UInt32), WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64, ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32, ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32, ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32, ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16, ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime, PredLastVisit Date, LastVisit Date, TotalVisits UInt32, `TraficSource.ID` Array(Int8), `TraficSource.SearchEngineID` Array(UInt16), `TraficSource.AdvEngineID` Array(UInt8), `TraficSource.PlaceID` Array(UInt16), `TraficSource.SocialSourceNetworkID` Array(UInt8), `TraficSource.Domain` Array(String), `TraficSource.SearchPhrase` Array(String), `TraficSource.SocialSourcePage` Array(String), Attendance FixedString(16), CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64, StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64, OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64, UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), `Market.Type` Array(UInt8), `Market.GoalID` Array(UInt32), `Market.OrderID` Array(String), `Market.OrderPrice` Array(Int64), `Market.PP` Array(UInt32), `Market.DirectPlaceID` Array(UInt32), `Market.DirectOrderID` Array(UInt32), `Market.DirectBannerID` Array(UInt32), `Market.GoodID` Array(String), `Market.GoodName` Array(String), `Market.GoodQuantity` Array(Int32), `Market.GoodPrice` Array(Int64), IslandID FixedString(16)) ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) SAMPLE BY intHash32(UserID) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID); + +clickhouse-client --max_insert_block_size 100000 --query "INSERT INTO test.hits FORMAT TSV" < hits_v1.tsv +clickhouse-client --max_insert_block_size 100000 --query "INSERT INTO test.visits FORMAT TSV" < visits_v1.tsv +``` + + + +# Creating Pull Request + +Navigate to your fork repository in GitHub's UI. If you have been developing in a branch, you need to select that branch. There will be a "Pull request" button located on the screen. In essence this means "create a request for accepting my changes into the main repository". + +A pull request can be created even if the work is not completed yet. In this case please put the word "WIP" (work in progress) at the beginning of the title, it can be changed later. This is useful for cooperative reviewing and discussion of changes as well as for running all of the available tests. It is important that you provide a brief description of your changes, it will later be used for generating realease changelogs. + +Testing will commence as soon as Yandex employees label your PR with a tag "can be tested". The results of some first checks (e.g. code style) will come in within several minutes. Build check results will arrive within a half an hour. And the main set of tests will report itself within an hour. + +The system will prepare ClickHouse binary builds for your pull request individually. To retrieve these builds click the "Details" link next to "ClickHouse build check" entry in the list of checks. There you will find direct links to the built .deb packages of ClickHouse which you can deploy even on your production servers (if you have no fear). + +Most probably some of the builds will fail at first times. This is due to the fact that we check builds both with gcc as well as with clang, with almost all of existing warnings (always with the `-Werror` flag) enabled for clang. On that same page you can find all of the build logs so that you do not have to build ClickHouse in all of the possible ways. diff --git a/docs/en/getting_started/example_datasets/metrica.md b/docs/en/getting_started/example_datasets/metrica.md index 34d4e0c9d75..d89fe54f4eb 100644 --- a/docs/en/getting_started/example_datasets/metrica.md +++ b/docs/en/getting_started/example_datasets/metrica.md @@ -1,51 +1,62 @@ # Anonymized Yandex.Metrica Data -Dataset consists of two tables containing anonymized data about hits (`hits_v1`) and visits (`visits_v1`) of Yandex.Metrica. Each of the tables can be downloaded as a compressed `tsv.xz` file or as prepared partitions. In addition to that, an extended version of the `hits` table containing 100 million rows is available as [TSV](https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_100m_obfuscated_v1.tsv.xz) and as [prepared partitions](https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits_100m_obfuscated_v1.tar.xz). +Dataset consists of two tables containing anonymized data about hits (`hits_v1`) and visits (`visits_v1`) of Yandex.Metrica. You can read more about Yandex.Metrica in [ClickHouse history](../../introduction/history.md) section. + +The dataset consists of two tables, either of them can be downloaded as a compressed `tsv.xz` file or as prepared partitions. In addition to that, an extended version of the `hits` table containing 100 million rows is available as TSV at and as prepared partitions at . ## Obtaining Tables from Prepared Partitions -**Download and import hits:** -```bash -$ curl -O https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits_v1.tar -$ tar xvf hits_v1.tar -C /var/lib/clickhouse # path to ClickHouse data directory -$ # check permissions on unpacked data, fix if required -$ sudo service clickhouse-server restart -$ clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" + +Download and import hits table: + +``` bash +curl -O https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits_v1.tar +tar xvf hits_v1.tar -C /var/lib/clickhouse # path to ClickHouse data directory +# check permissions on unpacked data, fix if required +sudo service clickhouse-server restart +clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" ``` -**Download and import visits:** -```bash -$ curl -O https://clickhouse-datasets.s3.yandex.net/visits/partitions/visits_v1.tar -$ tar xvf visits_v1.tar -C /var/lib/clickhouse # path to ClickHouse data directory -$ # check permissions on unpacked data, fix if required -$ sudo service clickhouse-server restart -$ clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" +Download and import visits: + +``` bash +curl -O https://clickhouse-datasets.s3.yandex.net/visits/partitions/visits_v1.tar +tar xvf visits_v1.tar -C /var/lib/clickhouse # path to ClickHouse data directory +# check permissions on unpacked data, fix if required +sudo service clickhouse-server restart +clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" ``` -## Obtaining Tables from Compressed tsv-file -**Download and import hits from compressed tsv-file** -```bash -$ curl https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv -$ # now create table -$ clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" -$ clickhouse-client --query "CREATE TABLE datasets.hits_v1 ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" -$ # import data -$ cat hits_v1.tsv | clickhouse-client --query "INSERT INTO datasets.hits_v1 FORMAT TSV" --max_insert_block_size=100000 -$ # optionally you can optimize table -$ clickhouse-client --query "OPTIMIZE TABLE datasets.hits_v1 FINAL" -$ clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" +## Obtaining Tables from Compressed TSV File + +Download and import hits from compressed TSV file: + +``` bash +curl https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv +# now create table +clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" +clickhouse-client --query "CREATE TABLE datasets.hits_v1 ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" +# import data +cat hits_v1.tsv | clickhouse-client --query "INSERT INTO datasets.hits_v1 FORMAT TSV" --max_insert_block_size=100000 +# optionally you can optimize table +clickhouse-client --query "OPTIMIZE TABLE datasets.hits_v1 FINAL" +clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" ``` -**Download and import visits from compressed tsv-file** -```bash -$ curl https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv -$ # now create table -$ clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" -$ clickhouse-client --query "CREATE TABLE datasets.visits_v1 ( CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8, VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32, Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String, EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32, SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32, SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16, UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16, FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8, Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), Params Array(String), Goals Nested(ID UInt32, Serial UInt32, EventTime DateTime, Price Int64, OrderID String, CurrencyID UInt32), WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64, ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32, ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32, ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32, ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16, ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime, PredLastVisit Date, LastVisit Date, TotalVisits UInt32, TraficSource Nested(ID Int8, SearchEngineID UInt16, AdvEngineID UInt8, PlaceID UInt16, SocialSourceNetworkID UInt8, Domain String, SearchPhrase String, SocialSourcePage String), Attendance FixedString(16), CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64, StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64, OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64, UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), Market Nested(Type UInt8, GoalID UInt32, OrderID String, OrderPrice Int64, PP UInt32, DirectPlaceID UInt32, DirectOrderID UInt32, DirectBannerID UInt32, GoodID String, GoodName String, GoodQuantity Int32, GoodPrice Int64), IslandID FixedString(16)) ENGINE = CollapsingMergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192, Sign)" -$ # import data -$ cat visits_v1.tsv | clickhouse-client --query "INSERT INTO datasets.visits_v1 FORMAT TSV" --max_insert_block_size=100000 -$ # optionally you can optimize table -$ clickhouse-client --query "OPTIMIZE TABLE datasets.visits_v1 FINAL" -$ clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" +Download and import visits from compressed tsv-file: + +``` bash +curl https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv +# now create table +clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" +clickhouse-client --query "CREATE TABLE datasets.visits_v1 ( CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8, VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32, Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String, EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32, SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32, SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16, UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16, FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8, Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), Params Array(String), Goals Nested(ID UInt32, Serial UInt32, EventTime DateTime, Price Int64, OrderID String, CurrencyID UInt32), WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64, ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32, ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32, ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32, ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16, ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime, PredLastVisit Date, LastVisit Date, TotalVisits UInt32, TraficSource Nested(ID Int8, SearchEngineID UInt16, AdvEngineID UInt8, PlaceID UInt16, SocialSourceNetworkID UInt8, Domain String, SearchPhrase String, SocialSourcePage String), Attendance FixedString(16), CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64, StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64, OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64, UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), Market Nested(Type UInt8, GoalID UInt32, OrderID String, OrderPrice Int64, PP UInt32, DirectPlaceID UInt32, DirectOrderID UInt32, DirectBannerID UInt32, GoodID String, GoodName String, GoodQuantity Int32, GoodPrice Int64), IslandID FixedString(16)) ENGINE = CollapsingMergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192, Sign)" +# import data +cat visits_v1.tsv | clickhouse-client --query "INSERT INTO datasets.visits_v1 FORMAT TSV" --max_insert_block_size=100000 +# optionally you can optimize table +clickhouse-client --query "OPTIMIZE TABLE datasets.visits_v1 FINAL" +clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" ``` -## Queries -Examples of queries to these tables (they are named `test.hits` and `test.visits`) can be found among [stateful tests](https://github.com/ClickHouse/ClickHouse/tree/master/dbms/tests/queries/1_stateful) and in some [performance tests](https://github.com/ClickHouse/ClickHouse/tree/master/dbms/tests/performance/test_hits) of ClickHouse. +## Example Queries + +[ClickHouse tutorial](../../getting_started/tutorial.md) is based on Yandex.Metrica dataset and the recommended way to get started with this dataset is to just go through tutorial. + +Additional examples of queries to these tables can be found among [stateful tests](https://github.com/yandex/ClickHouse/tree/master/dbms/tests/queries/1_stateful) of ClickHouse (they are named `test.hists` and `test.visits` there). diff --git a/docs/en/getting_started/index.md b/docs/en/getting_started/index.md index ed7335b748b..bfdcb0e108a 100644 --- a/docs/en/getting_started/index.md +++ b/docs/en/getting_started/index.md @@ -1,147 +1,8 @@ # Getting Started -## System Requirements - -ClickHouse can run on any Linux, FreeBSD or Mac OS X with x86\_64 CPU architecture. - -Though pre-built binaries are typically compiled to leverage SSE 4.2 instruction set, so unless otherwise stated usage of CPU that supports it becomes an additional system requirement. Here's the command to check if current CPU has support for SSE 4.2: - -``` bash -$ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" -``` - -## Installation - -### From DEB Packages - -Yandex ClickHouse team recommends using official pre-compiled `deb` packages for Debian or Ubuntu. - -To install official packages add the Yandex repository in `/etc/apt/sources.list` or in a separate `/etc/apt/sources.list.d/clickhouse.list` file: - -```bash -$ deb http://repo.yandex.ru/clickhouse/deb/stable/ main/ -``` - -If you want to use the most recent version, replace `stable` with `testing` (this is recommended for your testing environments). - -Then run these commands to actually install packages: - -```bash -$ sudo apt-get install dirmngr # optional -$ sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 # optional -$ sudo apt-get update -$ sudo apt-get install clickhouse-client clickhouse-server -``` - -You can also download and install packages manually from here: . - -### From RPM Packages - -Yandex ClickHouse team recommends using official pre-compiled `rpm` packages for CentOS, RedHat and all other rpm-based Linux distributions. - -First you need to add the official repository: - -```bash -$ sudo yum install yum-utils -$ sudo rpm --import https://repo.yandex.ru/clickhouse/CLICKHOUSE-KEY.GPG -$ sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/x86_64 -``` - -If you want to use the most recent version, replace `stable` with `testing` (this is recommended for your testing environments). - -Then run these commands to actually install packages: - -```bash -$ sudo yum install clickhouse-server clickhouse-client -``` - -You can also download and install packages manually from here: . - -### From Docker Image - -To run ClickHouse inside Docker follow the guide on [Docker Hub](https://hub.docker.com/r/yandex/clickhouse-server/). Those images use official `deb` packages inside. - -### From Sources - -To manually compile ClickHouse, follow the instructions for [Linux](../development/build.md) or [Mac OS X](../development/build_osx.md). - -You can compile packages and install them or use programs without installing packages. Also by building manually you can disable SSE 4.2 requirement or build for AArch64 CPUs. - -```text -Client: dbms/programs/clickhouse-client -Server: dbms/programs/clickhouse-server -``` - -You'll need to create a data and metadata folders and `chown` them for the desired user. Their paths can be changed in server config (src/dbms/programs/server/config.xml), by default they are: -```text -/opt/clickhouse/data/default/ -/opt/clickhouse/metadata/default/ -``` - -On Gentoo you can just use `emerge clickhouse` to install ClickHouse from sources. - -## Launch - -To start the server as a daemon, run: - -``` bash -$ sudo service clickhouse-server start -``` - -If you don't have `service` command, run as - -``` bash -$ sudo /etc/init.d/clickhouse-server start -``` - - -See the logs in the `/var/log/clickhouse-server/` directory. - -If the server doesn't start, check the configurations in the file `/etc/clickhouse-server/config.xml`. - -You can also manually launch the server from the console: - -``` bash -$ clickhouse-server --config-file=/etc/clickhouse-server/config.xml -``` - -In this case, the log will be printed to the console, which is convenient during development. -If the configuration file is in the current directory, you don't need to specify the `--config-file` parameter. By default, it uses `./config.xml`. - -ClickHouse supports access restriction settings. They are located in the `users.xml` file (next to `config.xml`). -By default, access is allowed from anywhere for the `default` user, without a password. See `user/default/networks`. -For more information, see the section ["Configuration Files"](../operations/configuration_files.md). - -After launching server, you can use the command-line client to connect to it: - -``` bash -$ clickhouse-client -``` - -By default it connects to `localhost:9000` on behalf of the user `default` without a password. It can also be used to connect to a remote server using `--host` argument. - -The terminal must use UTF-8 encoding. -For more information, see the section ["Command-line client"](../interfaces/cli.md). - -Example: -``` bash -$ ./clickhouse-client -ClickHouse client version 0.0.18749. -Connecting to localhost:9000. -Connected to ClickHouse server version 0.0.18749. -``` -```sql -SELECT 1 -``` -```text -┌─1─┐ -│ 1 │ -└───┘ -``` - -**Congratulations, the system works!** - -To continue experimenting, you can download one of test data sets or go through [tutorial](https://clickhouse.yandex/tutorial.html). +If you are new to ClickHouse and want to get a hands-on feeling of it's performance, first of all you need to go through the [installation process](install.md). After that you can: +* [Go through detailed tutorial](tutorial.md) +* [Experiment with example datasets](example_datasets/ontime.md) [Original article](https://clickhouse.yandex/docs/en/getting_started/) diff --git a/docs/en/getting_started/install.md b/docs/en/getting_started/install.md new file mode 100644 index 00000000000..e47500fa22f --- /dev/null +++ b/docs/en/getting_started/install.md @@ -0,0 +1,153 @@ +# Installation + +## System Requirements + +ClickHouse can run on any Linux, FreeBSD or Mac OS X with x86\_64, AArch64 or PowerPC64LE CPU architecture. + +Official pre-built binaries are typically compiled for x86\_64 and leverage SSE 4.2 instruction set, so unless otherwise stated usage of CPU that supports it becomes an additional system requirement. Here's the command to check if current CPU has support for SSE 4.2: + +``` bash +$ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" +``` + +To run ClickHouse on processors that do not support SSE 4.2 or have AArch64 or PowerPC64LE architecture, you should [build ClickHouse from sources](#from-sources) with proper configuration adjustments. + +## Available Installation Options + +### From DEB Packages + +It is recommended to use official pre-compiled `deb` packages for Debian or Ubuntu. + +To install official packages add the Yandex repository in `/etc/apt/sources.list` or in a separate `/etc/apt/sources.list.d/clickhouse.list` file: + +``` +deb http://repo.yandex.ru/clickhouse/deb/stable/ main/ +``` + +If you want to use the most recent version, replace `stable` with `testing` (this is recommended for your testing environments). + +Then run these commands to actually install packages: + +```bash +sudo apt-get install dirmngr # optional +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 # optional +sudo apt-get update +sudo apt-get install clickhouse-client clickhouse-server +``` + +You can also download and install packages manually from here: . + +### From RPM Packages + +It is recommended to use official pre-compiled `rpm` packages for CentOS, RedHat and all other rpm-based Linux distributions. + +First you need to add the official repository: + +```bash +sudo yum install yum-utils +sudo rpm --import https://repo.yandex.ru/clickhouse/CLICKHOUSE-KEY.GPG +sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/x86_64 +``` + +If you want to use the most recent version, replace `stable` with `testing` (this is recommended for your testing environments). + +Then run these commands to actually install packages: + +```bash +sudo yum install clickhouse-server clickhouse-client +``` + +You can also download and install packages manually from here: . + +### From Docker Image + +To run ClickHouse inside Docker follow the guide on [Docker Hub](https://hub.docker.com/r/yandex/clickhouse-server/). Those images use official `deb` packages inside. + +### From Sources + +To manually compile ClickHouse, follow the instructions for [Linux](../development/build.md) or [Mac OS X](../development/build_osx.md). + +You can compile packages and install them or use programs without installing packages. Also by building manually you can disable SSE 4.2 requirement or build for AArch64 CPUs. + +``` +Client: dbms/programs/clickhouse-client +Server: dbms/programs/clickhouse-server +``` + +You'll need to create a data and metadata folders and `chown` them for the desired user. Their paths can be changed in server config (src/dbms/programs/server/config.xml), by default they are: +``` +/opt/clickhouse/data/default/ +/opt/clickhouse/metadata/default/ +``` + +On Gentoo you can just use `emerge clickhouse` to install ClickHouse from sources. + +## Launch + +To start the server as a daemon, run: + +``` bash +$ sudo service clickhouse-server start +``` + +If you don't have `service` command, run as + +``` bash +$ sudo /etc/init.d/clickhouse-server start +``` + + +See the logs in the `/var/log/clickhouse-server/` directory. + +If the server doesn't start, check the configurations in the file `/etc/clickhouse-server/config.xml`. + +You can also manually launch the server from the console: + +``` bash +$ clickhouse-server --config-file=/etc/clickhouse-server/config.xml +``` + +In this case, the log will be printed to the console, which is convenient during development. +If the configuration file is in the current directory, you don't need to specify the `--config-file` parameter. By default, it uses `./config.xml`. + +ClickHouse supports access restriction settings. They are located in the `users.xml` file (next to `config.xml`). +By default, access is allowed from anywhere for the `default` user, without a password. See `user/default/networks`. +For more information, see the section ["Configuration Files"](../operations/configuration_files.md). + +After launching server, you can use the command-line client to connect to it: + +``` bash +$ clickhouse-client +``` + +By default it connects to `localhost:9000` on behalf of the user `default` without a password. It can also be used to connect to a remote server using `--host` argument. + +The terminal must use UTF-8 encoding. +For more information, see the section ["Command-line client"](../interfaces/cli.md). + +Example: +``` bash +$ ./clickhouse-client +ClickHouse client version 0.0.18749. +Connecting to localhost:9000. +Connected to ClickHouse server version 0.0.18749. + +:) SELECT 1 + +SELECT 1 + +┌─1─┐ +│ 1 │ +└───┘ + +1 rows in set. Elapsed: 0.003 sec. + +:) +``` + +**Congratulations, the system works!** + +To continue experimenting, you can download one of test data sets or go through [tutorial](https://clickhouse.yandex/tutorial.html). + + +[Original article](https://clickhouse.yandex/docs/en/getting_started/install/) diff --git a/docs/en/getting_started/tutorial.md b/docs/en/getting_started/tutorial.md new file mode 100644 index 00000000000..acdd9074beb --- /dev/null +++ b/docs/en/getting_started/tutorial.md @@ -0,0 +1,645 @@ +# ClickHouse Tutorial + +## What to Expect from This Tutorial? + +By going through this tutorial you'll learn how to set up basic ClickHouse cluster, it'll be small, but fault tolerant and scalable. We will use one of example datasets to fill it with data and execute some demo queries. + +## Single Node Setup + +To postpone complexities of distributed environment, we'll start with deploying ClickHouse on a single server or virtual machine. ClickHouse is usually installed from [deb](index.md#from-deb-packages) or [rpm](index.md#from-rpm-packages) packages, but there are [alternatives](index.md#from-docker-image) for the operating systems that do no support them. + +For example, you have chosen `deb` packages and executed: +``` bash +sudo apt-get install dirmngr +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 + +echo "deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" | sudo tee /etc/apt/sources.list.d/clickhouse.list +sudo apt-get update + +sudo apt-get install -y clickhouse-server clickhouse-client +``` + +What do we have in the packages that got installed: + +* `clickhouse-client` package contains [clickhouse-client](../interfaces/cli.md) application, interactive ClickHouse console client. +* `clickhouse-common` package contains a ClickHouse executable file. +* `clickhouse-server` package contains configuration files to run ClickHouse as a server. + +Server config files are located in `/etc/clickhouse-server/`. Before going further please notice the `` element in `config.xml`. Path determines the location for data storage, so it should be located on volume with large disk capacity, the default value is `/var/lib/clickhouse/`. If you want to adjust the configuration it's not really handy to directly edit `config.xml` file, considering it might get rewritten on future package updates. Recommended way to override the config elements is to create [files in config.d directory](../operations/configuration_files.md) which serve as "patches" to config.xml. + +As you might have noticed, `clickhouse-server` is not launched automatically after package installation. It won't be automatically restarted after updates either. The way you start the server depends on your init system, usually it's: + +``` bash +sudo service clickhouse-server start +``` +or + +``` bash +sudo /etc/init.d/clickhouse-server start +``` + +The default location for server logs is `/var/log/clickhouse-server/`. Server will be ready to handle client connections once `Ready for connections` message was logged. + +Once the `clickhouse-server` is up and running, we can use `clickhouse-client` to connect to the server and run some test queries like `SELECT "Hello, world!";`. + +
Quick tips for clickhouse-client +Interactive mode: +``` bash +clickhouse-client +clickhouse-client --host=... --port=... --user=... --password=... +``` + +Enable multiline queries: +``` bash +clickhouse-client -m +clickhouse-client --multiline +``` + +Run queries in batch-mode: +``` bash +clickhouse-client --query='SELECT 1' +echo 'SELECT 1' | clickhouse-client +clickhouse-client <<< 'SELECT 1' +``` + +Insert data from a file in specified format: +``` bash +clickhouse-client --query='INSERT INTO table VALUES' < data.txt +clickhouse-client --query='INSERT INTO table FORMAT TabSeparated' < data.tsv +``` +
+ +## Import Sample Dataset + +Now it's time to fill our ClickHouse server with some sample data. In this tutorial we'll use anonymized data of Yandex.Metrica, the first service that run ClickHouse in production way before it became open-source (more on that in [history section](../introduction/history.md)). There are [multiple ways to import Yandex.Metrica dataset](example_datasets/metrica.md) and for the sake of the tutorial we'll go with the most realistic one. + +### Download and Extract Table Data + +``` bash +curl https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv +curl https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv +``` + +The extracted files are about 10GB in size. + +### Create Tables + +Tables are logically grouped into "databases". There's a `default` database, but we'll create a new one named `tutorial`: + +``` bash +clickhouse-client --query "CREATE DATABASE IF NOT EXISTS tutorial" +``` + +Syntax for creating tables is way more complicated compared to databases (see [reference](../query_language/create.md). In general `CREATE TABLE` statement has to specify three key things: + +1. Name of table to create. +2. Table schema, i.e. list of columns and their [data types](../data_types/index.md). +3. [Table engine](../operations/table_engines/index.md) and it's settings, which determines all the details on how queries to this table will be physically executed. + +Yandex.Metrica is a web analytics service and sample dataset doesn't cover it's full functionality, so there are only two tables to create: + +* `hits` is a table with each action done by all users on all websites covered by the service. +* `visits` is a table that contains pre-built sessions instead of individual actions. + +Let's see and execute the real create table queries for these tables: + +``` sql +CREATE TABLE tutorial.hits_v1 +( + `WatchID` UInt64, + `JavaEnable` UInt8, + `Title` String, + `GoodEvent` Int16, + `EventTime` DateTime, + `EventDate` Date, + `CounterID` UInt32, + `ClientIP` UInt32, + `ClientIP6` FixedString(16), + `RegionID` UInt32, + `UserID` UInt64, + `CounterClass` Int8, + `OS` UInt8, + `UserAgent` UInt8, + `URL` String, + `Referer` String, + `URLDomain` String, + `RefererDomain` String, + `Refresh` UInt8, + `IsRobot` UInt8, + `RefererCategories` Array(UInt16), + `URLCategories` Array(UInt16), + `URLRegions` Array(UInt32), + `RefererRegions` Array(UInt32), + `ResolutionWidth` UInt16, + `ResolutionHeight` UInt16, + `ResolutionDepth` UInt8, + `FlashMajor` UInt8, + `FlashMinor` UInt8, + `FlashMinor2` String, + `NetMajor` UInt8, + `NetMinor` UInt8, + `UserAgentMajor` UInt16, + `UserAgentMinor` FixedString(2), + `CookieEnable` UInt8, + `JavascriptEnable` UInt8, + `IsMobile` UInt8, + `MobilePhone` UInt8, + `MobilePhoneModel` String, + `Params` String, + `IPNetworkID` UInt32, + `TraficSourceID` Int8, + `SearchEngineID` UInt16, + `SearchPhrase` String, + `AdvEngineID` UInt8, + `IsArtifical` UInt8, + `WindowClientWidth` UInt16, + `WindowClientHeight` UInt16, + `ClientTimeZone` Int16, + `ClientEventTime` DateTime, + `SilverlightVersion1` UInt8, + `SilverlightVersion2` UInt8, + `SilverlightVersion3` UInt32, + `SilverlightVersion4` UInt16, + `PageCharset` String, + `CodeVersion` UInt32, + `IsLink` UInt8, + `IsDownload` UInt8, + `IsNotBounce` UInt8, + `FUniqID` UInt64, + `HID` UInt32, + `IsOldCounter` UInt8, + `IsEvent` UInt8, + `IsParameter` UInt8, + `DontCountHits` UInt8, + `WithHash` UInt8, + `HitColor` FixedString(1), + `UTCEventTime` DateTime, + `Age` UInt8, + `Sex` UInt8, + `Income` UInt8, + `Interests` UInt16, + `Robotness` UInt8, + `GeneralInterests` Array(UInt16), + `RemoteIP` UInt32, + `RemoteIP6` FixedString(16), + `WindowName` Int32, + `OpenerName` Int32, + `HistoryLength` Int16, + `BrowserLanguage` FixedString(2), + `BrowserCountry` FixedString(2), + `SocialNetwork` String, + `SocialAction` String, + `HTTPError` UInt16, + `SendTiming` Int32, + `DNSTiming` Int32, + `ConnectTiming` Int32, + `ResponseStartTiming` Int32, + `ResponseEndTiming` Int32, + `FetchTiming` Int32, + `RedirectTiming` Int32, + `DOMInteractiveTiming` Int32, + `DOMContentLoadedTiming` Int32, + `DOMCompleteTiming` Int32, + `LoadEventStartTiming` Int32, + `LoadEventEndTiming` Int32, + `NSToDOMContentLoadedTiming` Int32, + `FirstPaintTiming` Int32, + `RedirectCount` Int8, + `SocialSourceNetworkID` UInt8, + `SocialSourcePage` String, + `ParamPrice` Int64, + `ParamOrderID` String, + `ParamCurrency` FixedString(3), + `ParamCurrencyID` UInt16, + `GoalsReached` Array(UInt32), + `OpenstatServiceName` String, + `OpenstatCampaignID` String, + `OpenstatAdID` String, + `OpenstatSourceID` String, + `UTMSource` String, + `UTMMedium` String, + `UTMCampaign` String, + `UTMContent` String, + `UTMTerm` String, + `FromTag` String, + `HasGCLID` UInt8, + `RefererHash` UInt64, + `URLHash` UInt64, + `CLID` UInt32, + `YCLID` UInt64, + `ShareService` String, + `ShareURL` String, + `ShareTitle` String, + `ParsedParams` Nested( + Key1 String, + Key2 String, + Key3 String, + Key4 String, + Key5 String, + ValueDouble Float64), + `IslandID` FixedString(16), + `RequestNum` UInt32, + `RequestTry` UInt8 +) +ENGINE = MergeTree() +PARTITION BY toYYYYMM(EventDate) +ORDER BY (CounterID, EventDate, intHash32(UserID)) +SAMPLE BY intHash32(UserID) +SETTINGS index_granularity = 8192 +``` + +``` sql +CREATE TABLE tutorial.visits_v1 +( + `CounterID` UInt32, + `StartDate` Date, + `Sign` Int8, + `IsNew` UInt8, + `VisitID` UInt64, + `UserID` UInt64, + `StartTime` DateTime, + `Duration` UInt32, + `UTCStartTime` DateTime, + `PageViews` Int32, + `Hits` Int32, + `IsBounce` UInt8, + `Referer` String, + `StartURL` String, + `RefererDomain` String, + `StartURLDomain` String, + `EndURL` String, + `LinkURL` String, + `IsDownload` UInt8, + `TraficSourceID` Int8, + `SearchEngineID` UInt16, + `SearchPhrase` String, + `AdvEngineID` UInt8, + `PlaceID` Int32, + `RefererCategories` Array(UInt16), + `URLCategories` Array(UInt16), + `URLRegions` Array(UInt32), + `RefererRegions` Array(UInt32), + `IsYandex` UInt8, + `GoalReachesDepth` Int32, + `GoalReachesURL` Int32, + `GoalReachesAny` Int32, + `SocialSourceNetworkID` UInt8, + `SocialSourcePage` String, + `MobilePhoneModel` String, + `ClientEventTime` DateTime, + `RegionID` UInt32, + `ClientIP` UInt32, + `ClientIP6` FixedString(16), + `RemoteIP` UInt32, + `RemoteIP6` FixedString(16), + `IPNetworkID` UInt32, + `SilverlightVersion3` UInt32, + `CodeVersion` UInt32, + `ResolutionWidth` UInt16, + `ResolutionHeight` UInt16, + `UserAgentMajor` UInt16, + `UserAgentMinor` UInt16, + `WindowClientWidth` UInt16, + `WindowClientHeight` UInt16, + `SilverlightVersion2` UInt8, + `SilverlightVersion4` UInt16, + `FlashVersion3` UInt16, + `FlashVersion4` UInt16, + `ClientTimeZone` Int16, + `OS` UInt8, + `UserAgent` UInt8, + `ResolutionDepth` UInt8, + `FlashMajor` UInt8, + `FlashMinor` UInt8, + `NetMajor` UInt8, + `NetMinor` UInt8, + `MobilePhone` UInt8, + `SilverlightVersion1` UInt8, + `Age` UInt8, + `Sex` UInt8, + `Income` UInt8, + `JavaEnable` UInt8, + `CookieEnable` UInt8, + `JavascriptEnable` UInt8, + `IsMobile` UInt8, + `BrowserLanguage` UInt16, + `BrowserCountry` UInt16, + `Interests` UInt16, + `Robotness` UInt8, + `GeneralInterests` Array(UInt16), + `Params` Array(String), + `Goals` Nested( + ID UInt32, + Serial UInt32, + EventTime DateTime, + Price Int64, + OrderID String, + CurrencyID UInt32), + `WatchIDs` Array(UInt64), + `ParamSumPrice` Int64, + `ParamCurrency` FixedString(3), + `ParamCurrencyID` UInt16, + `ClickLogID` UInt64, + `ClickEventID` Int32, + `ClickGoodEvent` Int32, + `ClickEventTime` DateTime, + `ClickPriorityID` Int32, + `ClickPhraseID` Int32, + `ClickPageID` Int32, + `ClickPlaceID` Int32, + `ClickTypeID` Int32, + `ClickResourceID` Int32, + `ClickCost` UInt32, + `ClickClientIP` UInt32, + `ClickDomainID` UInt32, + `ClickURL` String, + `ClickAttempt` UInt8, + `ClickOrderID` UInt32, + `ClickBannerID` UInt32, + `ClickMarketCategoryID` UInt32, + `ClickMarketPP` UInt32, + `ClickMarketCategoryName` String, + `ClickMarketPPName` String, + `ClickAWAPSCampaignName` String, + `ClickPageName` String, + `ClickTargetType` UInt16, + `ClickTargetPhraseID` UInt64, + `ClickContextType` UInt8, + `ClickSelectType` Int8, + `ClickOptions` String, + `ClickGroupBannerID` Int32, + `OpenstatServiceName` String, + `OpenstatCampaignID` String, + `OpenstatAdID` String, + `OpenstatSourceID` String, + `UTMSource` String, + `UTMMedium` String, + `UTMCampaign` String, + `UTMContent` String, + `UTMTerm` String, + `FromTag` String, + `HasGCLID` UInt8, + `FirstVisit` DateTime, + `PredLastVisit` Date, + `LastVisit` Date, + `TotalVisits` UInt32, + `TraficSource` Nested( + ID Int8, + SearchEngineID UInt16, + AdvEngineID UInt8, + PlaceID UInt16, + SocialSourceNetworkID UInt8, + Domain String, + SearchPhrase String, + SocialSourcePage String), + `Attendance` FixedString(16), + `CLID` UInt32, + `YCLID` UInt64, + `NormalizedRefererHash` UInt64, + `SearchPhraseHash` UInt64, + `RefererDomainHash` UInt64, + `NormalizedStartURLHash` UInt64, + `StartURLDomainHash` UInt64, + `NormalizedEndURLHash` UInt64, + `TopLevelDomain` UInt64, + `URLScheme` UInt64, + `OpenstatServiceNameHash` UInt64, + `OpenstatCampaignIDHash` UInt64, + `OpenstatAdIDHash` UInt64, + `OpenstatSourceIDHash` UInt64, + `UTMSourceHash` UInt64, + `UTMMediumHash` UInt64, + `UTMCampaignHash` UInt64, + `UTMContentHash` UInt64, + `UTMTermHash` UInt64, + `FromHash` UInt64, + `WebVisorEnabled` UInt8, + `WebVisorActivity` UInt32, + `ParsedParams` Nested( + Key1 String, + Key2 String, + Key3 String, + Key4 String, + Key5 String, + ValueDouble Float64), + `Market` Nested( + Type UInt8, + GoalID UInt32, + OrderID String, + OrderPrice Int64, + PP UInt32, + DirectPlaceID UInt32, + DirectOrderID UInt32, + DirectBannerID UInt32, + GoodID String, + GoodName String, + GoodQuantity Int32, + GoodPrice Int64), + `IslandID` FixedString(16) +) +ENGINE = CollapsingMergeTree(Sign) +PARTITION BY toYYYYMM(StartDate) +ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID) +SAMPLE BY intHash32(UserID) +SETTINGS index_granularity = 8192 +``` + +You can execute those queries using interactive mode of `clickhouse-client` (just launch it in terminal without specifying a query in advance) or try some [alternative interface](../interfaces/index.md) if you ant. + +As we can see, `hits_v1` uses the [basic MergeTree engine](../operations/table_engines/mergetree.md), while the `visits_v1` uses the [Collapsing](../operations/table_engines/collapsingmergetree.md) variant. + +### Import Data + +Data import to ClickHouse is done via [INSERT INTO](../query_language/insert_into.md) query like in many other SQL databases. However data is usually provided in one of the [supported formats](../interfaces/formats.md) instead of `VALUES` clause (which is also supported). + +The files we downloaded earlier are in tab-separated format, so here's how to import them via console client: + +``` bash +clickhouse-client --query "INSERT INTO tutorial.hits_v1 FORMAT TSV" --max_insert_block_size=100000 < hits_v1.tsv +clickhouse-client --query "INSERT INTO tutorial.visits_v1 FORMAT TSV" --max_insert_block_size=100000 < visits_v1.tsv +``` + +ClickHouse has a lot of [settings to tune](../operations/settings/index.md) and one way to specify them in console client is via arguments, as we can see with `--max_insert_block_size`. The easiest way to figure out what settings are available, what do they mean and what the defaults are is to query the `system.settings` table: + +``` sql +SELECT name, value, changed, description +FROM system.settings +WHERE name LIKE '%max_insert_b%' +FORMAT TSV + +max_insert_block_size 1048576 0 "The maximum block size for insertion, if we control the creation of blocks for insertion." +``` + +Optionally you can [OPTIMIZE](../query_language/misc/#misc_operations-optimize) the tables after import. Tables that are configured with MergeTree-family engine always do merges of data parts in background to optimize data storage (or at least check if it makes sense). These queries will just force table engine to do storage optimization right now instead of some time later: +``` bash +clickhouse-client --query "OPTIMIZE TABLE tutorial.hits_v1 FINAL" +clickhouse-client --query "OPTIMIZE TABLE tutorial.visits_v1 FINAL" +``` + +This is I/O and CPU intensive operation so if the table constantly receives new data it's better to leave it alone and let merges run in background. + +Now we can check that the tables are successfully imported: +``` bash +clickhouse-client --query "SELECT COUNT(*) FROM tutorial.hits_v1" +clickhouse-client --query "SELECT COUNT(*) FROM tutorial.visits_v1" +``` + +## Example Queries + +``` sql +SELECT + StartURL AS URL, + AVG(Duration) AS AvgDuration +FROM tutorial.visits_v1 +WHERE StartDate BETWEEN '2014-03-23' AND '2014-03-30' +GROUP BY URL +ORDER BY AvgDuration DESC +LIMIT 10 +``` + +``` sql +SELECT + sum(Sign) AS visits, + sumIf(Sign, has(Goals.ID, 1105530)) AS goal_visits, + (100. * goal_visits) / visits AS goal_percent +FROM tutorial.visits_v1 +WHERE (CounterID = 912887) AND (toYYYYMM(StartDate) = 201403) AND (domain(StartURL) = 'yandex.ru') +``` + +## Cluster Deployment + +ClickHouse cluster is a homogenous cluster. Steps to set up: + +1. Install ClickHouse server on all machines of the cluster +2. Set up cluster configs in configuration files +3. Create local tables on each instance +4. Create a [Distributed table](../operations/table_engines/distributed.md) + +[Distributed table](../operations/table_engines/distributed.md) is actually a kind of "view" to local tables of ClickHouse cluster. SELECT query from a distributed table will be executed using resources of all cluster's shards. You may specify configs for multiple clusters and create multiple distributed tables providing views to different clusters. + +Example config for cluster with three shards, one replica each: +``` xml + + + + + example-perftest01j.yandex.ru + 9000 + + + + + example-perftest02j.yandex.ru + 9000 + + + + + example-perftest03j.yandex.ru + 9000 + + + + +``` + +For further demonstration let's create new local table with exactly the same `CREATE TABLE` query that we used for `hits_v1`, but different table name: +``` sql +CREATE TABLE tutorial.hits_local (...) ENGINE = MergeTree() ... +``` + +Creating a distributed table providing a view into local tables of the cluster: +``` sql +CREATE TABLE tutorial.hits_all AS tutorial.hits_local +ENGINE = Distributed(perftest_3shards_1replicas, tutorial, hits_local, rand()); +``` + +Common practice is to create similar Distributed tables on all machines of the cluster. This would allow to run distributed queries on any machine of the cluster. Also there's an alternative option to create temporary distributed table for a given SELECT query using [remote](../query_language/table_functions/remote.md) table function. + +Let's run [INSERT SELECT](../query_language/insert_into.md) into Distributed table to spread the table to multiple servers. + +``` sql +INSERT INTO tutorial.hits_all SELECT * FROM tutorial.hits_v1; +``` + +!!! warning "Notice" + This approach is not suitable for sharding of large tables. There's a separate tool [clickhouse-copier](../operations/utils/clickhouse-copier.md) that can re-shard arbitrary large tables. + +As you could expect computationally heavy queries are executed N times faster being launched on 3 servers instead of one. + +In this case we have used a cluster with 3 shards each contains a single replica. + +To provide resilience in production environment we recommend that each shard should contain 2-3 replicas distributed between multiple data-centers. Note that ClickHouse supports unlimited number of replicas. + +Example config for cluster of one shard containing three replicas: +``` xml + + ... + + + + example-perftest01j.yandex.ru + 9000 + + + example-perftest02j.yandex.ru + 9000 + + + example-perftest03j.yandex.ru + 9000 + + + + +``` + +To enable native replication ZooKeeper is required. ClickHouse will take care of data consistency on all replicas and run restore procedure after failure + automatically. It's recommended to deploy ZooKeeper cluster to separate servers. + +ZooKeeper is not a strict requirement: in some simple cases you can duplicate the data by writing it into all the replicas from your application code. This approach is **not** recommended, in this case ClickHouse won't be able to + guarantee data consistency on all replicas. This remains the responsibility of your application. + +ZooKeeper locations need to be specified in configuration file: +``` xml + + + zoo01.yandex.ru + 2181 + + + zoo02.yandex.ru + 2181 + + + zoo03.yandex.ru + 2181 + + +``` + +Also we need to set macros for identifying each shard and replica, it will be used on table creation: +``` xml + + 01 + 01 + +``` + +If there are no replicas at the moment on replicated table creation, a new first replica will be instantiated. If there are already live replicas, new replica will clone the data from existing ones. You have an option to create all replicated tables first and that insert data to it. Another option is to create some replicas and add the others after or during data insertion. + +``` sql +CREATE TABLE tutorial.hits_replica (...) +ENGINE = ReplcatedMergeTree( + '/clickhouse_perftest/tables/{shard}/hits', + '{replica}' +) +... +``` + +Here we use [ReplicatedMergeTree](../operations/table_engines/replication.md) table engine. In parameters we specify ZooKeeper path containing shard and replica identifiers. + +``` sql +INSERT INTO tutorial.hits_replica SELECT * FROM tutorial.hits_local; +``` +Replication operates in multi-master mode. Data can be loaded into any replica and it will be synced with other instances automatically. Replication is asynchronous so at a given moment of time not all replicas may contain recently inserted data. To allow data insertion at least one replica should be up. Others will sync up data and repair consistency once they will become active again. Please notice that such approach allows for the low possibility of loss of just appended data. diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md index b582ab447d2..198e5f5c094 100644 --- a/docs/en/interfaces/cli.md +++ b/docs/en/interfaces/cli.md @@ -1,17 +1,23 @@ # Command-line Client -To work from the command line, you can use `clickhouse-client`: +ClickHouse provides a native command-line client: `clickhouse-client`. The client supports command-line options and configuration files. For more information, see [Configuring](#interfaces_cli_configuration). + +[Install](../getting_started/index.md) it from the `clickhouse-client` package and run it with the command `clickhouse-client`. ```bash $ clickhouse-client -ClickHouse client version 0.0.26176. -Connecting to localhost:9000. -Connected to ClickHouse server version 0.0.26176. +ClickHouse client version 19.17.1.1579 (official build). +Connecting to localhost:9000 as user default. +Connected to ClickHouse server version 19.17.1 revision 54428. :) ``` -The client supports command-line options and configuration files. For more information, see "[Configuring](#interfaces_cli_configuration)". +Different client and server versions are compatible with one another, but some features may not be available in older clients. We recommend using the same version of the client as the server app. When you try to use a client of the older version, then the server, `clickhouse-client` displays the message: + +``` +ClickHouse client version is older than ClickHouse server. It may lack support for new features. +``` ## Usage {#cli_usage} @@ -39,9 +45,9 @@ Similarly, to process a large number of queries, you can run 'clickhouse-client' In interactive mode, you get a command line where you can enter queries. -If 'multiline' is not specified (the default):To run the query, press Enter. The semicolon is not necessary at the end of the query. To enter a multiline query, enter a backslash `\` before the line feed. After you press Enter, you will be asked to enter the next line of the query. +If 'multiline' is not specified (the default): To run the query, press Enter. The semicolon is not necessary at the end of the query. To enter a multiline query, enter a backslash `\` before the line feed. After you press Enter, you will be asked to enter the next line of the query. -If multiline is specified:To run a query, end it with a semicolon and press Enter. If the semicolon was omitted at the end of the entered line, you will be asked to enter the next line of the query. +If multiline is specified: To run a query, end it with a semicolon and press Enter. If the semicolon was omitted at the end of the entered line, you will be asked to enter the next line of the query. Only a single query is run, so everything after the semicolon is ignored. @@ -125,7 +131,7 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va `clickhouse-client` uses the first existing file of the following: -- Defined in the `-config-file` parameter. +- Defined in the `--config-file` parameter. - `./clickhouse-client.xml` - `~/.clickhouse-client/config.xml` - `/etc/clickhouse-client/config.xml` diff --git a/docs/en/interfaces/cpp.md b/docs/en/interfaces/cpp.md new file mode 100644 index 00000000000..e10942ad8d5 --- /dev/null +++ b/docs/en/interfaces/cpp.md @@ -0,0 +1,5 @@ +# C++ Client Library + +See README at [clickhouse-cpp](https://github.com/ClickHouse/clickhouse-cpp) repository. + +[Original article](https://clickhouse.yandex/docs/en/interfaces/cpp/) diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index ee05a1cdb64..25a146f78b3 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -28,8 +28,12 @@ $ wget -O- -q 'http://localhost:8123/?query=SELECT 1' $ echo -ne 'GET /?query=SELECT%201 HTTP/1.0\r\n\r\n' | nc localhost 8123 HTTP/1.0 200 OK +Date: Wed, 27 Nov 2019 10:30:18 GMT Connection: Close -Date: Fri, 16 Nov 2012 19:21:50 GMT +Content-Type: text/tab-separated-values; charset=UTF-8 +X-ClickHouse-Server-Display-Name: clickhouse.ru-central1.internal +X-ClickHouse-Query-Id: 5abe861c-239c-467f-b955-8a201abb8b7f +X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} 1 ``` diff --git a/docs/en/interfaces/index.md b/docs/en/interfaces/index.md index 3f675867e22..57b22adaa69 100644 --- a/docs/en/interfaces/index.md +++ b/docs/en/interfaces/index.md @@ -10,6 +10,7 @@ In most cases it is recommended to use appropriate tool or library instead of in * [Command-line client](cli.md) * [JDBC driver](jdbc.md) * [ODBC driver](odbc.md) +* [C++ client library](cpp.md) There are also a wide range of third-party libraries for working with ClickHouse: diff --git a/docs/en/interfaces/third-party/client_libraries.md b/docs/en/interfaces/third-party/client_libraries.md index a619f647790..e0842ab36ef 100644 --- a/docs/en/interfaces/third-party/client_libraries.md +++ b/docs/en/interfaces/third-party/client_libraries.md @@ -27,7 +27,7 @@ - [HTTP-ClickHouse](https://metacpan.org/release/HTTP-ClickHouse) - [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse) - Ruby - - [clickhouse (Ruby)](https://github.com/archan937/clickhouse) + - [ClickHouse (Ruby)](https://github.com/shlima/click_house) - R - [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r) - [RClickhouse](https://github.com/IMSMWU/RClickhouse) @@ -41,8 +41,6 @@ - C# - [ClickHouse.Ado](https://github.com/killwort/ClickHouse-Net) - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) -- C++ - - [clickhouse-cpp](https://github.com/ClickHouse/clickhouse-cpp/) - Elixir - [clickhousex](https://github.com/appodeal/clickhousex/) - Nim diff --git a/docs/en/interfaces/third-party/gui.md b/docs/en/interfaces/third-party/gui.md index 3ed0d8924e2..f6842a63adb 100644 --- a/docs/en/interfaces/third-party/gui.md +++ b/docs/en/interfaces/third-party/gui.md @@ -50,16 +50,15 @@ Features: ### Redash -[Redash](https://github.com/getredash/redash) is an insights platform. +[Redash](https://github.com/getredash/redash) is a platform for data visualization. + +Supports for multiple data sources including ClickHouse, Redash can join results of queries from different data sources into one final dataset. Features: -- Connect with multiple sources (including ClickHouse). -- Autocompletion query editor. -- Schema/Database explorer. -- Data plotting. -- Create snippets for elements you frequently use. -- Use query results as data sources to join different databases. +- Powerful editor of queries. +- Database explorer. +- Visualization tools, that allow you to represent data in different forms. ### DBeaver @@ -77,6 +76,7 @@ Features: [clickhouse-cli](https://github.com/hatarist/clickhouse-cli) is an alternative command line client for ClickHouse, written in Python 3. Features: + - Autocompletion. - Syntax highlighting for the queries and data output. - Pager support for the data output. @@ -88,6 +88,18 @@ Features: ## Commercial +### Holistics Software + +[Holistics](https://www.holistics.io/) was listed by Gartner's Frontrunners in 2019 as one of the top 2 highest ranked business intelligence tools globally for usability. Holistics is a full-stack data platform and business intelligence tool for setting up your analytics processes, built on SQL. + +Features: + +- Automated email, Slack and Google Sheet schedules of reports. +- Powerful SQL editor with visualizations, version control, auto-completion, reusable query components and dynamic filters. +- Embedded analytics of reports and dashboards via iframe. +- Data preparation and ETL capabilities. +- SQL data modeling support for relational mapping of data. + ### DataGrip [DataGrip](https://www.jetbrains.com/datagrip/) is a database IDE from JetBrains with dedicated support for ClickHouse. It is also embedded into other IntelliJ-based tools: PyCharm, IntelliJ IDEA, GoLand, PhpStorm and others. @@ -101,4 +113,17 @@ Features: - Refactorings. - Search and Navigation. +### Looker +[Looker](https://looker.com) is a data platform and business intelligence tool with support for 50+ database dialects including ClickHouse. Looker is available as a SaaS platform and self-hosted. Users +can use Looker via the browser to explore data, build visualizations and dashboards, schedule reports, and share their +insights with colleagues. Looker provides a rich set of tools to embed these features in other applications, and an API +to integrate data with other applications. + +Features: + +- Designed around ease of use and self-service for end users. +- Easy and agile development using LookML, a language which supports currated +[Data Modeling](https://looker.com/platform/data-modeling) to support report writers and end users. +- Powerful workflow integration via Looker's [Data Actions](https://looker.com/platform/actions). + [Original article](https://clickhouse.yandex/docs/en/interfaces/third-party/gui/) diff --git a/docs/en/introduction/ya_metrika_task.md b/docs/en/introduction/history.md similarity index 98% rename from docs/en/introduction/ya_metrika_task.md rename to docs/en/introduction/history.md index 41b33eff581..e8f373880f1 100644 --- a/docs/en/introduction/ya_metrika_task.md +++ b/docs/en/introduction/history.md @@ -1,4 +1,4 @@ -# Yandex.Metrica Use Case +# ClickHouse History ClickHouse was originally developed to power [Yandex.Metrica](https://metrica.yandex.com/), [the second largest web analytics platform in the world](http://w3techs.com/technologies/overview/traffic_analysis/all), and continues to be the core component of this system. With more than 13 trillion records in the database and more than 20 billion events daily, ClickHouse allows generating custom reports on the fly directly from non-aggregated data. This article briefly covers the goals of ClickHouse in the early stages of its development. @@ -47,4 +47,4 @@ OLAPServer worked well for non-aggregated data, but it had many restrictions tha To remove the limitations of OLAPServer and solve the problem of working with non-aggregated data for all reports, we developed the ClickHouse DBMS. -[Original article](https://clickhouse.yandex/docs/en/introduction/ya_metrika_task/) +[Original article](https://clickhouse.yandex/docs/en/introduction/history/) diff --git a/docs/en/operations/access_rights.md b/docs/en/operations/access_rights.md index 9168cdf333b..e5b1781f59f 100644 --- a/docs/en/operations/access_rights.md +++ b/docs/en/operations/access_rights.md @@ -28,10 +28,10 @@ Users are recorded in the `users` section. Here is a fragment of the `users.xml` Each list item has one of the following forms: The IP address or subnet mask. For example: 198.51.100.0/24 or 2001:DB8::/32. Host name. For example: example01. A DNS query is made for verification, and all addresses obtained are compared with the address of the customer. - Regular expression for host names. For example, ^example\d\d-\d\d-\d\.yandex\.ru$ + Regular expression for host names. For example, ^example\d\d-\d\d-\d\.host\.ru$ To check it, a DNS PTR request is made for the client's address and a regular expression is applied to the result. Then another DNS query is made for the result of the PTR query, and all received address are compared to the client address. - We strongly recommend that the regex ends with \.yandex\.ru$. + We strongly recommend that the regex ends with \.host\.ru$. If you are installing ClickHouse yourself, specify here: @@ -104,5 +104,4 @@ The user can get a list of all databases and tables in them by using `SHOW` quer Database access is not related to the [readonly](settings/permissions_for_queries.md#settings_readonly) setting. You can't grant full access to one database and `readonly` access to another one. - [Original article](https://clickhouse.yandex/docs/en/operations/access_rights/) diff --git a/docs/en/operations/monitoring.md b/docs/en/operations/monitoring.md index eaa0ffdd406..331c3c0144f 100644 --- a/docs/en/operations/monitoring.md +++ b/docs/en/operations/monitoring.md @@ -34,4 +34,4 @@ You can configure ClickHouse to export metrics to [Graphite](https://github.com/ Additionally, you can monitor server availability through the HTTP API. Send the `HTTP GET` request to `/`. If the server is available, it responds with `200 OK`. -To monitor servers in a cluster configuration, you should set the [max_replica_delay_for_distributed_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries) parameter and use the HTTP resource `/replicas-delay`. A request to `/replicas-delay` returns `200 OK` if the replica is available and is not delayed behind the other replicas. If a replica is delayed, it returns information about the gap. +To monitor servers in a cluster configuration, you should set the [max_replica_delay_for_distributed_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries) parameter and use the HTTP resource `/replicas_status`. A request to `/replicas_status` returns `200 OK` if the replica is available and is not delayed behind the other replicas. If a replica is delayed, it returns information about the gap. diff --git a/docs/en/operations/server_settings/settings.md b/docs/en/operations/server_settings/settings.md index 56151911f50..c76637cc927 100644 --- a/docs/en/operations/server_settings/settings.md +++ b/docs/en/operations/server_settings/settings.md @@ -368,12 +368,12 @@ For more information, see the section "[Creating replicated tables](../../operat ## mark_cache_size {#server-mark-cache-size} -Approximate size (in bytes) of the cache of "marks" used by [MergeTree](../../operations/table_engines/mergetree.md). +Approximate size (in bytes) of the cache of marks used by table engines of the [MergeTree](../../operations/table_engines/mergetree.md) family. The cache is shared for the server and memory is allocated as needed. The cache size must be at least 5368709120. -!!! note IMPORTANT - This parameter could be exceeded by user's setting [mark_cache_min_lifetime](../settings/settings.md#settings-mark_cache_min_lifetime). +!!! warning "Warning" + This parameter could be exceeded by the [mark_cache_min_lifetime](../settings/settings.md#settings-mark_cache_min_lifetime) setting. **Example** @@ -556,13 +556,13 @@ The path to the directory containing data. Setting for logging queries received with the [log_queries=1](../settings/settings.md) setting. -Queries are logged in the [system.query_log](../system_tables.md#system_tables-query-log) table, not in a separate file. You can change the name of the table in the `table` parameter (see below). +Queries are logged in the [system.query_log](../system_tables.md#system_tables-query_log) table, not in a separate file. You can change the name of the table in the `table` parameter (see below). Use the following parameters to configure logging: - `database` – Name of the database. - `table` – Name of the system table the queries will be logged in. -- `partition_by` – Sets a [custom partitioning key](../../operations/table_engines/custom_partitioning_key.md) for a system table. +- `partition_by` – Sets a [custom partitioning key](../../operations/table_engines/custom_partitioning_key.md) for a table. - `flush_interval_milliseconds` – Interval for flushing data from the buffer in memory to the table. If the table doesn't exist, ClickHouse will create it. If the structure of the query log changed when the ClickHouse server was updated, the table with the old structure is renamed, and a new table is created automatically. @@ -578,6 +578,54 @@ If the table doesn't exist, ClickHouse will create it. If the structure of the q ``` +## query_thread_log {#server_settings-query-thread-log} + +Setting for logging threads of queries received with the [log_query_threads=1](../settings/settings.md#settings-log-query-threads) setting. + +Queries are logged in the [system.query_thread_log](../system_tables.md#system_tables-query-thread-log) table, not in a separate file. You can change the name of the table in the `table` parameter (see below). + +Use the following parameters to configure logging: + +- `database` – Name of the database. +- `table` – Name of the system table the queries will be logged in. +- `partition_by` – Sets a [custom partitioning key](../../operations/table_engines/custom_partitioning_key.md) for a system table. +- `flush_interval_milliseconds` – Interval for flushing data from the buffer in memory to the table. + +If the table doesn't exist, ClickHouse will create it. If the structure of the query thread log changed when the ClickHouse server was updated, the table with the old structure is renamed, and a new table is created automatically. + +**Example** + +```xml + + system + query_thread_log
+ toMonday(event_date) + 7500 + +``` + +## trace_log {#server_settings-trace_log} + +Settings for the [trace_log](../system_tables.md#system_tables-trace_log) system table operation. + +Parameters: + +- `database` — Database for storing a table. +- `table` — Table name. +- `partition_by` — [Custom partitioning key](../../operations/table_engines/custom_partitioning_key.md) for a system table. +- `flush_interval_milliseconds` — Interval for flushing data from the buffer in memory to the table. + +The default server configuration file `config.xml` contains the following settings section: + +```xml + + system + trace_log
+ toYYYYMM(event_date) + 7500 + +``` + ## query_masking_rules Regexp-based rules, which will be applied to queries as well as all log messages before storing them in server logs, @@ -625,7 +673,7 @@ For the value of the `incl` attribute, see the section "[Configuration files](.. - [skip_unavailable_shards](../settings/settings.md#settings-skip_unavailable_shards) -## timezone +## timezone {#server_settings-timezone} The server's time zone. @@ -731,12 +779,12 @@ This section contains the following parameters: For example: - ```xml +```xml example_host 2181 - ``` +``` The `index` attribute specifies the node order when trying to connect to the ZooKeeper cluster. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 159d0cbe7ff..9117fcb2543 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -130,6 +130,17 @@ Possible values: Default value: 0. +## max_http_get_redirects {#setting-max_http_get_redirects} + +Limits the maximum number of HTTP GET redirect hops for [URL](../table_engines/url.md)-engine tables. The setting applies to the both types of tables: created by [CREATE TABLE](../../query_language/create/#create-table-query) query and by [url](../../query_language/table_functions/url.md) table function. + +Possible values: + +- Positive integer number of hops. +- 0 — No hops allowed. + +Default value: 0. + ## input_format_allow_errors_num {#settings-input_format_allow_errors_num} Sets the maximum number of acceptable errors when reading from text formats (CSV, TSV, etc.). @@ -212,7 +223,7 @@ INSERT INTO test VALUES (lower('Hello')), (lower('world')), (lower('INSERT')), ( - if `input_format_values_interpret_expressions=0` and `format_values_deduce_templates_of_expressions=1` expressions in the first, second and third rows will be parsed using template `lower(String)` and interpreted together, expression is the forth row will be parsed with another template (`upper(String)`) - if `input_format_values_interpret_expressions=1` and `format_values_deduce_templates_of_expressions=1` - the same as in previous case, but also allows fallback to interpreting expressions separately if it's not possible to deduce template. - This feature is experimental, disabled by default. +Enabled by default. ## input_format_values_accurate_types_of_literals {#settings-input_format_values_accurate_types_of_literals} This setting is used only when `input_format_values_deduce_templates_of_expressions = 1`. It can happen, that expressions for some column have the same structure, but contain numeric literals of different types, e.g @@ -424,7 +435,7 @@ Default value: 163840. ## merge_tree_min_bytes_for_concurrent_read {#setting-merge_tree_min_bytes_for_concurrent_read} -If a number of bytes to read from one file of a [MergeTree*](../table_engines/mergetree.md)-engine table exceeds `merge_tree_min_bytes_for_concurrent_read` then ClickHouse tries to perform a concurrent reading from this file on several threads. +If the number of bytes to read from one file of a [MergeTree*](../table_engines/mergetree.md)-engine table exceeds `merge_tree_min_bytes_for_concurrent_read`, then ClickHouse tries to concurrently read from this file from several threads. Possible values: @@ -445,7 +456,7 @@ Default value: 0. ## merge_tree_min_bytes_for_seek {#setting-merge_tree_min_bytes_for_seek} -If the distance between two data blocks to be read in one file is less than `merge_tree_min_bytes_for_seek` rows, then ClickHouse does not seek through the file, but reads the data sequentially. +If the distance between two data blocks to be read in one file is less than `merge_tree_min_bytes_for_seek` bytes, then ClickHouse sequentially reads range of file that contains both blocks, thus avoiding extra seek. Possible values: @@ -466,9 +477,9 @@ Default value: 8. ## merge_tree_max_rows_to_use_cache {#setting-merge_tree_max_rows_to_use_cache} -If ClickHouse should read more than `merge_tree_max_rows_to_use_cache` rows in one query, it does not use the cache of uncompressed blocks. The [uncompressed_cache_size](../server_settings/settings.md#server-settings-uncompressed_cache_size) server setting defines the size of the cache of uncompressed blocks. +If ClickHouse should read more than `merge_tree_max_rows_to_use_cache` rows in one query, it doesn't use the cache of uncompressed blocks. -The cache of uncompressed blocks stores data extracted for queries. ClickHouse uses this cache to speed up responses to repeated small queries. This setting protects the cache from trashing by queries reading a large amount of data. +The cache of uncompressed blocks stores data extracted for queries. ClickHouse uses this cache to speed up responses to repeated small queries. This setting protects the cache from trashing by queries that read a large amount of data. The [uncompressed_cache_size](../server_settings/settings.md#server-settings-uncompressed_cache_size) server setting defines the size of the cache of uncompressed blocks. Possible values: @@ -479,9 +490,9 @@ Default value: 128 ✕ 8192. ## merge_tree_max_bytes_to_use_cache {#setting-merge_tree_max_bytes_to_use_cache} -If ClickHouse should read more than `merge_tree_max_bytes_to_use_cache` bytes in one query, it does not use the cache of uncompressed blocks. The [uncompressed_cache_size](../server_settings/settings.md#server-settings-uncompressed_cache_size) server setting defines the size of the cache of uncompressed blocks. +If ClickHouse should read more than `merge_tree_max_bytes_to_use_cache` bytes in one query, it doesn't use the cache of uncompressed blocks. -The cache of uncompressed blocks stores data extracted for queries. ClickHouse uses this cache to speed up responses to repeated small queries. This setting protects the cache from trashing by queries reading a large amount of data. +The cache of uncompressed blocks stores data extracted for queries. ClickHouse uses this cache to speed up responses to repeated small queries. This setting protects the cache from trashing by queries that read a large amount of data. The [uncompressed_cache_size](../server_settings/settings.md#server-settings-uncompressed_cache_size) server setting defines the size of the cache of uncompressed blocks. Possible values: @@ -513,6 +524,16 @@ Queries sent to ClickHouse with this setup are logged according to the rules in log_queries=1 +## log_query_threads {#settings-log-query-threads} + +Setting up query threads logging. + +Queries' threads runned by ClickHouse with this setup are logged according to the rules in the [query_thread_log](../server_settings/settings.md#server_settings-query-thread-log) server configuration parameter. + +**Example**: + + log_query_threads=1 + ## max_insert_block_size {#settings-max_insert_block_size} The size of blocks to form for insertion into a table. @@ -595,6 +616,13 @@ Timeouts in seconds on the socket used for communicating with the client. Default value: 10, 300, 300. +## cancel_http_readonly_queries_on_client_close + +Cancels HTTP readonly queries (e.g. SELECT) when a client closes the connection without waiting for response. + +Default value: 0 + + ## poll_interval Lock in a wait loop for the specified number of seconds. @@ -912,25 +940,32 @@ Default value: `uniqExact`. ## skip_unavailable_shards {#settings-skip_unavailable_shards} -Enables or disables silent skipping of: +Enables or disables silently skipping of unavailable shards. -- Node, if its name cannot be resolved through DNS. +Shard is considered unavailable if all its replicas are unavailable. A replica is unavailable in the following cases: - When skipping is disabled, ClickHouse requires that all the nodes in the [cluster configuration](../server_settings/settings.md#server_settings_remote_servers) can be resolvable through DNS. Otherwise, ClickHouse throws an exception when trying to perform a query on the cluster. +- ClickHouse can't connect to replica for any reason. - If skipping is enabled, ClickHouse considers unresolved nodes as unavailable and tries to resolve them at every connection attempt. Such behavior creates the risk of wrong cluster configuration because a user can specify the wrong node name, and ClickHouse doesn't report about it. However, this can be useful in systems with dynamic DNS, for example, [Kubernetes](https://kubernetes.io), where nodes can be unresolvable during downtime, and this is not an error. + When connecting to a replica, ClickHouse performs several attempts. If all these attempts fail, the replica is considered unavailable. -- Shard, if there are no available replicas of the shard. +- Replica can't be resolved through DNS. - When skipping is disabled, ClickHouse throws an exception. + If replica's hostname can't be resolved through DNS, it can indicate the following situations: - When skipping is enabled, ClickHouse returns a partial answer and doesn't report about issues with nodes availability. + - Replica's host has no DNS record. It can occur in systems with dynamic DNS, for example, [Kubernetes](https://kubernetes.io), where nodes can be unresolvable during downtime, and this is not an error. + + - Configuration error. ClickHouse configuration file contains a wrong hostname. Possible values: - 1 — skipping enabled. + + If a shard is unavailable, ClickHouse returns a result based on partial data and doesn't report node availability issues. + - 0 — skipping disabled. + If a shard is unavailable, ClickHouse throws an exception. + Default value: 0. ## optimize_throw_if_noop {#setting-optimize_throw_if_noop} @@ -972,6 +1007,41 @@ Error count of each replica is capped at this value, preventing a single replica - [Table engine Distributed](../../operations/table_engines/distributed.md) - [`distributed_replica_error_half_life`](#settings-distributed_replica_error_half_life) + +## distributed_directory_monitor_sleep_time_ms {#distributed_directory_monitor_sleep_time_ms} + +Base interval of data sending by the [Distributed](../table_engines/distributed.md) table engine. Actual interval grows exponentially in case of any errors. + +Possible values: + +- Positive integer number of milliseconds. + +Default value: 100 milliseconds. + + +## distributed_directory_monitor_max_sleep_time_ms {#distributed_directory_monitor_max_sleep_time_ms} + +Maximum interval of data sending by the [Distributed](../table_engines/distributed.md) table engine. Limits exponential growth of the interval set in the [distributed_directory_monitor_sleep_time_ms](#distributed_directory_monitor_sleep_time_ms) setting. + +Possible values: + +- Positive integer number of milliseconds. + +Default value: 30000 milliseconds (30 seconds). + +## distributed_directory_monitor_batch_inserts {#distributed_directory_monitor_batch_inserts} + +Enables/disables sending of inserted data in batches. + +When batch sending is enabled, [Distributed](../table_engines/distributed.md) table engine tries to send multiple files of inserted data in one operation instead of sending them separately. Batch sending improves cluster performance by better server and network resources utilization. + +Possible values: + +- 1 — Enabled. +- 0 — Disabled. + +Defaule value: 0. + ## os_thread_priority {#setting-os_thread_priority} Sets the priority ([nice](https://en.wikipedia.org/wiki/Nice_(Unix))) for threads that execute queries. The OS scheduler considers this priority when choosing the next thread to run on each available CPU core. @@ -987,4 +1057,76 @@ Lower values mean higher priority. Threads with low `nice` priority values are e Default value: 0. + +## query_profiler_real_time_period_ns {#query_profiler_real_time_period_ns} + +Sets the period for a real clock timer of the query profiler. Real clock timer counts wall-clock time. + +Possible values: + +- Positive integer number, in nanoseconds. + + Recommended values: + + - 10000000 (100 times a second) nanoseconds and less for single queries. + - 1000000000 (once a second) for cluster-wide profiling. + +- 0 for turning off the timer. + +Type: [UInt64](../../data_types/int_uint.md). + +Default value: 1000000000 nanoseconds (once a second). + +**See Also** + +- [system.trace_log](../system_tables.md#system_tables-trace_log) + +## query_profiler_cpu_time_period_ns {#query_profiler_cpu_time_period_ns} + +Sets the period for a CPU clock timer of the query profiler. This timer counts only CPU time. + +Possible values: + +- Positive integer number of nanoseconds. + + Recommended values: + + - 10000000 (100 times a second) nanosecods and more for for single queries. + - 1000000000 (once a second) for cluster-wide profiling. + +- 0 for turning off the timer. + +Type: [UInt64](../../data_types/int_uint.md). + +Default value: 1000000000 nanoseconds. + +**See Also** + +- [system.trace_log](../system_tables.md#system_tables-trace_log) + +## allow_introspection_functions {#settings-allow_introspection_functions} + +Enables of disables [introspections functions](../../query_language/functions/introspection.md) for query profiling. + +Possible values: + +- 1 — Introspection functions enabled. +- 0 — Introspection functions disabled. + +Default value: 0. + +## input_format_parallel_parsing + +- Type: bool +- Default value: True + +Enable order-preserving parallel parsing of data formats. Supported only for TSV, TKSV, CSV and JSONEachRow formats. + +## min_chunk_bytes_for_parallel_parsing + +- Type: unsigned int +- Default value: 1 MiB + +The minimum chunk size in bytes, which each thread will parse in parallel. + [Original article](https://clickhouse.yandex/docs/en/operations/settings/settings/) diff --git a/docs/en/operations/settings/settings_users.md b/docs/en/operations/settings/settings_users.md index 99d558ff295..8937c59b667 100644 --- a/docs/en/operations/settings/settings_users.md +++ b/docs/en/operations/settings/settings_users.md @@ -64,13 +64,13 @@ Each element of the list can have one of the following forms: - `` — Hostname. - Example: `server01.yandex.ru`. + Example: `example01.host.ru`. To check access, a DNS query is performed, and all returned IP addresses are compared to the peer address. - `` — Regular expression for hostnames. - Example, `^server\d\d-\d\d-\d\.yandex\.ru$` + Example, `^example\d\d-\d\d-\d\.host\.ru$` To check access, a [DNS PTR query](https://en.wikipedia.org/wiki/Reverse_DNS_lookup) is performed for the peer address and then the specified regexp is applied. Then, another DNS query is performed for the results of the PTR query and all the received addresses are compared to the peer address. We strongly recommend that regexp ends with $. diff --git a/docs/en/operations/system_tables.md b/docs/en/operations/system_tables.md index de0c277f100..77964c7377f 100644 --- a/docs/en/operations/system_tables.md +++ b/docs/en/operations/system_tables.md @@ -206,7 +206,7 @@ Columns: ## system.graphite_retentions -Contains information about parameters [graphite_rollup](server_settings/settings.md#server_settings-graphite_rollup) which are used in tables with [*GraphiteMergeTree](table_engines/graphitemergetree.md) engines. +Contains information about parameters [graphite_rollup](server_settings/settings.md#server_settings-graphite_rollup) which are used in tables with [\*GraphiteMergeTree](table_engines/graphitemergetree.md) engines. Columns: @@ -338,6 +338,7 @@ Columns: - `table` (`String`) – Name of the table. - `engine` (`String`) – Name of the table engine without parameters. - `path` (`String`) – Absolute path to the folder with data part files. +- `disk` (`String`) – Name of a disk that stores the data part. - `hash_of_all_files` (`String`) – [sipHash128](../query_language/functions/hash_functions.md#hash_functions-siphash128) of compressed files. - `hash_of_uncompressed_files` (`String`) – [sipHash128](../query_language/functions/hash_functions.md#hash_functions-siphash128) of uncompressed files (files with marks, index file etc.). - `uncompressed_hash_of_compressed_files` (`String`) – [sipHash128](../query_language/functions/hash_functions.md#hash_functions-siphash128) of data in the compressed files as if they were uncompressed. @@ -354,11 +355,12 @@ This table contains information about events that occurred with [data parts](tab The `system.part_log` table contains the following columns: - `event_type` (Enum) — Type of the event that occurred with the data part. Can have one of the following values: - - `NEW_PART` — inserting - - `MERGE_PARTS` — merging - - `DOWNLOAD_PART` — downloading - - `REMOVE_PART` — removing or detaching using [DETACH PARTITION](../query_language/alter.md#alter_detach-partition) - - `MUTATE_PART` — updating. + - `NEW_PART` — Inserting of a new data part. + - `MERGE_PARTS` — Merging of data parts. + - `DOWNLOAD_PART` — Downloading a data part. + - `REMOVE_PART` — Removing or detaching a data part using [DETACH PARTITION](../query_language/alter.md#alter_detach-partition). + - `MUTATE_PART` — Mutating of a data part. + - `MOVE_PART` — Moving the data part from the one disk to another one. - `event_date` (Date) — Event date. - `event_time` (DateTime) — Event time. - `duration_ms` (UInt64) — Duration. @@ -377,26 +379,27 @@ The `system.part_log` table contains the following columns: The `system.part_log` table is created after the first inserting data to the `MergeTree` table. -## system.processes +## system.processes {#system_tables-processes} This system table is used for implementing the `SHOW PROCESSLIST` query. + Columns: -- `user` (String) – Name of the user who made the request. For distributed query processing, this is the user who helped the requestor server send the query to this server, not the user who made the distributed request on the requestor server. -- `address` (String) - The IP address the request was made from. The same for distributed processing. -- `elapsed` (Float64) - The time in seconds since request execution started. -- `rows_read` (UInt64) - The number of rows read from the table. For distributed processing, on the requestor server, this is the total for all remote servers. -- `bytes_read` (UInt64) - The number of uncompressed bytes read from the table. For distributed processing, on the requestor server, this is the total for all remote servers. -- `total_rows_approx` (UInt64) - The approximation of the total number of rows that should be read. For distributed processing, on the requestor server, this is the total for all remote servers. It can be updated during request processing, when new sources to process become known. -- `memory_usage` (UInt64) - How much memory the request uses. It might not include some types of dedicated memory. -- `query` (String) - The query text. For INSERT, it doesn't include the data to insert. -- `query_id` (String) - Query ID, if defined. +- `user` (String) – The user who made the query. Keep in mind that for distributed processing, queries are sent to remote servers under the `default` user. The field contains the username for a specific query, not for a query that this query initiated. +- `address` (String) – The IP address the request was made from. The same for distributed processing. To track where a distributed query was originally made from, look at `system.processes` on the query requestor server. +- `elapsed` (Float64) – The time in seconds since request execution started. +- `rows_read` (UInt64) – The number of rows read from the table. For distributed processing, on the requestor server, this is the total for all remote servers. +- `bytes_read` (UInt64) – The number of uncompressed bytes read from the table. For distributed processing, on the requestor server, this is the total for all remote servers. +- `total_rows_approx` (UInt64) – The approximation of the total number of rows that should be read. For distributed processing, on the requestor server, this is the total for all remote servers. It can be updated during request processing, when new sources to process become known. +- `memory_usage` (UInt64) – Amount of RAM the request uses. It might not include some types of dedicated memory. See the [max_memory_usage](../operations/settings/query_complexity.md#settings_max_memory_usage) setting. +- `query` (String) – The query text. For `INSERT`, it doesn't include the data to insert. +- `query_id` (String) – Query ID, if defined. -## system.query_log {#system_tables-query-log} +## system.query_log {#system_tables-query_log} Contains information about execution of queries. For each query, you can see processing start time, duration of processing, error messages and other information. -!!! note +!!! note "Note" The table doesn't contain input data for `INSERT` queries. ClickHouse creates this table only if the [query_log](server_settings/settings.md#server_settings-query-log) server parameter is specified. This parameter sets the logging rules, such as the logging interval or the name of the table the queries will be logged in. @@ -410,13 +413,13 @@ The `system.query_log` table registers two kinds of queries: Columns: -- `type` (UInt8) — Type of event that occurred when executing the query. Possible values: - - 1 — Successful start of query execution. - - 2 — Successful end of query execution. - - 3 — Exception before the start of query execution. - - 4 — Exception during the query execution. -- `event_date` (Date) — Event date. -- `event_time` (DateTime) — Event time. +- `type` (`Enum8`) — Type of event that occurred when executing the query. Values: + - `'QueryStart' = 1` — Successful start of query execution. + - `'QueryFinish' = 2` — Successful end of query execution. + - `'ExceptionBeforeStart' = 3` — Exception before the start of query execution. + - `'ExceptionWhileProcessing' = 4` — Exception during the query execution. +- `event_date` (Date) — Query starting date. +- `event_time` (DateTime) — Query starting time. - `query_start_time` (DateTime) — Start time of query execution. - `query_duration_ms` (UInt64) — Duration of query execution. - `read_rows` (UInt64) — Number of read rows. @@ -434,36 +437,32 @@ Columns: - 0 — Query was initiated by another query for distributed query execution. - `user` (String) — Name of the user who initiated the current query. - `query_id` (String) — ID of the query. -- `address` (FixedString(16)) — IP address the query was initiated from. -- `port` (UInt16) — The server port that was used to receive the query. -- `initial_user` (String) — Name of the user who ran the parent query (for distributed query execution). -- `initial_query_id` (String) — ID of the parent query. -- `initial_address` (FixedString(16)) — IP address that the parent query was launched from. -- `initial_port` (UInt16) — The server port that was used to receive the parent query from the client. +- `address` (IPv6) — IP address that was used to make the query. +- `port` (UInt16) — The client port that was used to make the query. +- `initial_user` (String) — Name of the user who ran the initial query (for distributed query execution). +- `initial_query_id` (String) — ID of the initial query (for distributed query execution). +- `initial_address` (IPv6) — IP address that the parent query was launched from. +- `initial_port` (UInt16) — The client port that was used to make the parent query. - `interface` (UInt8) — Interface that the query was initiated from. Possible values: - 1 — TCP. - 2 — HTTP. -- `os_user` (String) — User's OS. -- `client_hostname` (String) — Server name that the [clickhouse-client](../interfaces/cli.md) is connected to. -- `client_name` (String) — The [clickhouse-client](../interfaces/cli.md) name. -- `client_revision` (UInt32) — Revision of the [clickhouse-client](../interfaces/cli.md). -- `client_version_major` (UInt32) — Major version of the [clickhouse-client](../interfaces/cli.md). -- `client_version_minor` (UInt32) — Minor version of the [clickhouse-client](../interfaces/cli.md). -- `client_version_patch` (UInt32) — Patch component of the [clickhouse-client](../interfaces/cli.md) version. +- `os_user` (String) — OS's username who runs [clickhouse-client](../interfaces/cli.md). +- `client_hostname` (String) — Hostname of the client machine where the [clickhouse-client](../interfaces/cli.md) or another TCP client is run. +- `client_name` (String) — The [clickhouse-client](../interfaces/cli.md) or another TCP client name. +- `client_revision` (UInt32) — Revision of the [clickhouse-client](../interfaces/cli.md) or another TCP client. +- `client_version_major` (UInt32) — Major version of the [clickhouse-client](../interfaces/cli.md) or another TCP client. +- `client_version_minor` (UInt32) — Minor version of the [clickhouse-client](../interfaces/cli.md) or another TCP client. +- `client_version_patch` (UInt32) — Patch component of the [clickhouse-client](../interfaces/cli.md) or another TCP client version. - `http_method` (UInt8) — HTTP method that initiated the query. Possible values: - 0 — The query was launched from the TCP interface. - 1 — `GET` method was used. - 2 — `POST` method was used. - `http_user_agent` (String) — The `UserAgent` header passed in the HTTP request. -- `quota_key` (String) — The quota key specified in the [quotas](quotas.md) setting. +- `quota_key` (String) — The "quota key" specified in the [quotas](quotas.md) setting (see `keyed`). - `revision` (UInt32) — ClickHouse revision. - `thread_numbers` (Array(UInt32)) — Number of threads that are participating in query execution. -- `ProfileEvents.Names` (Array(String)) — Counters that measure the following metrics: - - Time spent on reading and writing over the network. - - Time spent on reading and writing to a disk. - - Number of network errors. - - Time spent on waiting when the network bandwidth is limited. -- `ProfileEvents.Values` (Array(UInt64)) — Values of metrics that are listed in the `ProfileEvents.Names` column. +- `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics. The description of them could be found in the table [system.events](#system_tables-events) +- `ProfileEvents.Values` (Array(UInt64)) — Values of metrics that are listed in the `ProfileEvents.Names` column. - `Settings.Names` (Array(String)) — Names of settings that were changed when the client ran the query. To enable logging changes to settings, set the `log_query_settings` parameter to 1. - `Settings.Values` (Array(String)) — Values of settings that are listed in the `Settings.Names` column. @@ -482,6 +481,115 @@ When the table is deleted manually, it will be automatically created on the fly. You can specify an arbitrary partitioning key for the `system.query_log` table in the [query_log](server_settings/settings.md#server_settings-query-log) server setting (see the `partition_by` parameter). +## system.query_thread_log {#system_tables-query-thread-log} + +The table contains information about each query execution thread. + +ClickHouse creates this table only if the [query_thread_log](server_settings/settings.md#server_settings-query-thread-log) server parameter is specified. This parameter sets the logging rules, such as the logging interval or the name of the table the queries will be logged in. + +To enable query logging, set the [log_query_threads](settings/settings.md#settings-log-query-threads) parameter to 1. For details, see the [Settings](settings/settings.md) section. + +Columns: + +- `event_date` (Date) — the date when the thread has finished execution of the query. +- `event_time` (DateTime) — the date and time when the thread has finished execution of the query. +- `query_start_time` (DateTime) — Start time of query execution. +- `query_duration_ms` (UInt64) — Duration of query execution. +- `read_rows` (UInt64) — Number of read rows. +- `read_bytes` (UInt64) — Number of read bytes. +- `written_rows` (UInt64) — For `INSERT` queries, the number of written rows. For other queries, the column value is 0. +- `written_bytes` (UInt64) — For `INSERT` queries, the number of written bytes. For other queries, the column value is 0. +- `memory_usage` (Int64) — The difference between the amount of allocated and freed memory in context of this thread. +- `peak_memory_usage` (Int64) — The maximum difference between the amount of allocated and freed memory in context of this thread. +- `thread_name` (String) — Name of the thread. +- `thread_number` (UInt32) — Internal thread ID. +- `os_thread_id` (Int32) — OS thread ID. +- `master_thread_number` (UInt32) — Internal ID of initial thread. +- `master_os_thread_id` (Int32) — OS initial ID of initial thread. +- `query` (String) — Query string. +- `is_initial_query` (UInt8) — Query type. Possible values: + - 1 — Query was initiated by the client. + - 0 — Query was initiated by another query for distributed query execution. +- `user` (String) — Name of the user who initiated the current query. +- `query_id` (String) — ID of the query. +- `address` (IPv6) — IP address that was used to make the query. +- `port` (UInt16) — The client port that was used to make the query. +- `initial_user` (String) — Name of the user who ran the initial query (for distributed query execution). +- `initial_query_id` (String) — ID of the initial query (for distributed query execution). +- `initial_address` (IPv6) — IP address that the parent query was launched from. +- `initial_port` (UInt16) — The client port that was used to make the parent query. +- `interface` (UInt8) — Interface that the query was initiated from. Possible values: + - 1 — TCP. + - 2 — HTTP. +- `os_user` (String) — OS's username who runs [clickhouse-client](../interfaces/cli.md). +- `client_hostname` (String) — Hostname of the client machine where the [clickhouse-client](../interfaces/cli.md) or another TCP client is run. +- `client_name` (String) — The [clickhouse-client](../interfaces/cli.md) or another TCP client name. +- `client_revision` (UInt32) — Revision of the [clickhouse-client](../interfaces/cli.md) or another TCP client. +- `client_version_major` (UInt32) — Major version of the [clickhouse-client](../interfaces/cli.md) or another TCP client. +- `client_version_minor` (UInt32) — Minor version of the [clickhouse-client](../interfaces/cli.md) or another TCP client. +- `client_version_patch` (UInt32) — Patch component of the [clickhouse-client](../interfaces/cli.md) or another TCP client version. +- `http_method` (UInt8) — HTTP method that initiated the query. Possible values: + - 0 — The query was launched from the TCP interface. + - 1 — `GET` method was used. + - 2 — `POST` method was used. +- `http_user_agent` (String) — The `UserAgent` header passed in the HTTP request. +- `quota_key` (String) — The "quota key" specified in the [quotas](quotas.md) setting (see `keyed`). +- `revision` (UInt32) — ClickHouse revision. +- `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics for this thread. The description of them could be found in the table [system.events](#system_tables-events) +- `ProfileEvents.Values` (Array(UInt64)) — Values of metrics for this thread that are listed in the `ProfileEvents.Names` column. + +By default, logs are added to the table at intervals of 7.5 seconds. You can set this interval in the [query_thread_log](server_settings/settings.md#server_settings-query-thread-log) server setting (see the `flush_interval_milliseconds` parameter). To flush the logs forcibly from the memory buffer into the table, use the `SYSTEM FLUSH LOGS` query. + +When the table is deleted manually, it will be automatically created on the fly. Note that all the previous logs will be deleted. + +!!! note + The storage period for logs is unlimited. Logs aren't automatically deleted from the table. You need to organize the removal of outdated logs yourself. + +You can specify an arbitrary partitioning key for the `system.query_thread_log` table in the [query_thread_log](server_settings/settings.md#server_settings-query-thread-log) server setting (see the `partition_by` parameter). + +## system.trace_log {#system_tables-trace_log} + +Contains stack traces collected by the sampling query profiler. + +ClickHouse creates this table when the [trace_log](server_settings/settings.md#server_settings-trace_log) server configuration section is set. Also the [query_profiler_real_time_period_ns](settings/settings.md#query_profiler_real_time_period_ns) and [query_profiler_cpu_time_period_ns](settings/settings.md#query_profiler_cpu_time_period_ns) settings should be set. + +To analyze logs, use the `addressToLine`, `addressToSymbol` and `demangle` introspection functions. + +Columns: + +- `event_date`([Date](../data_types/date.md)) — Date of sampling moment. +- `event_time`([DateTime](../data_types/datetime.md)) — Timestamp of sampling moment. +- `revision`([UInt32](../data_types/int_uint.md)) — ClickHouse server build revision. + + When connecting to server by `clickhouse-client`, you see the string similar to `Connected to ClickHouse server version 19.18.1 revision 54429.`. This field contains the `revision`, but not the `version` of a server. + +- `timer_type`([Enum8](../data_types/enum.md)) — Timer type: + + - `Real` represents wall-clock time. + - `CPU` represents CPU time. + +- `thread_number`([UInt32](../data_types/int_uint.md)) — Thread identifier. +- `query_id`([String](../data_types/string.md)) — Query identifier that can be used to get details about a query that was running from the [query_log](#system_tables-query_log) system table. +- `trace`([Array(UInt64)](../data_types/array.md)) — Stack trace at the moment of sampling. Each element is a virtual memory address inside ClickHouse server process. + +**Example** + +```sql +SELECT * FROM system.trace_log LIMIT 1 \G +``` + +```text +Row 1: +────── +event_date: 2019-11-15 +event_time: 2019-11-15 15:09:38 +revision: 54428 +timer_type: Real +thread_number: 48 +query_id: acc4d61f-5bd1-4a3e-bc91-2180be37c915 +trace: [94222141367858,94222152240175,94222152325351,94222152329944,94222152330796,94222151449980,94222144088167,94222151682763,94222144088167,94222151682763,94222144088167,94222144058283,94222144059248,94222091840750,94222091842302,94222091831228,94222189631488,140509950166747,140509942945935] +``` + ## system.replicas {#system_tables-replicas} Contains information and status for replicated tables residing on the local server. @@ -760,6 +868,30 @@ If there were problems with mutating some parts, the following columns contain a ## system.disks {#system_tables-disks} +Contains information about disks defined in the [server configuration](table_engines/mergetree.md#table_engine-mergetree-multiple-volumes_configure). + +Columns: + +- `name` ([String](../data_types/string.md)) — Name of a disk in the server configuration. +- `path` ([String](../data_types/string.md)) — Path to the mount point in the file system. +- `free_space` ([UInt64](../data_types/int_uint.md)) — Free space on disk in bytes. +- `total_space` ([UInt64](../data_types/int_uint.md)) — Disk volume in bytes. +- `keep_free_space` ([UInt64](../data_types/int_uint.md)) — Amount of disk space that should stay free on disk in bytes. Defined in the `keep_free_space_bytes` parameter of disk configuration. + + ## system.storage_policies {#system_tables-storage_policies} +Contains information about storage policies and volumes defined in the [server configuration](table_engines/mergetree.md#table_engine-mergetree-multiple-volumes_configure). + +Columns: + +- `policy_name` ([String](../data_types/string.md)) — Name of the storage policy. +- `volume_name` ([String](../data_types/string.md)) — Volume name defined in the storage policy. +- `volume_priority` ([UInt64](../data_types/int_uint.md)) — Volume order number in the configuration. +- `disks` ([Array(String)](../data_types/array.md)) — Disk names, defined in the storage policy. +- `max_data_part_size` ([UInt64](../data_types/int_uint.md)) — Maximum size of a data part that can be stored on volume disks (0 — no limit). +- `move_factor` ([Float64](../data_types/float.md)) — Ratio of free disk space. When the ratio exceeds the value of configuration parameter, ClickHouse start to move data to the next volume in order. + +If the storage policy contains more then one volume, then information for each volume is stored in the individual row of the table. + [Original article](https://clickhouse.yandex/docs/en/operations/system_tables/) diff --git a/docs/en/operations/table_engines/buffer.md b/docs/en/operations/table_engines/buffer.md index 8f4035da19b..2c9e0dd8dc0 100644 --- a/docs/en/operations/table_engines/buffer.md +++ b/docs/en/operations/table_engines/buffer.md @@ -10,7 +10,7 @@ Engine parameters: - `database` – Database name. Instead of the database name, you can use a constant expression that returns a string. - `table` – Table to flush data to. -- `num_layers` – Parallelism layer. Physically, the table will be represented as 'num_layers' of independent buffers. Recommended value: 16. +- `num_layers` – Parallelism layer. Physically, the table will be represented as `num_layers` of independent buffers. Recommended value: 16. - `min_time`, `max_time`, `min_rows`, `max_rows`, `min_bytes`, and `max_bytes` – Conditions for flushing data from the buffer. Data is flushed from the buffer and written to the destination table if all the `min*` conditions or at least one `max*` condition are met. diff --git a/docs/en/operations/table_engines/distributed.md b/docs/en/operations/table_engines/distributed.md index 67b6697d01f..a22fd43b34f 100644 --- a/docs/en/operations/table_engines/distributed.md +++ b/docs/en/operations/table_engines/distributed.md @@ -87,12 +87,9 @@ The Distributed engine requires writing clusters to the config file. Clusters fr There are two methods for writing data to a cluster: -First, you can define which servers to write which data to, and perform the write directly on each shard. In other words, perform INSERT in the tables that the distributed table "looks at". -This is the most flexible solution – you can use any sharding scheme, which could be non-trivial due to the requirements of the subject area. -This is also the most optimal solution, since data can be written to different shards completely independently. +First, you can define which servers to write which data to, and perform the write directly on each shard. In other words, perform INSERT in the tables that the distributed table "looks at". This is the most flexible solution – you can use any sharding scheme, which could be non-trivial due to the requirements of the subject area. This is also the most optimal solution, since data can be written to different shards completely independently. -Second, you can perform INSERT in a Distributed table. In this case, the table will distribute the inserted data across servers itself. -In order to write to a Distributed table, it must have a sharding key set (the last parameter). In addition, if there is only one shard, the write operation works without specifying the sharding key, since it doesn't have any meaning in this case. +Second, you can perform INSERT in a Distributed table. In this case, the table will distribute the inserted data across servers itself. In order to write to a Distributed table, it must have a sharding key set (the last parameter). In addition, if there is only one shard, the write operation works without specifying the sharding key, since it doesn't have any meaning in this case. Each shard can have a weight defined in the config file. By default, the weight is equal to one. Data is distributed across shards in the amount proportional to the shard weight. For example, if there are two shards and the first has a weight of 9 while the second has a weight of 10, the first will be sent 9 / 19 parts of the rows, and the second will be sent 10 / 19. @@ -115,11 +112,22 @@ You should be concerned about the sharding scheme in the following cases: - Queries are used that require joining data (IN or JOIN) by a specific key. If data is sharded by this key, you can use local IN or JOIN instead of GLOBAL IN or GLOBAL JOIN, which is much more efficient. - A large number of servers is used (hundreds or more) with a large number of small queries (queries of individual clients - websites, advertisers, or partners). In order for the small queries to not affect the entire cluster, it makes sense to locate data for a single client on a single shard. Alternatively, as we've done in Yandex.Metrica, you can set up bi-level sharding: divide the entire cluster into "layers", where a layer may consist of multiple shards. Data for a single client is located on a single layer, but shards can be added to a layer as necessary, and data is randomly distributed within them. Distributed tables are created for each layer, and a single shared distributed table is created for global queries. -Data is written asynchronously. For an INSERT to a Distributed table, the data block is just written to the local file system. The data is sent to the remote servers in the background as soon as possible. You should check whether data is sent successfully by checking the list of files (data waiting to be sent) in the table directory: /var/lib/clickhouse/data/database/table/. +Data is written asynchronously. When inserted to the table, the data block is just written to the local file system. The data is sent to the remote servers in the background as soon as possible. The period of data sending is managed by the [distributed_directory_monitor_sleep_time_ms](../settings/settings.md#distributed_directory_monitor_sleep_time_ms) and [distributed_directory_monitor_max_sleep_time_ms](../settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) settings. The `Distributed` engine sends each file with inserted data separately, but you can enable batch sending of files with the [distributed_directory_monitor_batch_inserts](../settings/settings.md#distributed_directory_monitor_batch_inserts) setting. This setting improves cluster performance by better local server and network resources utilization. You should check whether data is sent successfully by checking the list of files (data waiting to be sent) in the table directory: `/var/lib/clickhouse/data/database/table/`. If the server ceased to exist or had a rough restart (for example, after a device failure) after an INSERT to a Distributed table, the inserted data might be lost. If a damaged data part is detected in the table directory, it is transferred to the 'broken' subdirectory and no longer used. When the max_parallel_replicas option is enabled, query processing is parallelized across all replicas within a single shard. For more information, see the section [max_parallel_replicas](../settings/settings.md#settings-max_parallel_replicas). +## Virtual Columns + +- `_shard_num` — Contains the `shard_num` (from `system.clusters`). Type: [UInt32](../../data_types/int_uint.md). + +!!! note "Note" + Since [`remote`](../../query_language/table_functions/remote.md)/`cluster` table functions internally create temporary instance of the same Distributed engine, `_shard_num` is available there too. + +**See Also** + +- [Virtual columns](index.md#table_engines-virtual_columns) + [Original article](https://clickhouse.yandex/docs/en/operations/table_engines/distributed/) diff --git a/docs/en/operations/table_engines/file.md b/docs/en/operations/table_engines/file.md index 71e29bdff6d..ba106856968 100644 --- a/docs/en/operations/table_engines/file.md +++ b/docs/en/operations/table_engines/file.md @@ -72,6 +72,8 @@ $ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64 ## Details of Implementation - Multiple `SELECT` queries can be performed concurrently, but `INSERT` queries will wait each other. +- Supported creating new file by `INSERT` query. +- If file exists, `INSERT` would append new values in it. - Not supported: - `ALTER` - `SELECT ... SAMPLE` diff --git a/docs/en/operations/table_engines/hdfs.md b/docs/en/operations/table_engines/hdfs.md index 9e2947341bf..22760c02d83 100644 --- a/docs/en/operations/table_engines/hdfs.md +++ b/docs/en/operations/table_engines/hdfs.md @@ -1,6 +1,6 @@ # HDFS {#table_engines-hdfs} -This engine provides integration with [Apache Hadoop](https://en.wikipedia.org/wiki/Apache_Hadoop) ecosystem by allowing to manage data on [HDFS](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HdfsDesign.htmll)via ClickHouse. This engine is similar +This engine provides integration with [Apache Hadoop](https://en.wikipedia.org/wiki/Apache_Hadoop) ecosystem by allowing to manage data on [HDFS](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html)via ClickHouse. This engine is similar to the [File](file.md) and [URL](url.md) engines, but provides Hadoop-specific features. ## Usage diff --git a/docs/en/operations/table_engines/mergetree.md b/docs/en/operations/table_engines/mergetree.md index 7ae6c1b6834..f1c888e4480 100644 --- a/docs/en/operations/table_engines/mergetree.md +++ b/docs/en/operations/table_engines/mergetree.md @@ -41,7 +41,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] [ORDER BY expr] [PRIMARY KEY expr] [SAMPLE BY expr] -[TTL expr] +[TTL expr [DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'], ...] [SETTINGS name=value, ...] ``` @@ -70,22 +70,26 @@ For a description of parameters, see the [CREATE query description](../../query_ If a sampling expression is used, the primary key must contain it. Example: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`. -- `TTL` — An expression for setting storage time for rows. +- `TTL` — A list of rules specifying storage duration of rows and defining logic of automatic parts movement [between disks and volumes](#table_engine-mergetree-multiple-volumes). - It must depend on the `Date` or `DateTime` column and have one `Date` or `DateTime` column as a result. Example: + Expression must have one `Date` or `DateTime` column as a result. Example: `TTL date + INTERVAL 1 DAY` + Type of the rule `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'` specifies an action to be done with the part if the expression is satisfied (reaches current time): removal of expired rows, moving a part (if expression is satisfied for all rows in a part) to specified disk (`TO DISK 'xxx'`) or to volume (`TO VOLUME 'xxx'`). Default type of the rule is removal (`DELETE`). List of multiple rules can specified, but there should be no more than one `DELETE` rule. + For more details, see [TTL for columns and tables](#table_engine-mergetree-ttl) - `SETTINGS` — Additional parameters that control the behavior of the `MergeTree`: - `index_granularity` — Maximum number of data rows between the marks of an index. Default value: 8192. See [Data Storage](#mergetree-data-storage). - - `index_granularity_bytes` — Maximum size of data granule in bytes. Default value: 10Mb. To restrict the size of granule only by number of rows set 0 (not recommended). See [Data Storage](#mergetree-data-storage). - - `enable_mixed_granularity_parts` — Enables or disables transition to controlling the granule size with the `index_granularity_bytes` setting. Before the version 19.11 there was the only `index_granularity` setting for the granule size restriction. The `index_granularity_bytes` setting improves ClickHouse performance when selecting data from the tables with big rows (tens and hundreds of megabytes). So if you have tables with big rows, you can turn the setting on for the tables to get better efficiency of your `SELECT` queries. + - `index_granularity_bytes` — Maximum size of data granules in bytes. Default value: 10Mb. To restrict the granule size only by number of rows, set to 0 (not recommended). See [Data Storage](#mergetree-data-storage). + - `enable_mixed_granularity_parts` — Enables or disables transitioning to control the granule size with the `index_granularity_bytes` setting. Before version 19.11, there was only the `index_granularity` setting for restricting granule size. The `index_granularity_bytes` setting improves ClickHouse performance when selecting data from tables with big rows (tens and hundreds of megabytes). If you have tables with big rows, you can enable this setting for the tables to improve the efficiency of `SELECT` queries. - `use_minimalistic_part_header_in_zookeeper` — Storage method of the data parts headers in ZooKeeper. If `use_minimalistic_part_header_in_zookeeper=1`, then ZooKeeper stores less data. For more information, see the [setting description](../server_settings/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) in "Server configuration parameters". - `min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation that is required for using direct I/O access to the storage disk. When merging data parts, ClickHouse calculates the total storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, ClickHouse reads and writes the data to the storage disk using the direct I/O interface (`O_DIRECT` option). If `min_merge_bytes_to_use_direct_io = 0`, then direct I/O is disabled. Default value: `10 * 1024 * 1024 * 1024` bytes. - `merge_with_ttl_timeout` — Minimum delay in seconds before repeating a merge with TTL. Default value: 86400 (1 day). - `write_final_mark` — Enables or disables writing the final index mark at the end of data part. Default value: 1. Don't turn it off. + - `storage_policy` — Storage policy. See [Using Multiple Block Devices for Data Storage](#table_engine-mergetree-multiple-volumes). + **Example of Sections Setting** @@ -137,9 +141,9 @@ When data is inserted in a table, separate data parts are created and each of th Data belonging to different partitions are separated into different parts. In the background, ClickHouse merges data parts for more efficient storage. Parts belonging to different partitions are not merged. The merge mechanism does not guarantee that all rows with the same primary key will be in the same data part. -Each data part is logically divided by granules. A granule is the smallest indivisible data set that ClickHouse reads when selecting data. ClickHouse doesn't split rows or values, so each granule always contains an integer number of rows. The first row of a granule is marked with the value of the primary key for this row. For each data part, ClickHouse creates an index file that stores the marks. For each column, whether it is in the primary key or not, ClickHouse also stores the same marks. These marks allow finding the data directly in the columns. +Each data part is logically divided into granules. A granule is the smallest indivisible data set that ClickHouse reads when selecting data. ClickHouse doesn't split rows or values, so each granule always contains an integer number of rows. The first row of a granule is marked with the value of the primary key for the row. For each data part, ClickHouse creates an index file that stores the marks. For each column, whether it's in the primary key or not, ClickHouse also stores the same marks. These marks let you find data directly in column files. -The size of a granule is restricted by the `index_granularity` and `index_granularity_bytes` settings of the table engine. The number of rows in granule lays in the `[1, index_granularity]` range, depending on the size of rows. The size of a granule can exceed `index_granularity_bytes` if the size of the single row is greater than the value of the setting. In this case, the size of the granule equals the size of the row. +The granule size is restricted by the `index_granularity` and `index_granularity_bytes` settings of the table engine. The number of rows in a granule lays in the `[1, index_granularity]` range, depending on the size of the rows. The size of a granule can exceed `index_granularity_bytes` if the size of a single row is greater than the value of the setting. In this case, the size of the granule equals the size of the row. ## Primary Keys and Indexes in Queries {#primary-keys-and-indexes-in-queries} @@ -164,7 +168,7 @@ The examples above show that it is always more effective to use an index than a A sparse index allows extra data to be read. When reading a single range of the primary key, up to `index_granularity * 2` extra rows in each data block can be read. -Sparse indexes allow you to work with a very large number of table rows, because such indexes fit the computer's RAM in the very most cases. +Sparse indexes allow you to work with a very large number of table rows, because in most cases, such indexes fit in the computer's RAM. ClickHouse does not require a unique primary key. You can insert multiple rows with the same primary key. @@ -175,7 +179,7 @@ The number of columns in the primary key is not explicitly limited. Depending on - Improve the performance of an index. If the primary key is `(a, b)`, then adding another column `c` will improve the performance if the following conditions are met: - + - There are queries with a condition on column `c`. - Long data ranges (several times longer than the `index_granularity`) with identical values for `(a, b)` are common. In other words, when adding another column allows you to skip quite long data ranges. @@ -306,9 +310,9 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 The optional `false_positive` parameter is the probability of receiving a false positive response from the filter. Possible values: (0, 1). Default value: 0.025. - Supported data types: `Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`. + Supported data types: `Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`, `Array`, `LowCardinality`, `Nullable`. - The following functions can use it: [equals](../../query_language/functions/comparison_functions.md), [notEquals](../../query_language/functions/comparison_functions.md), [in](../../query_language/functions/in_functions.md), [notIn](../../query_language/functions/in_functions.md). + The following functions can use it: [equals](../../query_language/functions/comparison_functions.md), [notEquals](../../query_language/functions/comparison_functions.md), [in](../../query_language/functions/in_functions.md), [notIn](../../query_language/functions/in_functions.md), [has](../../query_language/functions/array_functions.md). ```sql INDEX sample_index (u64 * length(s)) TYPE minmax GRANULARITY 4 @@ -369,9 +373,11 @@ Reading from a table is automatically parallelized. Determines the lifetime of values. -The `TTL` clause can be set for the whole table and for each individual column. If both `TTL` are set, ClickHouse uses that `TTL` which expires earlier. +The `TTL` clause can be set for the whole table and for each individual column. Table-level TTL can also specify logic of automatic move of data between disks and volumes. -The table must have the column in the [Date](../../data_types/date.md) or [DateTime](../../data_types/datetime.md) data type. To define the lifetime of data, use operations on this time column, for example: +Expressions must evaluate to [Date](../../data_types/date.md) or [DateTime](../../data_types/datetime.md) data type. + +Example: ```sql TTL time_column @@ -426,7 +432,17 @@ ALTER TABLE example_table **Table TTL** -When data in a table expires, ClickHouse deletes all corresponding rows. +Table can have an expression for removal of expired rows, and multiple expressions for automatic move of parts between [disks or volumes](#table_engine-mergetree-multiple-volumes). When rows in the table expire, ClickHouse deletes all corresponding rows. For parts moving feature, all rows of a part must satisfy the movement expression criteria. + +```sql +TTL expr [DELETE|TO DISK 'aaa'|TO VOLUME 'bbb'], ... +``` + +Type of TTL rule may follow each TTL expression. It affects an action which is to be done once the expression is satisfied (reaches current time): + +- `DELETE` - delete expired rows (default action); +- `TO DISK 'aaa'` - move part to the disk `aaa`; +- `TO VOLUME 'bbb'` - move part to the disk `bbb`. Examples: @@ -441,7 +457,9 @@ CREATE TABLE example_table ENGINE = MergeTree PARTITION BY toYYYYMM(d) ORDER BY d -TTL d + INTERVAL 1 MONTH; +TTL d + INTERVAL 1 MONTH [DELETE], + d + INTERVAL 1 WEEK TO VOLUME 'aaa', + d + INTERVAL 2 WEEK TO DISK 'bbb'; ``` Altering TTL of the table @@ -462,53 +480,89 @@ If you perform the `SELECT` query between merges, you may get expired data. To a [Original article](https://clickhouse.yandex/docs/en/operations/table_engines/mergetree/) -## Using multiple block devices for data storage {#table_engine-mergetree-multiple-volumes} +## Using Multiple Block Devices for Data Storage {#table_engine-mergetree-multiple-volumes} -### General +### Introduction -Tables of the MergeTree family are able to store their data on multiple block devices, which may be useful when, for instance, the data of a certain table are implicitly split into "hot" and "cold". The most recent data is regularly requested but requires only a small amount of space. On the contrary, the fat-tailed historical data is requested rarely. If several disks are available, the "hot" data may be located on fast disks (NVMe SSDs or even in memory), while the "cold" data - on relatively slow ones (HDD). +`MergeTree` family table engines can store data on multiple block devices. For example, it can be useful when the data of a certain table are implicitly split into "hot" and "cold". The most recent data is regularly requested but requires only a small amount of space. On the contrary, the fat-tailed historical data is requested rarely. If several disks are available, the "hot" data may be located on fast disks (for example, NVMe SSDs or in memory), while the "cold" data - on relatively slow ones (for example, HDD). -Part is the minimum movable unit for MergeTree tables. The data belonging to one part are stored on one disk. Parts can be moved between disks in the background (according to user settings) as well as by means of the [ALTER](../../query_language/alter.md#alter_move-partition) queries. +Data part is the minimum movable unit for `MergeTree`-engine tables. The data belonging to one part are stored on one disk. Data parts can be moved between disks in the background (according to user settings) as well as by means of the [ALTER](../../query_language/alter.md#alter_move-partition) queries. ### Terms -* Disk — a block device mounted to the filesystem. -* Default disk — a disk that contains the path specified in the `` tag in `config.xml`. -* Volume — an ordered set of equal disks (similar to [JBOD](https://en.wikipedia.org/wiki/Non-RAID_drive_architectures)). -* Storage policy — a number of volumes together with the rules for moving data between them. -The names given to the described entities can be found in the system tables, [system.storage_policies](../system_tables.md#system_tables-storage_policies) and [system.disks](../system_tables.md#system_tables-disks). Storage policy name can be used as a parameter for tables of the MergeTree family. +- Disk — Block device mounted to the filesystem. +- Default disk — Disk that stores the path specified in the [path](../server_settings/settings.md#server_settings-path) server setting. +- Volume — Ordered set of equal disks (similar to [JBOD](https://en.wikipedia.org/wiki/Non-RAID_drive_architectures)). +- Storage policy — Set of volumes and the rules for moving data between them. + +The names given to the described entities can be found in the system tables, [system.storage_policies](../system_tables.md#system_tables-storage_policies) and [system.disks](../system_tables.md#system_tables-disks). To apply one of the configured storage policies for a table, use the `storage_policy` setting of `MergeTree`-engine family tables. ### Configuration {#table_engine-mergetree-multiple-volumes_configure} -Disks, volumes and storage policies should be declared inside the `` tag either in the main file `config.xml` or in a distinct file in the `config.d` directory. This section in a configuration file has the following structure: +Disks, volumes and storage policies should be declared inside the `` tag either in the main file `config.xml` or in a distinct file in the `config.d` directory. + +Configuration structure: ```xml - + /mnt/fast_ssd/clickhouse - - + + /mnt/hdd1/clickhouse 10485760_ - - + + /mnt/hdd2/clickhouse 10485760_ - + ... ``` -where +Tags: -* the disk name is given as a tag name. -* `path` — path under which a server will store data (`data` and `shadow` folders), should be terminated with '/'. -* `keep_free_space_bytes` — the amount of free disk space to be reserved. +- `` — Disk name. Names must be different for all disks. +- `path` — path under which a server will store data (`data` and `shadow` folders), should be terminated with '/'. +- `keep_free_space_bytes` — the amount of free disk space to be reserved. The order of the disk definition is not important. -Storage policies configuration: +Storage policies configuration markup: + +```xml + + + + + disk_name_from_disks_configuration + 1073741824 + + + + + + + 0.2 + + + + + + + +``` + +Tags: + +- `policy_name_N` — Policy name. Policy names must be unique. +- `volume_name_N` — Volume name. Volume names must be unique. +- `disk` — a disk within a volume. +- `max_data_part_size_bytes` — the maximum size of a part that can be stored on any of the volume's disks. +- `move_factor` — when the amount of available space gets lower than this factor, data automatically start to move on the next volume if any (by default, 0.1). + +Cofiguration examples: ```xml @@ -536,16 +590,9 @@ Storage policies configuration: ``` -where +In given example, the `hdd_in_order` policy implements the [round-robin](https://en.wikipedia.org/wiki/Round-robin_scheduling) approach. Thus this policy defines only one volume (`single`), the data parts are stored on all its disks in circular order. Such policy can be quite useful if there are several similar disks are mounted to the system, but RAID is not configured. Keep in mind that each individual disk drive is not reliable and you might want to compensate it with replication factor of 3 or more. -* volume and storage policy names are given as tag names. -* `disk` — a disk within a volume. -* `max_data_part_size_bytes` — the maximum size of a part that can be stored on any of the volume's disks. -* `move_factor` — when the amount of available space gets lower than this factor, data automatically start to move on the next volume if any (by default, 0.1). - - -In the given example, the `hdd_in_order` policy implements the [round-robin](https://en.wikipedia.org/wiki/Round-robin_scheduling) approach. Since the policy defines only one volume (`single`), the data are stored on all its disks in circular order. Such a policy can be quite useful if there are several similar disks mounted to the system. If there are different disks, the policy `moving_from_ssd_to_hdd` can be used instead. -The volume `hot` consists of an SSD disk (`fast_ssd`), and the maximum size of a part that can be stored on this volume is 1GB. All the parts with the size larger than 1GB will be stored directly on the `cold` volume, which contains an HDD disk `disk1`. +If there are different kinds of disks available in the system, `moving_from_ssd_to_hdd` policy can be used instead. The volume `hot` consists of an SSD disk (`fast_ssd`), and the maximum size of a part that can be stored on this volume is 1GB. All the parts with the size larger than 1GB will be stored directly on the `cold` volume, which contains an HDD disk `disk1`. Also, once the disk `fast_ssd` gets filled by more than 80%, data will be transferred to the `disk1` by a background process. The order of volume enumeration within a storage policy is important. Once a volume is overfilled, data are moved to the next one. The order of disk enumeration is important as well because data are stored on them in turns. @@ -568,12 +615,12 @@ The `default` storage policy implies using only one volume, which consists of on ### Details -In the case of MergeTree tables, data is getting to disk in different ways: +In the case of `MergeTree` tables, data is getting to disk in different ways: -* as a result of an insert (`INSERT` query). -* during background merges and [mutations](../../query_language/alter.md#alter-mutations). -* when downloading from another replica. -* as a result of partition freezing [ALTER TABLE ... FREEZE PARTITION](../../query_language/alter.md#alter_freeze-partition). +- As a result of an insert (`INSERT` query). +- During background merges and [mutations](../../query_language/alter.md#alter-mutations). +- When downloading from another replica. +- As a result of partition freezing [ALTER TABLE ... FREEZE PARTITION](../../query_language/alter.md#alter_freeze-partition). In all these cases except for mutations and partition freezing, a part is stored on a volume and a disk according to the given storage policy: @@ -592,3 +639,4 @@ Moving data does not interfere with data replication. Therefore, different stora After the completion of background merges and mutations, old parts are removed only after a certain amount of time (`old_parts_lifetime`). During this time, they are not moved to other volumes or disks. Therefore, until the parts are finally removed, they are still taken into account for evaluation of the occupied disk space. +[Original article](https://clickhouse.yandex/docs/ru/operations/table_engines/mergetree/) diff --git a/docs/en/operations/table_engines/replication.md b/docs/en/operations/table_engines/replication.md index 63589b73bf7..bbb21cc2e81 100644 --- a/docs/en/operations/table_engines/replication.md +++ b/docs/en/operations/table_engines/replication.md @@ -22,7 +22,14 @@ Compressed data for `INSERT` and `ALTER` queries is replicated (for more informa - The `DROP TABLE` query deletes the replica located on the server where the query is run. - The `RENAME` query renames the table on one of the replicas. In other words, replicated tables can have different names on different replicas. -To use replication, set the addresses of the ZooKeeper cluster in the config file. Example: +ClickHouse uses [Apache ZooKeeper](https://zookeeper.apache.org) for storing replicas meta information. Use ZooKeeper version 3.4.5 or newer. + +To use replication, set parameters in the [zookeeper](../server_settings/settings.md#server-settings_zookeeper) server configuration section. + +!!! attention "Attention" + Don't neglect the securiry setting. ClickHouse supports the `digest` [ACL scheme](https://zookeeper.apache.org/doc/current/zookeeperProgrammers.html#sc_ZooKeeperAccessControl) of the ZooKeeper security subsystem. + +Example of setting the addresses of the ZooKeeper cluster: ```xml @@ -40,9 +47,7 @@ To use replication, set the addresses of the ZooKeeper cluster in the config fil ``` - -Use ZooKeeper version 3.4.5 or later. - + You can specify any existing ZooKeeper cluster and the system will use a directory on it for its own data (the directory is specified when creating a replicatable table). If ZooKeeper isn't set in the config file, you can't create replicated tables, and any existing replicated tables will be read-only. diff --git a/docs/en/operations/table_engines/url.md b/docs/en/operations/table_engines/url.md index 6521604171c..cb7b57b35c3 100644 --- a/docs/en/operations/table_engines/url.md +++ b/docs/en/operations/table_engines/url.md @@ -17,6 +17,8 @@ additional headers for getting a response from the server. respectively. For processing `POST` requests, the remote server must support [Chunked transfer encoding](https://en.wikipedia.org/wiki/Chunked_transfer_encoding). +You can limit the maximum number of HTTP GET redirect hops by the [max_http_get_redirects](../settings/settings.md#setting-max_http_get_redirects) setting. + **Example:** **1.** Create a `url_engine_table` table on the server : diff --git a/docs/en/query_language/agg_functions/parametric_functions.md b/docs/en/query_language/agg_functions/parametric_functions.md index 13cbc2b05d8..8784811a311 100644 --- a/docs/en/query_language/agg_functions/parametric_functions.md +++ b/docs/en/query_language/agg_functions/parametric_functions.md @@ -73,7 +73,7 @@ In this case, you should remember that you don't know the histogram bin borders. ## sequenceMatch(pattern)(timestamp, cond1, cond2, ...) {#function-sequencematch} -Checks whether the sequence contains the event chain that matches the pattern. +Checks whether the sequence contains an event chain that matches the pattern. ```sql sequenceMatch(pattern)(timestamp, cond1, cond2, ...) @@ -87,9 +87,9 @@ sequenceMatch(pattern)(timestamp, cond1, cond2, ...) - `pattern` — Pattern string. See [Pattern syntax](#sequence-function-pattern-syntax). -- `timestamp` — Column that considered to contain time data. Typical data types are `Date`, and `DateTime`. You can use also any of the supported [UInt](../../data_types/int_uint.md) data types. +- `timestamp` — Column considered to contain time data. Typical data types are `Date` and `DateTime`. You can also use any of the supported [UInt](../../data_types/int_uint.md) data types. -- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. You can pass up to 32 condition arguments. The function takes into account only the events described in these conditions. If the sequence contains data that are not described with conditions the function skips them. +- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. You can pass up to 32 condition arguments. The function takes only the events described in these conditions into account. If the sequence contains data that isn't described in a condition, the function skips them. **Returned values** @@ -104,11 +104,11 @@ Type: `UInt8`. **Pattern syntax** -- `(?N)` — Matches the condition argument at the position `N`. Conditions are numbered in the `[1, 32]` range. For example, `(?1)` matches the argument passed to the `cond1` parameter. +- `(?N)` — Matches the condition argument at position `N`. Conditions are numbered in the `[1, 32]` range. For example, `(?1)` matches the argument passed to the `cond1` parameter. -- `.*` — Matches any number of any events. You don't need the conditional arguments to match this element of the pattern. +- `.*` — Matches any number of events. You don't need conditional arguments to match this element of the pattern. -- `(?t operator value)` — Sets the time in seconds that should separate two events. For example, pattern `(?1)(?t>1800)(?2)` matches events that distanced from each other for more than 1800 seconds. An arbitrary number of any events can lay between these events. You can use the `>=`, `>`, `<`, `<=` operators. +- `(?t operator value)` — Sets the time in seconds that should separate two events. For example, pattern `(?1)(?t>1800)(?2)` matches events that occur more than 1800 seconds from each other. An arbitrary number of any events can lay between these events. You can use the `>=`, `>`, `<`, `<=` operators. **Examples** @@ -133,7 +133,7 @@ SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2) FROM t └───────────────────────────────────────────────────────────────────────┘ ``` -The function has found the event chain where number 2 follows number 1. It skipped number 3 between them, because the number is not described as an event. If we want to take this number into account when searching for the event chain, showed in the example, we should make a condition for it. +The function found the event chain where number 2 follows number 1. It skipped number 3 between them, because the number is not described as an event. If we want to take this number into account when searching for the event chain given in the example, we should make a condition for it. ```sql SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 3) FROM t @@ -144,7 +144,7 @@ SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 3) FROM └──────────────────────────────────────────────────────────────────────────────────────────┘ ``` -In this case the function couldn't find the event chain matching the pattern, because there is the event for number 3 occured between 1 and 2. If in the same case we checked the condition for number 4, the sequence would match the pattern. +In this case, the function couldn't find the event chain matching the pattern, because the event for number 3 occured between 1 and 2. If in the same case we checked the condition for number 4, the sequence would match the pattern. ```sql SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 4) FROM t @@ -163,7 +163,7 @@ SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 4) FROM ## sequenceCount(pattern)(time, cond1, cond2, ...) {#function-sequencecount} -Counts the number of event chains that matched the pattern. The function searches event chains that not overlap. It starts to search for the next chain after the current chain is matched. +Counts the number of event chains that matched the pattern. The function searches event chains that don't overlap. It starts to search for the next chain after the current chain is matched. !!! warning "Warning" Events that occur at the same second may lay in the sequence in an undefined order affecting the result. @@ -176,14 +176,14 @@ sequenceCount(pattern)(timestamp, cond1, cond2, ...) - `pattern` — Pattern string. See [Pattern syntax](#sequence-function-pattern-syntax). -- `timestamp` — Column that considered to contain time data. Typical data types are `Date`, and `DateTime`. You can also use any of the supported [UInt](../../data_types/int_uint.md) data types. +- `timestamp` — Column considered to contain time data. Typical data types are `Date` and `DateTime`. You can also use any of the supported [UInt](../../data_types/int_uint.md) data types. -- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. You can pass up to 32 condition arguments. The function takes into account only the events described in these conditions. If the sequence contains data that are not described with conditions the function skips them. +- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. You can pass up to 32 condition arguments. The function takes only the events described in these conditions into account. If the sequence contains data that isn't described in a condition, the function skips them. **Returned values** -- Number of non-overlapping event chains that are matched +- Number of non-overlapping event chains that are matched. Type: `UInt64`. @@ -219,18 +219,20 @@ SELECT sequenceCount('(?1).*(?2)')(time, number = 1, number = 2) FROM t - [sequenceMatch](#function-sequencematch) -## windowFunnel(window)(timestamp, cond1, cond2, cond3, ...) +## windowFunnel(window, [mode])(timestamp, cond1, cond2, cond3, ...) Searches for event chains in a sliding time window and calculates the maximum number of events that occurred from the chain. ``` -windowFunnel(window)(timestamp, cond1, cond2, cond3, ...) +windowFunnel(window, [mode])(timestamp, cond1, cond2, cond3, ...) ``` **Parameters:** - `window` — Length of the sliding window in seconds. -- `timestamp` — Name of the column containing the timestamp. Data type support: `Date`,`DateTime`, and other unsigned integer types (note that though timestamp support `UInt64` type, there is a limitation it's value can't overflow maximum of Int64, which is 2^63 - 1). +- `mode` - It is an optional argument. + * `'strict'` - When the `'strict'` is set, the windowFunnel() applies conditions only for the unique values. +- `timestamp` — Name of the column containing the timestamp. Data types supported: `Date`, `DateTime`, and other unsigned integer types (note that even though timestamp supports the `UInt64` type, it's value can't exceed the Int64 maximum, which is 2^63 - 1). - `cond1`, `cond2`... — Conditions or data describing the chain of events. Data type: `UInt8`. Values can be 0 or 1. **Algorithm** diff --git a/docs/en/query_language/agg_functions/reference.md b/docs/en/query_language/agg_functions/reference.md index 31810390fe1..cee43211fcd 100644 --- a/docs/en/query_language/agg_functions/reference.md +++ b/docs/en/query_language/agg_functions/reference.md @@ -858,7 +858,7 @@ Don't use this function for calculating timings. There is a more suitable functi ## quantileTiming {#agg_function-quantiletiming} -Computes the quantile of the specified level with determined precision. The function is intended for calculating page loading time quantiles in milliseconds. +Computes the quantile of the specified level with determined precision. The function is intended for calculating page loading time quantiles in milliseconds. ```sql quantileTiming(level)(expr) @@ -868,7 +868,7 @@ quantileTiming(level)(expr) - `level` — Quantile level. Range: [0, 1]. - `expr` — [Expression](../syntax.md#syntax-expressions) returning a [Float*](../../data_types/float.md)-type number. The function expects input values in unix timestamp format in milliseconds, but it doesn't validate format. - + - If negative values are passed to the function, the behavior is undefined. - If the value is greater than 30,000 (a page loading time of more than 30 seconds), it is assumed to be 30,000. @@ -1007,6 +1007,16 @@ Calculates the value of `Σ((x - x̅)(y - y̅)) / n`. Calculates the Pearson correlation coefficient: `Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)^2) * Σ((y - y̅)^2))`. +## categoricalInformationValue + +Calculates the value of `(P(tag = 1) - P(tag = 0))(log(P(tag = 1)) - log(P(tag = 0)))` for each category. + +```sql +categoricalInformationValue(category1, category2, ..., tag) +``` + +The result indicates how a discrete (categorical) feature `[category1, category2, ...]` contribute to a learning model which predicting the value of `tag`. + ## simpleLinearRegression Performs simple (unidimensional) linear regression. @@ -1069,39 +1079,40 @@ stochasticLinearRegression(1.0, 1.0, 10, 'SGD') To predict we use function [evalMLMethod](../functions/machine_learning_functions.md#machine_learning_methods-evalmlmethod), which takes a state as an argument as well as features to predict on. -1. Fitting - Such query may be used. +**1.** Fitting - ```sql - CREATE TABLE IF NOT EXISTS train_data - ( - param1 Float64, - param2 Float64, - target Float64 - ) ENGINE = Memory; +Such query may be used. - CREATE TABLE your_model ENGINE = Memory AS SELECT - stochasticLinearRegressionState(0.1, 0.0, 5, 'SGD')(target, param1, param2) - AS state FROM train_data; +```sql +CREATE TABLE IF NOT EXISTS train_data +( + param1 Float64, + param2 Float64, + target Float64 +) ENGINE = Memory; - ``` +CREATE TABLE your_model ENGINE = Memory AS SELECT +stochasticLinearRegressionState(0.1, 0.0, 5, 'SGD')(target, param1, param2) +AS state FROM train_data; - Here we also need to insert data into `train_data` table. The number of parameters is not fixed, it depends only on number of arguments, passed into `linearRegressionState`. They all must be numeric values. - Note that the column with target value(which we would like to learn to predict) is inserted as the first argument. +``` -2. Predicting +Here we also need to insert data into `train_data` table. The number of parameters is not fixed, it depends only on number of arguments, passed into `linearRegressionState`. They all must be numeric values. +Note that the column with target value(which we would like to learn to predict) is inserted as the first argument. - After saving a state into the table, we may use it multiple times for prediction, or even merge with other states and create new even better models. +**2.** Predicting - ```sql - WITH (SELECT state FROM your_model) AS model SELECT - evalMLMethod(model, param1, param2) FROM test_data - ``` +After saving a state into the table, we may use it multiple times for prediction, or even merge with other states and create new even better models. - The query will return a column of predicted values. Note that first argument of `evalMLMethod` is `AggregateFunctionState` object, next are columns of features. +```sql +WITH (SELECT state FROM your_model) AS model SELECT +evalMLMethod(model, param1, param2) FROM test_data +``` - `test_data` is a table like `train_data` but may not contain target value. +The query will return a column of predicted values. Note that first argument of `evalMLMethod` is `AggregateFunctionState` object, next are columns of features. + +`test_data` is a table like `train_data` but may not contain target value. ### Notes {#agg_functions-stochasticlinearregression-notes} diff --git a/docs/en/query_language/alter.md b/docs/en/query_language/alter.md index f275a908fd9..e486e269865 100644 --- a/docs/en/query_language/alter.md +++ b/docs/en/query_language/alter.md @@ -26,10 +26,10 @@ These actions are described in detail below. #### ADD COLUMN {#alter_add-column} ```sql -ADD COLUMN [IF NOT EXISTS] name [type] [default_expr] [AFTER name_after] +ADD COLUMN [IF NOT EXISTS] name [type] [default_expr] [codec] [AFTER name_after] ``` -Adds a new column to the table with the specified `name`, `type`, and `default_expr` (see the section [Default expressions](create.md#create-default-values)). +Adds a new column to the table with the specified `name`, `type`, [`codec`](create.md#codecs) and `default_expr` (see the section [Default expressions](create.md#create-default-values)). If the `IF NOT EXISTS` clause is included, the query won't return an error if the column already exists. If you specify `AFTER name_after` (the name of another column), the column is added after the specified one in the list of table columns. Otherwise, the column is added to the end of the table. Note that there is no way to add a column to the beginning of a table. For a chain of actions, `name_after` can be the name of a column that is added in one of the previous actions. @@ -189,12 +189,13 @@ The following operations with [partitions](../operations/table_engines/custom_pa - [DETACH PARTITION](#alter_detach-partition) – Moves a partition to the `detached` directory and forget it. - [DROP PARTITION](#alter_drop-partition) – Deletes a partition. - [ATTACH PART|PARTITION](#alter_attach-partition) – Adds a part or partition from the `detached` directory to the table. -- [REPLACE PARTITION](#alter_replace-partition) - Copies the data partition from one table to another. +- [ATTACH PARTITION FROM](#alter_attach-partition-from) – Copies the data partition from one table to another and adds. +- [REPLACE PARTITION](#alter_replace-partition) - Copies the data partition from one table to another and replaces. - [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) - Resets the value of a specified column in a partition. - [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) - Resets the specified secondary index in a partition. - [FREEZE PARTITION](#alter_freeze-partition) – Creates a backup of a partition. - [FETCH PARTITION](#alter_fetch-partition) – Downloads a partition from another server. - +- [MOVE PARTITION|PART](#alter_move-partition) – Move partition/data part to another disk or volume. #### DETACH PARTITION {#alter_detach-partition} ```sql @@ -252,17 +253,30 @@ ALTER TABLE visits ATTACH PART 201901_2_2_0; Read more about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr). -This query is replicated. Each replica checks whether there is data in the `detached` directory. If the data is in this directory, the query checks the integrity, verifies that it matches the data on the server that initiated the query. If everything is correct, the query adds data to the replica. If not, it downloads data from the query requestor replica, or from another replica where the data has already been added. +This query is replicated. The replica-initiator checks whether there is data in the `detached` directory. If data exists, the query checks its integrity. If everything is correct, the query adds the data to the table. All other replicas download the data from the replica-initiator. So you can put data to the `detached` directory on one replica, and use the `ALTER ... ATTACH` query to add it to the table on all replicas. +#### ATTACH PARTITION FROM {#alter_attach-partition-from} + +```sql +ALTER TABLE table2 ATTACH PARTITION partition_expr FROM table1 +``` + +This query copies the data partition from the `table1` to `table2` adds data to exsisting in the `table2`. Note that data won't be deleted from `table1`. + +For the query to run successfully, the following conditions must be met: + +- Both tables must have the same structure. +- Both tables must have the same partition key. + #### REPLACE PARTITION {#alter_replace-partition} ```sql ALTER TABLE table2 REPLACE PARTITION partition_expr FROM table1 ``` -This query copies the data partition from the `table1` to `table2`. Note that data won't be deleted from `table1`. +This query copies the data partition from the `table1` to `table2` and replaces existing partition in the `table2`. Note that data won't be deleted from `table1`. For the query to run successfully, the following conditions must be met: @@ -291,16 +305,19 @@ ALTER TABLE table_name FREEZE [PARTITION partition_expr] This query creates a local backup of a specified partition. If the `PARTITION` clause is omitted, the query creates the backup of all partitions at once. -Note that for old-styled tables you can specify the prefix of the partition name (for example, '2019') - then the query creates the backup for all the corresponding partitions. Read about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr). - -!!! note +!!! note "Note" The entire backup process is performed without stopping the server. +Note that for old-styled tables you can specify the prefix of the partition name (for example, '2019') - then the query creates the backup for all the corresponding partitions. Read about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr). + At the time of execution, for a data snapshot, the query creates hardlinks to a table data. Hardlinks are placed in the directory `/var/lib/clickhouse/shadow/N/...`, where: - `/var/lib/clickhouse/` is the working ClickHouse directory specified in the config. - `N` is the incremental number of the backup. +!!! note "Note" + If you use [a set of disks for data storage in a table](../operations/table_engines/mergetree.md#table_engine-mergetree-multiple-volumes), the `shadow/N` directory appears on every disk, storing data parts that matched by the `PARTITION` expression. + The same structure of directories is created inside the backup as inside `/var/lib/clickhouse/`. The query performs 'chmod' for all files, forbidding writing into them. After creating the backup, you can copy the data from `/var/lib/clickhouse/shadow/` to the remote server and then delete it from the local server. Note that the `ALTER t FREEZE PARTITION` query is not replicated. It creates a local backup only on the local server. @@ -357,6 +374,25 @@ Although the query is called `ALTER TABLE`, it does not change the table structu #### MOVE PARTITION|PART {#alter_move-partition} +Moves partitions or data parts to another volume or disk for `MergeTree`-engine tables. See [Using Multiple Block Devices for Data Storage](../operations/table_engines/mergetree.md#table_engine-mergetree-multiple-volumes). + +```sql +ALTER TABLE table_name MOVE PARTITION|PART partition_expr TO DISK|VOLUME 'disk_name' +``` + +The `ALTER TABLE t MOVE` query: + +- Not replicated, because different replicas can have different storage policies. +- Returns an error if the specified disk or volume is not configured. Query also returns an error if conditions of data moving, that specified in the storage policy, can't be applied. +- Can return an error in the case, when data to be moved is already moved by a background process, concurrent `ALTER TABLE t MOVE` query or as a result of background data merging. A user shouldn't perform any additional actions in this case. + +Example: + +```sql +ALTER TABLE hits MOVE PART '20190301_14343_16206_438' TO VOLUME 'slow' +ALTER TABLE hits MOVE PARTITION '2019-09-01' TO DISK 'fast_ssd' +``` + #### How To Set Partition Expression {#alter-how-to-specify-part-expr} You can specify the partition expression in `ALTER ... PARTITION` queries in different ways: @@ -387,9 +423,7 @@ Possible values: `0` – do not wait; `1` – only wait for own execution (defau ### Mutations {#alter-mutations} -Mutations are an ALTER query variant that allows changing or deleting rows in a table. In contrast to standard `UPDATE` and `DELETE` queries that are intended for point data changes, mutations are intended for heavy operations that change a lot of rows in a table. - -Currently `*MergeTree` table engines are supported (both replicated and unreplicated). +Mutations are an ALTER query variant that allows changing or deleting rows in a table. In contrast to standard `UPDATE` and `DELETE` queries that are intended for point data changes, mutations are intended for heavy operations that change a lot of rows in a table. Supported for the `MergeTree` family of table engines including the engines with replication support. Existing tables are ready for mutations as-is (no conversion necessary), but after the first mutation is applied to a table, its metadata format becomes incompatible with previous server versions and falling back to a previous version becomes impossible. @@ -399,13 +433,13 @@ Currently available commands: ALTER TABLE [db.]table DELETE WHERE filter_expr ``` -The `filter_expr` must be of type UInt8. The query deletes rows in the table for which this expression takes a non-zero value. +The `filter_expr` must be of type `UInt8`. The query deletes rows in the table for which this expression takes a non-zero value. ```sql ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] WHERE filter_expr ``` -The `filter_expr` must be of type UInt8. This query updates values of specified columns to the values of corresponding expressions in rows for which the `filter_expr` takes a non-zero value. Values are casted to the column type using the `CAST` operator. Updating columns that are used in the calculation of the primary or the partition key is not supported. +The `filter_expr` must be of type `UInt8`. This query updates values of specified columns to the values of corresponding expressions in rows for which the `filter_expr` takes a non-zero value. Values are casted to the column type using the `CAST` operator. Updating columns that are used in the calculation of the primary or the partition key is not supported. ```sql ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name diff --git a/docs/en/query_language/create.md b/docs/en/query_language/create.md index 9a0f694fc42..3ca6249acfa 100644 --- a/docs/en/query_language/create.md +++ b/docs/en/query_language/create.md @@ -127,7 +127,7 @@ Adding large amount of constraints can negatively affect performance of big `INS Defines storage time for values. Can be specified only for MergeTree-family tables. For the detailed description, see [TTL for columns and tables](../operations/table_engines/mergetree.md#table_engine-mergetree-ttl). -### Column Compression Codecs +### Column Compression Codecs {#codecs} By default, ClickHouse applies the compression method, defined in [server settings](../operations/server_settings/settings.md#compression), to columns. You can also define the compression method for each individual column in the `CREATE TABLE` query. @@ -196,15 +196,16 @@ High compression levels are useful for asymmetric scenarios, like compress once, ClickHouse supports temporary tables which have the following characteristics: - Temporary tables disappear when the session ends, including if the connection is lost. -- A temporary table use the Memory engine only. +- A temporary table uses the Memory engine only. - The DB can't be specified for a temporary table. It is created outside of databases. +- Impossible to create a temporary table with distributed DDL query on all cluster servers (by using `ON CLUSTER`): this table exists only in the current session. - If a temporary table has the same name as another one and a query specifies the table name without specifying the DB, the temporary table will be used. - For distributed query processing, temporary tables used in a query are passed to remote servers. To create a temporary table, use the following syntax: ```sql -CREATE TEMPORARY TABLE [IF NOT EXISTS] table_name [ON CLUSTER cluster] +CREATE TEMPORARY TABLE [IF NOT EXISTS] table_name ( name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], @@ -214,6 +215,8 @@ CREATE TEMPORARY TABLE [IF NOT EXISTS] table_name [ON CLUSTER cluster] In most cases, temporary tables are not created manually, but when using external data for a query, or for distributed `(GLOBAL) IN`. For more information, see the appropriate sections +It's possible to use tables with [ENGINE = Memory](../operations/table_engines/memory.md) instead of temporary tables. + ## Distributed DDL queries (ON CLUSTER clause) The `CREATE`, `DROP`, `ALTER`, and `RENAME` queries support distributed execution on a cluster. @@ -225,7 +228,6 @@ CREATE TABLE IF NOT EXISTS all_hits ON CLUSTER cluster (p Date, i Int32) ENGINE In order to run these queries correctly, each host must have the same cluster definition (to simplify syncing configs, you can use substitutions from ZooKeeper). They must also connect to the ZooKeeper servers. The local version of the query will eventually be implemented on each host in the cluster, even if some hosts are currently not available. The order for executing queries within a single host is guaranteed. -`ALTER` queries are not yet supported for replicated tables. ## CREATE VIEW @@ -272,3 +274,27 @@ Views look the same as normal tables. For example, they are listed in the result There isn't a separate query for deleting views. To delete a view, use `DROP TABLE`. [Original article](https://clickhouse.yandex/docs/en/query_language/create/) + +## CREATE DICTIONARY {#create-dictionary-query} + +```sql +CREATE DICTIONARY [IF NOT EXISTS] [db.]dictionary_name +( + key1 type1 [DEFAULT|EXPRESSION expr1] [HIERARCHICAL|INJECTIVE|IS_OBJECT_ID], + key2 type2 [DEFAULT|EXPRESSION expr2] [HIERARCHICAL|INJECTIVE|IS_OBJECT_ID], + attr1 type2 [DEFAULT|EXPRESSION expr3], + attr2 type2 [DEFAULT|EXPRESSION expr4] +) +PRIMARY KEY key1, key2 +SOURCE(SOURCE_NAME([param1 value1 ... paramN valueN])) +LAYOUT(LAYOUT_NAME([param_name param_value])) +LIFETIME([MIN val1] MAX val2) +``` + +Creates [external dictionary](dicts/external_dicts.md) with given [structure](dicts/external_dicts_dict_structure.md), [source](dicts/external_dicts_dict_sources.md), [layout](dicts/external_dicts_dict_layout.md) and [lifetime](dicts/external_dicts_dict_lifetime.md). + +External dictionary structure consists of attributes. Dictionary attributes are specified similarly to table columns. The only required attribute property is its type, all other properties may have default values. + +Depending on dictionary [layout](dicts/external_dicts_dict_layout.md) one or more attributes can be specified as dictionary keys. + +For more information, see [External Dictionaries](dicts/external_dicts.md) section. diff --git a/docs/en/query_language/dicts/external_dicts.md b/docs/en/query_language/dicts/external_dicts.md index f547e8a4679..abae466fcb4 100644 --- a/docs/en/query_language/dicts/external_dicts.md +++ b/docs/en/query_language/dicts/external_dicts.md @@ -4,10 +4,11 @@ You can add your own dictionaries from various data sources. The data source for ClickHouse: -> - Fully or partially stores dictionaries in RAM. +- Fully or partially stores dictionaries in RAM. - Periodically updates dictionaries and dynamically loads missing values. In other words, dictionaries can be loaded dynamically. +- Allows to create external dictionaries with xml-files or [DDL queries](../create.md#create-dictionary-query). -The configuration of external dictionaries is located in one or more files. The path to the configuration is specified in the [dictionaries_config](../../operations/server_settings/settings.md#server_settings-dictionaries_config) parameter. +The configuration of external dictionaries can be located in one or more xml-files. The path to the configuration is specified in the [dictionaries_config](../../operations/server_settings/settings.md#server_settings-dictionaries_config) parameter. Dictionaries can be loaded at server startup or at first use, depending on the [dictionaries_lazy_load](../../operations/server_settings/settings.md#server_settings-dictionaries_lazy_load) setting. @@ -31,6 +32,8 @@ The dictionary configuration file has the following format: You can [configure](external_dicts_dict.md) any number of dictionaries in the same file. +[DDL queries for dictionaries](../create.md#create-dictionary-query) doesn't require any additional records in server configuration. They allow to work with dictionaries as first-class entities, like tables or views. + !!! attention You can convert values for a small dictionary by describing it in a `SELECT` query (see the [transform](../functions/other_functions.md) function). This functionality is not related to external dictionaries. diff --git a/docs/en/query_language/dicts/external_dicts_dict.md b/docs/en/query_language/dicts/external_dicts_dict.md index 05bb1eac5cf..9ff58c4f72d 100644 --- a/docs/en/query_language/dicts/external_dicts_dict.md +++ b/docs/en/query_language/dicts/external_dicts_dict.md @@ -1,11 +1,15 @@ # Configuring an External Dictionary {#dicts-external_dicts_dict} -The dictionary configuration has the following structure: +If dictionary is configured using xml-file, than dictionary configuration has the following structure: ```xml dict_name + + + + @@ -14,16 +18,25 @@ The dictionary configuration has the following structure: - - - - ``` +Corresponding [DDL-query](../create.md#create-dictionary-query) has the following structure: + +```sql +CREATE DICTIONARY dict_name +( + ... -- attributes +) +PRIMARY KEY ... -- complex or single key configuration +SOURCE(...) -- Source configuration +LAYOUT(...) -- Memory layout configuration +LIFETIME(...) -- Lifetime of dictionary in memory +``` + - name – The identifier that can be used to access the dictionary. Use the characters `[a-zA-Z0-9_\-]`. - [source](external_dicts_dict_sources.md) — Source of the dictionary. - [layout](external_dicts_dict_layout.md) — Dictionary layout in memory. diff --git a/docs/en/query_language/dicts/external_dicts_dict_layout.md b/docs/en/query_language/dicts/external_dicts_dict_layout.md index 15bb4850cb9..3da10ef660c 100644 --- a/docs/en/query_language/dicts/external_dicts_dict_layout.md +++ b/docs/en/query_language/dicts/external_dicts_dict_layout.md @@ -34,6 +34,15 @@ The configuration looks like this: ``` +in case of [DDL-query](../create.md#create-dictionary-query), equal configuration will looks like + +```sql +CREATE DICTIONARY (...) +... +LAYOUT(LAYOUT_TYPE(param value)) -- layout settings +... +``` + ## Ways to Store Dictionaries in Memory @@ -64,6 +73,12 @@ Configuration example: ``` +or + +```sql +LAYOUT(FLAT()) +``` + ### hashed {#dicts-external_dicts_dict_layout-hashed} The dictionary is completely stored in memory in the form of a hash table. The dictionary can contain any number of elements with any identifiers In practice, the number of keys can reach tens of millions of items. @@ -78,6 +93,12 @@ Configuration example: ``` +or + +```sql +LAYOUT(HASHED()) +``` + ### sparse_hashed {#dicts-external_dicts_dict_layout-sparse_hashed} Similar to `hashed`, but uses less memory in favor more CPU usage. @@ -90,6 +111,9 @@ Configuration example: ``` +```sql +LAYOUT(SPARSE_HASHED()) +``` ### complex_key_hashed @@ -103,6 +127,9 @@ Configuration example: ``` +```sql +LAYOUT(COMPLEX_KEY_HASHED()) +``` ### range_hashed @@ -113,15 +140,15 @@ This storage method works the same way as hashed and allows using date/time (arb Example: The table contains discounts for each advertiser in the format: ```text -+---------------+---------------------+-------------------+--------+ ++---------------|---------------------|-------------------|--------+ | advertiser id | discount start date | discount end date | amount | +===============+=====================+===================+========+ | 123 | 2015-01-01 | 2015-01-15 | 0.15 | -+---------------+---------------------+-------------------+--------+ ++---------------|---------------------|-------------------|--------+ | 123 | 2015-01-16 | 2015-01-31 | 0.25 | -+---------------+---------------------+-------------------+--------+ ++---------------|---------------------|-------------------|--------+ | 456 | 2015-01-01 | 2015-01-15 | 0.05 | -+---------------+---------------------+-------------------+--------+ ++---------------|---------------------|-------------------|--------+ ``` To use a sample for date ranges, define the `range_min` and `range_max` elements in the [structure](external_dicts_dict_structure.md). These elements must contain elements `name` and` type` (if `type` is not specified, the default type will be used - Date). `type` can be any numeric type (Date / DateTime / UInt64 / Int32 / others). @@ -144,6 +171,19 @@ Example: ... ``` +or + +```sql +CREATE DICTIONARY somedict ( + id UInt64, + first Date, + last Date +) +PRIMARY KEY id +LAYOUT(RANGE_HASHED()) +RANGE(MIN first MAX last) +``` + To work with these dictionaries, you need to pass an additional argument to the `dictGetT` function, for which a range is selected: ```sql @@ -193,6 +233,18 @@ Configuration example: ``` +or + +```sql +CREATE DICTIONARY somedict( + Abcdef UInt64, + StartTimeStamp UInt64, + EndTimeStamp UInt64, + XXXType String DEFAULT '' +) +PRIMARY KEY Abcdef +RANGE(MIN StartTimeStamp MAX EndTimeStamp) +``` ### cache @@ -218,6 +270,12 @@ Example of settings: ``` +or + +```sql +LAYOUT(CACHE(SIZE_IN_CELLS 1000000000)) +``` + Set a large enough cache size. You need to experiment to select the number of cells: 1. Set some value. @@ -241,17 +299,17 @@ This type of storage is for mapping network prefixes (IP addresses) to metadata Example: The table contains network prefixes and their corresponding AS number and country code: ```text - +-----------------+-------+--------+ + +-----------------|-------|--------+ | prefix | asn | cca2 | +=================+=======+========+ | 202.79.32.0/20 | 17501 | NP | - +-----------------+-------+--------+ + +-----------------|-------|--------+ | 2620:0:870::/48 | 3856 | US | - +-----------------+-------+--------+ + +-----------------|-------|--------+ | 2a02:6b8:1::/48 | 13238 | RU | - +-----------------+-------+--------+ + +-----------------|-------|--------+ | 2001:db8::/32 | 65536 | ZZ | - +-----------------+-------+--------+ + +-----------------|-------|--------+ ``` When using this type of layout, the structure must have a composite key. @@ -279,6 +337,17 @@ Example: ... ``` +or + +```sql +CREATE DICTIONARY somedict ( + prefix String, + asn UInt32, + cca2 String DEFAULT '??' +) +PRIMARY KEY prefix +``` + The key must have only one String type attribute that contains an allowed IP prefix. Other types are not supported yet. For queries, you must use the same functions (`dictGetT` with a tuple) as for dictionaries with composite keys: diff --git a/docs/en/query_language/dicts/external_dicts_dict_lifetime.md b/docs/en/query_language/dicts/external_dicts_dict_lifetime.md index cbd78da16ad..0cebf459698 100644 --- a/docs/en/query_language/dicts/external_dicts_dict_lifetime.md +++ b/docs/en/query_language/dicts/external_dicts_dict_lifetime.md @@ -15,7 +15,14 @@ Example of settings: ``` -Setting ` 0 ` prevents updating dictionaries. +```sql +CREATE DICTIONARY (...) +... +LIFETIME(300) +... +``` + +Setting `0` (`LIFETIME(0)`) prevents dictionaries from updating. You can set a time interval for upgrades, and ClickHouse will choose a uniformly random time within this range. This is necessary in order to distribute the load on the dictionary source when upgrading on a large number of servers. @@ -32,6 +39,12 @@ Example of settings: ``` +or + +```sql +LIFETIME(MIN 300 MAX 360) +``` + When upgrading the dictionaries, the ClickHouse server applies different logic depending on the type of [ source](external_dicts_dict_sources.md): - For a text file, it checks the time of modification. If the time differs from the previously recorded time, the dictionary is updated. @@ -56,5 +69,13 @@ Example of settings: ``` +or + +```sql +... +SOURCE(ODBC(... invalidate_query 'SELECT update_time FROM dictionary_source where id = 1')) +... +``` + [Original article](https://clickhouse.yandex/docs/en/query_language/dicts/external_dicts_dict_lifetime/) diff --git a/docs/en/query_language/dicts/external_dicts_dict_sources.md b/docs/en/query_language/dicts/external_dicts_dict_sources.md index d371fcab6a0..7b8303eb700 100644 --- a/docs/en/query_language/dicts/external_dicts_dict_sources.md +++ b/docs/en/query_language/dicts/external_dicts_dict_sources.md @@ -2,7 +2,7 @@ An external dictionary can be connected from many different sources. -The configuration looks like this: +If dictionary is configured using xml-file, the configuration looks like this: ```xml @@ -19,6 +19,15 @@ The configuration looks like this: ``` +In case of [DDL-query](../create.md#create-dictionary-query), equal configuration will looks like: + +```sql +CREATE DICTIONARY dict_name (...) +... +SOURCE(SOURCE_TYPE(param1 val1 ... paramN valN)) -- Source configuration +... +``` + The source is configured in the `source` section. Types of sources (`source_type`): @@ -47,6 +56,12 @@ Example of settings: ``` +or + +```sql +SOURCE(FILE(path '/opt/dictionaries/os.tsv' format 'TabSeparated')) +``` + Setting fields: - `path` – The absolute path to the file. @@ -68,6 +83,12 @@ Example of settings: ``` +or + +```sql +SOURCE(EXECUTABLE(command 'cat /opt/dictionaries/os.tsv' format 'TabSeparated')) +``` + Setting fields: - `command` – The absolute path to the executable file, or the file name (if the program directory is written to `PATH`). @@ -99,6 +120,17 @@ Example of settings: ``` +or + +```sql +SOURCE(HTTP( + url 'http://[::1]/os.tsv' + format 'TabSeparated' + credentials(user 'user' password 'password') + headers(header(name 'API-KEY' value 'key')) +)) +``` + In order for ClickHouse to access an HTTPS resource, you must [configure openSSL](../../operations/server_settings/settings.md#server_settings-openssl) in the server configuration. Setting fields: @@ -121,12 +153,25 @@ You can use this method to connect any database that has an ODBC driver. Example of settings: ```xml - - DatabaseName - ShemaName.TableName
- DSN=some_parameters - SQL_QUERY - + + + DatabaseName + ShemaName.TableName
+ DSN=some_parameters + SQL_QUERY + + +``` + +or + +```sql +SOURCE(ODBC( + db 'DatabaseName' + table 'SchemaName.TableName' + connection_string 'DSN=some_parameters' + invalidate_query 'SQL_QUERY' +)) ``` Setting fields: @@ -233,6 +278,19 @@ The dictionary configuration in ClickHouse: ``` +or + +```sql +CREATE DICTIONARY table_name ( + id UInt64, + some_column UInt64 DEFAULT 0 +) +PRIMARY KEY id +SOURCE(ODBC(connection_string 'DSN=myconnection' table 'postgresql_table')) +LAYOUT(HASHED()) +LIFETIME(MIN 300 MAX 360) +``` + You may need to edit `odbc.ini` to specify the full path to the library with the driver `DRIVER=/usr/local/lib/psqlodbcw.so`. ### Example of Connecting MS SQL Server @@ -316,6 +374,19 @@ Configuring the dictionary in ClickHouse: ``` +or + +```sql +CREATE DICTIONARY test ( + k UInt64, + s String DEFAULT '' +) +PRIMARY KEY k +SOURCE(ODBC(table 'dict' connection_string 'DSN=MSSQL;UID=test;PWD=test')) +LAYOUT(FLAT()) +LIFETIME(MIN 300 MAX 360) +``` + ## DBMS @@ -345,6 +416,22 @@ Example of settings: ``` +or + +```sql +SOURCE(MYSQL( + port 3306 + user 'clickhouse' + password 'qwerty' + replica(host 'example01-1' priority 1) + replica(host 'example01-2' priority 1) + db 'db_name' + table 'table_name' + where 'id=10' + invalidate_query 'SQL_QUERY' +)) +``` + Setting fields: - `port` – The port on the MySQL server. You can specify it for all replicas, or for each one individually (inside ``). @@ -385,6 +472,21 @@ Example of settings: ``` +or + +```sql +SOURCE(MYSQL( + host 'localhost' + socket '/path/to/socket/file.sock' + user 'clickhouse' + password 'qwerty' + db 'db_name' + table 'table_name' + where 'id=10' + invalidate_query 'SQL_QUERY' +)) +``` + ### ClickHouse {#dicts-external_dicts_dict_sources-clickhouse} @@ -404,6 +506,20 @@ Example of settings: ``` +or + +```sql +SOURCE(CLICKHOUSE( + host 'example01-01-1' + port 9000 + user 'default' + password '' + db 'default' + table 'ids' + where 'id=10' +)) +``` + Setting fields: - `host` – The ClickHouse host. If it is a local host, the query is processed without any network activity. To improve fault tolerance, you can create a [Distributed](../../operations/table_engines/distributed.md) table and enter it in subsequent configurations. @@ -433,6 +549,19 @@ Example of settings: ``` +or + +```sql +SOURCE(MONGO( + host 'localhost' + port 27017 + user '' + password '' + db 'test' + collection 'dictionary_source' +)) +``` + Setting fields: - `host` – The MongoDB host. @@ -458,6 +587,17 @@ Example of settings: ``` +or + +```sql +SOURCE(REDIS( + host 'localhost' + port 6379 + storage_type 'simple' + db_index 0 +)) +``` + Setting fields: - `host` – The Redis host. diff --git a/docs/en/query_language/dicts/external_dicts_dict_structure.md b/docs/en/query_language/dicts/external_dicts_dict_structure.md index d5377c39289..bc960f19596 100644 --- a/docs/en/query_language/dicts/external_dicts_dict_structure.md +++ b/docs/en/query_language/dicts/external_dicts_dict_structure.md @@ -22,23 +22,35 @@ Overall structure: ``` -Columns are described in the structure: +or + +```sql +CREATE DICTIONARY ( + Id UInt64, + -- attributes +) +PRIMARY KEY Id +... +``` + + +In xml-file attributes are described in the structure section: - `` — [Key column](external_dicts_dict_structure.md#ext_dict_structure-key). -- `` — [Data column](external_dicts_dict_structure.md#ext_dict_structure-attributes). There can be a large number of columns. +- `` — [Data column](external_dicts_dict_structure.md#ext_dict_structure-attributes). There can be a large number of attributes. +In DDL-query attributes are described the body of `CREATE` query: +- `PRIMARY KEY` — [Key column](external_dicts_dict_structure.md#ext_dict_structure-key) +- `AttrName AttrType` — [Data column](external_dicts_dict_structure.md#ext_dict_structure-attributes) ## Key {#ext_dict_structure-key} ClickHouse supports the following types of keys: -- Numeric key. UInt64. Defined in the tag `` . -- Composite key. Set of values of different types. Defined in the tag `` . +- Numeric key. UInt64. Defined in the `` tag or using `PRIMARY KEY` keyword. +- Composite key. Set of values of different types. Defined in the tag `` or `PRIMARY KEY` keyword. -A structure can contain either `` or `` . - -!!! warning - The key doesn't need to be defined separately in attributes. +A xml-structure can contain either `` or ``. DDL-query must contain single `PRIMARY KEY`. ### Numeric Key @@ -56,6 +68,20 @@ Configuration fields: - `name` – The name of the column with keys. + +For DDL-query: + +```sql +CREATE DICTIONARY ( + Id UInt64, + ... +) +PRIMARY KEY Id +... +``` + +- `PRIMARY KEY` – The name of the column with keys. + ### Composite Key The key can be a `tuple` from any types of fields. The [layout](external_dicts_dict_layout.md) in this case must be `complex_key_hashed` or `complex_key_cache`. @@ -81,6 +107,18 @@ The key structure is set in the element ``. Key fields are specified in the ... ``` +or + +```sql +CREATE DICTIONARY ( + field1 String, + field2 String + ... +) +PRIMARY KEY field1, field2 +... +``` + For a query to the `dictGet*` function, a tuple is passed as the key. Example: `dictGetString('dict_name', 'attr_name', tuple('string for field1', num_for_field2))`. @@ -103,6 +141,14 @@ Configuration example: ``` +or + +```sql +CREATE DICTIONARY somename ( + Name ClickHouseDataType DEFAULT '' EXPRESSION rand64() HIERARCHICAL INJECTIVE IS_OBJECT_ID +) +``` + Configuration fields: Tag | Description | Required diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md index a43f975254f..339cc04aa0a 100644 --- a/docs/en/query_language/functions/array_functions.md +++ b/docs/en/query_language/functions/array_functions.md @@ -34,9 +34,12 @@ Accepts zero arguments and returns an empty array of the appropriate type. Accepts an empty array and returns a one-element array that is equal to the default value. -## range(N) +## range(end), range(start, end [, step]) -Returns an array of numbers from 0 to N-1. +Returns an array of numbers from start to end-1 by step. +If the argument `start` is not specified, defaults to 0. +If the argument `step` is not specified, defaults to 1. +It behaviors almost like pythonic `range`. But the difference is that all the arguments type must be `UInt` numbers. Just in case, an exception is thrown if arrays with a total length of more than 100,000,000 elements are created in a data block. ## array(x1, ...), operator \[x1, ...\] @@ -377,7 +380,7 @@ arrayPushFront(array, single_value) **Example** ```sql -SELECT arrayPushBack(['b'], 'a') AS res +SELECT arrayPushFront(['b'], 'a') AS res ``` ```text ┌─res───────┐ @@ -679,7 +682,7 @@ SELECT arrayDifference([0, 10000000000000000000]) ## arrayDistinct(arr) {#array_functions-arraydistinct} -Takes an array, returns an array containing the distinct elements. +Takes an array, returns an array containing the distinct elements. Example: @@ -695,7 +698,7 @@ SELECT arrayDistinct([1, 2, 2, 3, 1]) ## arrayEnumerateDense(arr) {#array_functions-arrayenumeratedense} -Returns an array of the same size as the source array, indicating where each element first appears in the source array. +Returns an array of the same size as the source array, indicating where each element first appears in the source array. Example: @@ -789,5 +792,112 @@ SELECT arrayReverse([1, 2, 3]) Synonym for ["arrayReverse"](#array_functions-arrayreverse) +## arrayFlatten {#arrayflatten} + +Converts array of arrays to a flat array. + +Function: + +- Applies for any depth of nested arrays, but all the elements should lay at the same level. + + For example, the `[[[1]], [[2], [3]]]` array can be flattened, but the `[[1], [[2], [3]]]` array can't be flattened. + +- Does not change arrays that are already flat. + +The flattened array contains all the elements from all source arrays. + +**Syntax** + +```sql +flatten(array_of_arrays) +``` + +Alias: `flatten`. + + +**Parameters** + +- `array_of_arrays` — [Array](../../data_types/array.md) of arrays. For example, `[[1,2,3], [4,5]]`. + +**Examples** + +```sql +SELECT flatten([[[1]], [[2], [3]]]) +``` +```text +┌─flatten(array(array([1]), array([2], [3])))─┐ +│ [1,2,3] │ +└─────────────────────────────────────────────┘ +``` + +## arrayCompact {#arraycompact} + +Removes consecutive duplicate elements from an array. The order of result values is determined by the order in the source array. + +**Syntax** + +```sql +arrayCompact(arr) +``` + +**Parameters** + +`arr` — The [array](../../data_types/array.md) to inspect. + +**Returned value** + +The array without duplicate. + +Type: `Array`. + +**Example** + +Query: + +```sql +SELECT arrayCompact([1, 1, nan, nan, 2, 3, 3, 3]) +``` + +Result: + +```text +┌─arrayCompact([1, 1, nan, nan, 2, 3, 3, 3])─┐ +│ [1,nan,nan,2,3] │ +└────────────────────────────────────────────┘ +``` + +## arrayZip {#arrayzip} + +Combine multiple Array type columns into one Array[Tuple(...)] column + +**Syntax** + +```sql +arrayZip(arr1, arr2, ..., arrN) +``` + +**Parameters** + +`arr` — Any number of [array](../../data_types/array.md) type columns to combine. + +**Returned value** + +The result of Array[Tuple(...)] type after the combination of these arrays + +**Example** + +Query: + +```sql +SELECT arrayZip(['a', 'b', 'c'], ['d', 'e', 'f']); +``` + +Result: + +```text +┌─arrayZip(['a', 'b', 'c'], ['d', 'e', 'f'])─┐ +│ [('a','d'),('b','e'),('c','f')] │ +└────────────────────────────────────────────┘ +``` [Original article](https://clickhouse.yandex/docs/en/query_language/functions/array_functions/) diff --git a/docs/en/query_language/functions/bit_functions.md b/docs/en/query_language/functions/bit_functions.md index c08a80e2bbf..3c36a1b28bc 100644 --- a/docs/en/query_language/functions/bit_functions.md +++ b/docs/en/query_language/functions/bit_functions.md @@ -20,12 +20,183 @@ The result type is an integer with bits equal to the maximum bits of its argumen ## bitRotateRight(a, b) -## bitTest(a, b) +## bitTest {#bittest} -## bitTestAll(a, b) +Takes any integer and converts it into [binary form](https://en.wikipedia.org/wiki/Binary_number), returns the value of a bit at specified position. The countdown starts from 0 from the right to the left. -## bitTestAny(a, b) +**Syntax** +```sql +SELECT bitTest(number, index) +``` +**Parameters** + +- `number` – integer number. +- `index` – position of bit. + +**Returned values** + +Returns a value of bit at specified position. + +Type: `UInt8`. + +**Example** + +For example, the number 43 in base-2 (binary) numeral system is 101011. + +Query: + +```sql +SELECT bitTest(43, 1) +``` + +Result: + +```text +┌─bitTest(43, 1)─┐ +│ 1 │ +└────────────────┘ +``` + +Another example: + +Query: + +```sql +SELECT bitTest(43, 2) +``` + +Result: + +```text +┌─bitTest(43, 2)─┐ +│ 0 │ +└────────────────┘ +``` + +## bitTestAll {#bittestall} + +Returns result of [logical conjuction](https://en.wikipedia.org/wiki/Logical_conjunction) (AND operator) of all bits at given positions. The countdown starts from 0 from the right to the left. + +The conjuction for bitwise operations: + +0 AND 0 = 0 +0 AND 1 = 0 +1 AND 0 = 0 +1 AND 1 = 1 + +**Syntax** + +```sql +SELECT bitTestAll(number, index1, index2, index3, index4, ...) +``` + +**Parameters** + +- `number` – integer number. +- `index1`, `index2`, `index3`, `index4` – positions of bit. For example, for set of positions (`index1`, `index2`, `index3`, `index4`) is true if and only if all of its positions are true (`index1` ⋀ `index2`, ⋀ `index3` ⋀ `index4`). + +**Returned values** + +Returns result of logical conjuction. + +Type: `UInt8`. + +**Example** + +For example, the number 43 in base-2 (binary) numeral system is 101011. + +Query: + +```sql +SELECT bitTestAll(43, 0, 1, 3, 5) +``` + +Result: + +```text +┌─bitTestAll(43, 0, 1, 3, 5)─┐ +│ 1 │ +└────────────────────────────┘ +``` + +Another example: + +Query: + +```sql +SELECT bitTestAll(43, 0, 1, 3, 5, 2) +``` + +Result: + +```text +┌─bitTestAll(43, 0, 1, 3, 5, 2)─┐ +│ 0 │ +└───────────────────────────────┘ +``` + +## bitTestAny {#bittestany} + +Returns result of [logical disjunction](https://en.wikipedia.org/wiki/Logical_disjunction) (OR operator) of all bits at given positions. The countdown starts from 0 from the right to the left. + +The disjunction for bitwise operations: + +0 OR 0 = 0 +0 OR 1 = 1 +1 OR 0 = 1 +1 OR 1 = 1 + +**Syntax** + +```sql +SELECT bitTestAny(number, index1, index2, index3, index4, ...) +``` + +**Parameters** + +- `number` – integer number. +- `index1`, `index2`, `index3`, `index4` – positions of bit. + +**Returned values** + +Returns result of logical disjuction. + +Type: `UInt8`. + +**Example** + +For example, the number 43 in base-2 (binary) numeral system is 101011. + +Query: + +```sql +SELECT bitTestAny(43, 0, 2) +``` + +Result: + +```text +┌─bitTestAny(43, 0, 2)─┐ +│ 1 │ +└──────────────────────┘ +``` + +Another example: + +Query: + +```sql +SELECT bitTestAny(43, 4, 2) +``` + +Result: + +```text +┌─bitTestAny(43, 4, 2)─┐ +│ 0 │ +└──────────────────────┘ +``` [Original article](https://clickhouse.yandex/docs/en/query_language/functions/bit_functions/) diff --git a/docs/en/query_language/functions/bitmap_functions.md b/docs/en/query_language/functions/bitmap_functions.md index f50097b08cb..a4aba098dbb 100644 --- a/docs/en/query_language/functions/bitmap_functions.md +++ b/docs/en/query_language/functions/bitmap_functions.md @@ -46,7 +46,7 @@ bitmapToArray(bitmap) **Example** -``` sql +```sql SELECT bitmapToArray(bitmapBuild([1, 2, 3, 4, 5])) AS res ``` @@ -98,7 +98,7 @@ bitmapSubsetLimit(bitmap, range_start, limit) **Example** -``` sql +```sql SELECT bitmapToArray(bitmapSubsetLimit(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,100,200,500]), toUInt32(30), toUInt32(200))) AS res ``` @@ -130,7 +130,7 @@ Type: `UInt8`. **Example** -``` sql +```sql SELECT bitmapContains(bitmapBuild([1,5,7,9]), toUInt32(9)) AS res ``` ```text @@ -196,6 +196,108 @@ SELECT bitmapHasAll(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res ``` +## bitmapCardinality + +Retrun bitmap cardinality of type UInt64. + + +```sql +bitmapCardinality(bitmap) +``` + +**Parameters** + +- `bitmap` – bitmap object. + +**Example** + +```sql +SELECT bitmapCardinality(bitmapBuild([1, 2, 3, 4, 5])) AS res +``` + +```text +┌─res─┐ +│ 5 │ +└─────┘ +``` + +## bitmapMin + +Retrun the smallest value of type UInt64 in the set, UINT32_MAX if the set is empty. + + +``` +bitmapMin(bitmap) +``` + +**Parameters** + +- `bitmap` – bitmap object. + +**Example** + +```sql +SELECT bitmapMin(bitmapBuild([1, 2, 3, 4, 5])) AS res +``` + +``` +┌─res─┐ +│ 1 │ +└─────┘ +``` + +## bitmapMax + +Retrun the greatest value of type UInt64 in the set, 0 if the set is empty. + + +``` +bitmapMax(bitmap) +``` + +**Parameters** + +- `bitmap` – bitmap object. + +**Example** + +```sql +SELECT bitmapMax(bitmapBuild([1, 2, 3, 4, 5])) AS res +``` + +``` +┌─res─┐ +│ 5 │ +└─────┘ +``` + +## bitmapTransform + +Transform an array of values in a bitmap to another array of values, the result is a new bitmap. + + +``` +bitmapTransform(bitmap, from_array, to_array) +``` + +**Parameters** + +- `bitmap` – bitmap object. +- `from_array` – UInt32 array. For idx in range [0, from_array.size()), if bitmap contains from_array[idx], then replace it with to_array[idx]. Note that the result depends on array ordering if there are common elements between from_array and to_array. +- `to_array` – UInt32 array, its size shall be the same to from_array. + +**Example** + +```sql +SELECT bitmapToArray(bitmapTransform(bitmapBuild([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), cast([5,999,2] as Array(UInt32)), cast([2,888,20] as Array(UInt32)))) AS res +``` + +``` +┌─res───────────────────┐ +│ [1,3,4,6,7,8,9,10,20] │ +└───────────────────────┘ +``` + ## bitmapAnd Two bitmap and calculation, the result is a new bitmap. @@ -210,7 +312,7 @@ bitmapAnd(bitmap,bitmap) **Example** -``` sql +```sql SELECT bitmapToArray(bitmapAnd(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res ``` @@ -293,81 +395,6 @@ SELECT bitmapToArray(bitmapAndnot(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS └───────┘ ``` -## bitmapCardinality - -Retrun bitmap cardinality of type UInt64. - - -```sql -bitmapCardinality(bitmap) -``` - -**Parameters** - -- `bitmap` – bitmap object. - -**Example** - -```sql -SELECT bitmapCardinality(bitmapBuild([1, 2, 3, 4, 5])) AS res -``` - -```text -┌─res─┐ -│ 5 │ -└─────┘ -``` - -## bitmapMin - -Retrun the smallest value of type UInt64 in the set, UINT32_MAX if the set is empty. - - -``` -bitmapMin(bitmap) -``` - -**Parameters** - -- `bitmap` – bitmap object. - -**Example** - -``` sql -SELECT bitmapMin(bitmapBuild([1, 2, 3, 4, 5])) AS res -``` - -``` -┌─res─┐ -│ 1 │ -└─────┘ -``` - -## bitmapMax - -Retrun the greatest value of type UInt64 in the set, 0 if the set is empty. - - -``` -bitmapMax(bitmap) -``` - -**Parameters** - -- `bitmap` – bitmap object. - -**Example** - -``` sql -SELECT bitmapMax(bitmapBuild([1, 2, 3, 4, 5])) AS res -``` - -``` -┌─res─┐ -│ 5 │ -└─────┘ -``` - ## bitmapAndCardinality Two bitmap and calculation, return cardinality of type UInt64. diff --git a/docs/en/query_language/functions/date_time_functions.md b/docs/en/query_language/functions/date_time_functions.md index c22457d3fcc..26c44a6a391 100644 --- a/docs/en/query_language/functions/date_time_functions.md +++ b/docs/en/query_language/functions/date_time_functions.md @@ -334,7 +334,7 @@ For a time interval starting at 'StartTime' and continuing for 'Duration' second For example, `timeSlots(toDateTime('2012-01-01 12:20:00'), 600) = [toDateTime('2012-01-01 12:00:00'), toDateTime('2012-01-01 12:30:00')]`. This is necessary for searching for pageviews in the corresponding session. -## formatDateTime(Time, Format\[, Timezone\]) +## formatDateTime(Time, Format\[, Timezone\]) {#formatdatetime} Function formats a Time according given Format string. N.B.: Format is a constant expression, e.g. you can not have multiple formats for single result column. @@ -345,7 +345,7 @@ Supported modifiers for Format: | ----------- | -------- | --------------- | |%C|year divided by 100 and truncated to integer (00-99)|20 |%d|day of the month, zero-padded (01-31)|02 -|%D|Short MM/DD/YY date, equivalent to %m/%d/%y|01/02/2018| +|%D|Short MM/DD/YY date, equivalent to %m/%d/%y|01/02/18| |%e|day of the month, space-padded ( 1-31)| 2| |%F|short YYYY-MM-DD date, equivalent to %Y-%m-%d|2018-01-02 |%H|hour in 24h format (00-23)|22| diff --git a/docs/en/query_language/functions/encoding_functions.md b/docs/en/query_language/functions/encoding_functions.md index 74ef53f82f7..7cf93071b93 100644 --- a/docs/en/query_language/functions/encoding_functions.md +++ b/docs/en/query_language/functions/encoding_functions.md @@ -1,8 +1,11 @@ # Encoding functions +## char +Accepts multiple arguments of numberic types. Returns a string with the length as the number of passed arguments and each byte has the value of corresponding argument. + ## hex -Accepts arguments of types: `String`, `unsigned integer`, `Date`, or `DateTime`. Returns a string containing the argument's hexadecimal representation. Uses uppercase letters `A-F`. Does not use `0x` prefixes or `h` suffixes. For strings, all bytes are simply encoded as two hexadecimal numbers. Numbers are converted to big endian ("human readable") format. For numbers, older zeros are trimmed, but only by entire bytes. For example, `hex (1) = '01'`. `Date` is encoded as the number of days since the beginning of the Unix epoch. `DateTime` is encoded as the number of seconds since the beginning of the Unix epoch. +Accepts arguments of types: `String`, `unsigned integer`, `float`, `decimal`, `Date`, or `DateTime`. Returns a string containing the argument's hexadecimal representation. Uses uppercase letters `A-F`. Does not use `0x` prefixes or `h` suffixes. For strings, all bytes are simply encoded as two hexadecimal numbers. Numbers are converted to big endian ("human readable") format. For numbers, older zeros are trimmed, but only by entire bytes. For example, `hex (1) = '01'`. `Date` is encoded as the number of days since the beginning of the Unix epoch. `DateTime` is encoded as the number of seconds since the beginning of the Unix epoch. `float` and `decimal` is encoded as their hexadecimal representation in memory. ## unhex(str) diff --git a/docs/en/query_language/functions/hash_functions.md b/docs/en/query_language/functions/hash_functions.md index fde41d97da4..d98c56cd584 100644 --- a/docs/en/query_language/functions/hash_functions.md +++ b/docs/en/query_language/functions/hash_functions.md @@ -179,6 +179,8 @@ SELECT farmHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:0 Calculates [JavaHash](http://hg.openjdk.java.net/jdk8u/jdk8u/jdk/file/478a4add975b/src/share/classes/java/lang/String.java#l1452) from a string. This hash function is neither fast nor having a good quality. The only reason to use it is when this algorithm is already used in another system and you have to calculate exactly the same result. +**Syntax** + ```sql SELECT javaHash(''); ``` @@ -187,8 +189,6 @@ SELECT javaHash(''); A `Int32` data type hash value. -Type: `javaHash`. - **Example** Query: @@ -205,6 +205,42 @@ Result: └───────────────────────────┘ ``` +## javaHashUTF16LE {#javahashutf16le} + +Calculates [JavaHash](http://hg.openjdk.java.net/jdk8u/jdk8u/jdk/file/478a4add975b/src/share/classes/java/lang/String.java#l1452) from a string, assuming it contains bytes representing a string in UTF-16LE encoding. + +**Syntax** + +```sql +javaHashUTF16LE(stringUtf16le) +``` + +**Parameters** + +- `stringUtf16le` — a string in UTF-16LE encoding. + +**Returned value** + +A `Int32` data type hash value. + +**Example** + +Correct query with UTF-16LE encoded string. + +Query: + +```sql +SELECT javaHashUTF16LE(convertCharset('test', 'utf-8', 'utf-16le')) +``` + +Result: + +```text +┌─javaHashUTF16LE(convertCharset('test', 'utf-8', 'utf-16le'))─┐ +│ 3556498 │ +└──────────────────────────────────────────────────────────────┘ +``` + ## hiveHash {#hash_functions-hivehash} Calculates `HiveHash` from a string. diff --git a/docs/en/query_language/functions/higher_order_functions.md b/docs/en/query_language/functions/higher_order_functions.md index c114f912980..fdf39b4403f 100644 --- a/docs/en/query_language/functions/higher_order_functions.md +++ b/docs/en/query_language/functions/higher_order_functions.md @@ -16,12 +16,16 @@ A lambda function can't be omitted for the following functions: - [arrayMap](#higher_order_functions-array-map) - [arrayFilter](#higher_order_functions-array-filter) +- [arrayFill](#higher_order_functions-array-fill) +- [arrayReverseFill](#higher_order_functions-array-reverse-fill) +- [arraySplit](#higher_order_functions-array-split) +- [arrayReverseSplit](#higher_order_functions-array-reverse-split) - [arrayFirst](#higher_order_functions-array-first) - [arrayFirstIndex](#higher_order_functions-array-first-index) ### arrayMap(func, arr1, ...) {#higher_order_functions-array-map} -Returns an array obtained from the original application of the `func` function to each element in the `arr` array. +Returns an array obtained from the original application of the `func` function to each element in the `arr` array. Examples: @@ -79,6 +83,78 @@ SELECT Note that the first argument (lambda function) can't be omitted in the `arrayFilter` function. +### arrayFill(func, arr1, ...) {#higher_order_functions-array-fill} + +Scan through `arr1` from the first element to the last element and replace `arr1[i]` by `arr1[i - 1]` if `func` returns 0. The first element of `arr1` will not be replaced. + +Examples: + +```sql +SELECT arrayFill(x -> not isNull(x), [1, null, 3, 11, 12, null, null, 5, 6, 14, null, null]) AS res +``` + +```text +┌─res──────────────────────────────┐ +│ [1,1,3,11,12,12,12,5,6,14,14,14] │ +└──────────────────────────────────┘ +``` + +Note that the first argument (lambda function) can't be omitted in the `arrayFill` function. + +### arrayReverseFill(func, arr1, ...) {#higher_order_functions-array-reverse-fill} + +Scan through `arr1` from the last element to the first element and replace `arr1[i]` by `arr1[i + 1]` if `func` returns 0. The last element of `arr1` will not be replaced. + +Examples: + +```sql +SELECT arrayReverseFill(x -> not isNull(x), [1, null, 3, 11, 12, null, null, 5, 6, 14, null, null]) AS res +``` + +```text +┌─res────────────────────────────────┐ +│ [1,3,3,11,12,5,5,5,6,14,NULL,NULL] │ +└────────────────────────────────────┘ +``` + +Note that the first argument (lambda function) can't be omitted in the `arrayReverseFill` function. + +### arraySplit(func, arr1, ...) {#higher_order_functions-array-split} + +Split `arr1` into multiple arrays. When `func` returns something other than 0, the array will be split on the left hand side of the element. The array will not be split before the first element. + +Examples: + +```sql +SELECT arraySplit((x, y) -> y, [1, 2, 3, 4, 5], [1, 0, 0, 1, 0]) AS res +``` + +```text +┌─res─────────────┐ +│ [[1,2,3],[4,5]] │ +└─────────────────┘ +``` + +Note that the first argument (lambda function) can't be omitted in the `arraySplit` function. + +### arrayReverseSplit(func, arr1, ...) {#higher_order_functions-array-reverse-split} + +Split `arr1` into multiple arrays. When `func` returns something other than 0, the array will be split on the right hand side of the element. The array will not be split after the last element. + +Examples: + +```sql +SELECT arrayReverseSplit((x, y) -> y, [1, 2, 3, 4, 5], [1, 0, 0, 1, 0]) AS res +``` + +```text +┌─res───────────────┐ +│ [[1],[2,3,4],[5]] │ +└───────────────────┘ +``` + +Note that the first argument (lambda function) can't be omitted in the `arraySplit` function. + ### arrayCount(\[func,\] arr1, ...) {#higher_order_functions-array-count} Returns the number of elements in the arr array for which func returns something other than 0. If 'func' is not specified, it returns the number of non-zero elements in the array. @@ -139,7 +215,7 @@ SELECT arrayCumSumNonNegative([1, 1, -4, 1]) AS res ### arraySort(\[func,\] arr1, ...) -Returns an array as result of sorting the elements of `arr1` in ascending order. If the `func` function is specified, sorting order is determined by the result of the function `func` applied to the elements of array (arrays) +Returns an array as result of sorting the elements of `arr1` in ascending order. If the `func` function is specified, sorting order is determined by the result of the function `func` applied to the elements of array (arrays) The [Schwartzian transform](https://en.wikipedia.org/wiki/Schwartzian_transform) is used to improve sorting efficiency. diff --git a/docs/en/query_language/functions/introspection.md b/docs/en/query_language/functions/introspection.md new file mode 100644 index 00000000000..520c89feaeb --- /dev/null +++ b/docs/en/query_language/functions/introspection.md @@ -0,0 +1,298 @@ +# Introspection Functions + +You can use functions described in this chapter to introspect [ELF](https://en.wikipedia.org/wiki/Executable_and_Linkable_Format) and [DWARF](https://en.wikipedia.org/wiki/DWARF) for query profiling. + +!!! warning "Warning" + These functions are slow and may impose security considerations. + +For proper operation of introspection functions: + +- Install the `clickhouse-common-static-dbg` package. +- Set the [allow_introspection_functions](../../operations/settings/settings.md#settings-allow_introspection_functions) setting to 1. + + For security reasons introspection functions are disabled by default. + +ClickHouse saves profiler reports to the [trace_log](../../operations/system_tables.md#system_tables-trace_log) system table. Make sure the table and profiler are configured properly. + +## addressToLine {#addresstoline} + +Converts virtual memory address inside ClickHouse server process to the filename and the line number in ClickHouse source code. + +If you use official ClickHouse packages, you need to install the `clickhouse-common-static-dbg` package. + +**Syntax** + +```sql +addressToLine(address_of_binary_instruction) +``` + +**Parameters** + +- `address_of_binary_instruction` ([UInt64](../../data_types/int_uint.md)) — Address of instruction in a running process. + +**Returned value** + +- Source code filename and the line number in this file delimited by colon. + + For example, `/build/obj-x86_64-linux-gnu/../dbms/src/Common/ThreadPool.cpp:199`, where `199` is a line number. + +- Name of a binary, if the function couldn't find the debug information. +- Empty string, if the address is not valid. + +Type: [String](../../data_types/string.md). + +**Example** + +Enabling introspection functions: + +```sql +SET allow_introspection_functions=1 +``` + +Selecting the first string from the `trace_log` system table: + +```sql +SELECT * FROM system.trace_log LIMIT 1 \G +``` +```text +Row 1: +────── +event_date: 2019-11-19 +event_time: 2019-11-19 18:57:23 +revision: 54429 +timer_type: Real +thread_number: 48 +query_id: 421b6855-1858-45a5-8f37-f383409d6d72 +trace: [140658411141617,94784174532828,94784076370703,94784076372094,94784076361020,94784175007680,140658411116251,140658403895439] +``` + +The `trace` field contains the stack trace at the moment of sampling. + +Getting the source code filename and the line number for a single address: + +```sql +SELECT addressToLine(94784076370703) \G +``` +```text +Row 1: +────── +addressToLine(94784076370703): /build/obj-x86_64-linux-gnu/../dbms/src/Common/ThreadPool.cpp:199 +``` + +Applying the function to the whole stack trace: + +```sql +SELECT + arrayStringConcat(arrayMap(x -> addressToLine(x), trace), '\n') AS trace_source_code_lines +FROM system.trace_log +LIMIT 1 +\G +``` + +The [arrayMap](higher_order_functions.md#higher_order_functions-array-map) function allows to process each individual element of the `trace` array by the `addressToLine` function. The result of this processing you see in the `trace_source_code_lines` column of output. + +```text +Row 1: +────── +trace_source_code_lines: /lib/x86_64-linux-gnu/libpthread-2.27.so +/usr/lib/debug/usr/bin/clickhouse +/build/obj-x86_64-linux-gnu/../dbms/src/Common/ThreadPool.cpp:199 +/build/obj-x86_64-linux-gnu/../dbms/src/Common/ThreadPool.h:155 +/usr/include/c++/9/bits/atomic_base.h:551 +/usr/lib/debug/usr/bin/clickhouse +/lib/x86_64-linux-gnu/libpthread-2.27.so +/build/glibc-OTsEL5/glibc-2.27/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:97 +``` + +## addressToSymbol {#addresstosymbol} + +Converts virtual memory address inside ClickHouse server process to the symbol from ClickHouse object files. + + +**Syntax** + +```sql +addressToSymbol(address_of_binary_instruction) +``` + +**Parameters** + +- `address_of_binary_instruction` ([UInt64](../../data_types/int_uint.md)) — Address of instruction in a running process. + +**Returned value** + +- Symbol from ClickHouse object files. +- Empty string, if the address is not valid. + +Type: [String](../../data_types/string.md). + +**Example** + +Enabling introspection functions: + +```sql +SET allow_introspection_functions=1 +``` + +Selecting the first string from the `trace_log` system table: + +```sql +SELECT * FROM system.trace_log LIMIT 1 \G +``` +```text +Row 1: +────── +event_date: 2019-11-20 +event_time: 2019-11-20 16:57:59 +revision: 54429 +timer_type: Real +thread_number: 48 +query_id: 724028bf-f550-45aa-910d-2af6212b94ac +trace: [94138803686098,94138815010911,94138815096522,94138815101224,94138815102091,94138814222988,94138806823642,94138814457211,94138806823642,94138814457211,94138806823642,94138806795179,94138806796144,94138753770094,94138753771646,94138753760572,94138852407232,140399185266395,140399178045583] +``` + +The `trace` field contains the stack trace at the moment of sampling. + +Getting a symbol for a single address: + +```sql +SELECT addressToSymbol(94138803686098) \G +``` +```text +Row 1: +────── +addressToSymbol(94138803686098): _ZNK2DB24IAggregateFunctionHelperINS_20AggregateFunctionSumImmNS_24AggregateFunctionSumDataImEEEEE19addBatchSinglePlaceEmPcPPKNS_7IColumnEPNS_5ArenaE +``` + +Applying the function to the whole stack trace: + +```sql +SELECT + arrayStringConcat(arrayMap(x -> addressToSymbol(x), trace), '\n') AS trace_symbols +FROM system.trace_log +LIMIT 1 +\G +``` + +The [arrayMap](higher_order_functions.md#higher_order_functions-array-map) function allows to process each individual element of the `trace` array by the `addressToSymbols` function. The result of this processing you see in the `trace_symbols` column of output. + + +```text +Row 1: +────── +trace_symbols: _ZNK2DB24IAggregateFunctionHelperINS_20AggregateFunctionSumImmNS_24AggregateFunctionSumDataImEEEEE19addBatchSinglePlaceEmPcPPKNS_7IColumnEPNS_5ArenaE +_ZNK2DB10Aggregator21executeWithoutKeyImplERPcmPNS0_28AggregateFunctionInstructionEPNS_5ArenaE +_ZN2DB10Aggregator14executeOnBlockESt6vectorIN3COWINS_7IColumnEE13immutable_ptrIS3_EESaIS6_EEmRNS_22AggregatedDataVariantsERS1_IPKS3_SaISC_EERS1_ISE_SaISE_EERb +_ZN2DB10Aggregator14executeOnBlockERKNS_5BlockERNS_22AggregatedDataVariantsERSt6vectorIPKNS_7IColumnESaIS9_EERS6_ISB_SaISB_EERb +_ZN2DB10Aggregator7executeERKSt10shared_ptrINS_17IBlockInputStreamEERNS_22AggregatedDataVariantsE +_ZN2DB27AggregatingBlockInputStream8readImplEv +_ZN2DB17IBlockInputStream4readEv +_ZN2DB26ExpressionBlockInputStream8readImplEv +_ZN2DB17IBlockInputStream4readEv +_ZN2DB26ExpressionBlockInputStream8readImplEv +_ZN2DB17IBlockInputStream4readEv +_ZN2DB28AsynchronousBlockInputStream9calculateEv +_ZNSt17_Function_handlerIFvvEZN2DB28AsynchronousBlockInputStream4nextEvEUlvE_E9_M_invokeERKSt9_Any_data +_ZN14ThreadPoolImplI20ThreadFromGlobalPoolE6workerESt14_List_iteratorIS0_E +_ZZN20ThreadFromGlobalPoolC4IZN14ThreadPoolImplIS_E12scheduleImplIvEET_St8functionIFvvEEiSt8optionalImEEUlvE1_JEEEOS4_DpOT0_ENKUlvE_clEv +_ZN14ThreadPoolImplISt6threadE6workerESt14_List_iteratorIS0_E +execute_native_thread_routine +start_thread +clone +``` + +## demangle {#demangle} + +Converts a symbol that you can get using the [addressToSymbol](#addresstosymbol) function to the C++ function name. + + +**Syntax** + +```sql +demangle(symbol) +``` + +**Parameters** + +- `symbol` ([String](../../data_types/string.md)) — Symbol from an object file. + +**Returned value** + +- Name of the C++ function. +- Empty string if a symbol is not valid. + +Type: [String](../../data_types/string.md). + +**Example** + +Enabling introspection functions: + +```sql +SET allow_introspection_functions=1 +``` + +Selecting the first string from the `trace_log` system table: + +```sql +SELECT * FROM system.trace_log LIMIT 1 \G +``` +```text +Row 1: +────── +event_date: 2019-11-20 +event_time: 2019-11-20 16:57:59 +revision: 54429 +timer_type: Real +thread_number: 48 +query_id: 724028bf-f550-45aa-910d-2af6212b94ac +trace: [94138803686098,94138815010911,94138815096522,94138815101224,94138815102091,94138814222988,94138806823642,94138814457211,94138806823642,94138814457211,94138806823642,94138806795179,94138806796144,94138753770094,94138753771646,94138753760572,94138852407232,140399185266395,140399178045583] +``` + +The `trace` field contains the stack trace at the moment of sampling. + +Getting a function name for a single address: + +```sql +SELECT demangle(addressToSymbol(94138803686098)) \G +``` +```text +Row 1: +────── +demangle(addressToSymbol(94138803686098)): DB::IAggregateFunctionHelper > >::addBatchSinglePlace(unsigned long, char*, DB::IColumn const**, DB::Arena*) const +``` + +Applying the function to the whole stack trace: + +```sql +SELECT + arrayStringConcat(arrayMap(x -> demangle(addressToSymbol(x)), trace), '\n') AS trace_functions +FROM system.trace_log +LIMIT 1 +\G +``` + +The [arrayMap](higher_order_functions.md#higher_order_functions-array-map) function allows to process each individual element of the `trace` array by the `demangle` function. The result of this processing you see in the `trace_functions` column of output. + +```text +Row 1: +────── +trace_functions: DB::IAggregateFunctionHelper > >::addBatchSinglePlace(unsigned long, char*, DB::IColumn const**, DB::Arena*) const +DB::Aggregator::executeWithoutKeyImpl(char*&, unsigned long, DB::Aggregator::AggregateFunctionInstruction*, DB::Arena*) const +DB::Aggregator::executeOnBlock(std::vector::immutable_ptr, std::allocator::immutable_ptr > >, unsigned long, DB::AggregatedDataVariants&, std::vector >&, std::vector >, std::allocator > > >&, bool&) +DB::Aggregator::executeOnBlock(DB::Block const&, DB::AggregatedDataVariants&, std::vector >&, std::vector >, std::allocator > > >&, bool&) +DB::Aggregator::execute(std::shared_ptr const&, DB::AggregatedDataVariants&) +DB::AggregatingBlockInputStream::readImpl() +DB::IBlockInputStream::read() +DB::ExpressionBlockInputStream::readImpl() +DB::IBlockInputStream::read() +DB::ExpressionBlockInputStream::readImpl() +DB::IBlockInputStream::read() +DB::AsynchronousBlockInputStream::calculate() +std::_Function_handler::_M_invoke(std::_Any_data const&) +ThreadPoolImpl::worker(std::_List_iterator) +ThreadFromGlobalPool::ThreadFromGlobalPool::scheduleImpl(std::function, int, std::optional)::{lambda()#3}>(ThreadPoolImpl::scheduleImpl(std::function, int, std::optional)::{lambda()#3}&&)::{lambda()#1}::operator()() const +ThreadPoolImpl::worker(std::_List_iterator) +execute_native_thread_routine +start_thread +clone +``` diff --git a/docs/en/query_language/functions/json_functions.md b/docs/en/query_language/functions/json_functions.md index 6ab942bd012..eeb41870112 100644 --- a/docs/en/query_language/functions/json_functions.md +++ b/docs/en/query_language/functions/json_functions.md @@ -206,4 +206,16 @@ Example: SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = '[-100, 200.0, 300]' ``` +## JSONExtractArrayRaw(json[, indices_or_keys]...) + +Returns an array with elements of JSON array, each represented as unparsed string. + +If the part does not exist or isn't array, an empty array will be returned. + +Example: + +```sql +SELECT JSONExtractArrayRaw('{"a": "hello", "b": [-100, 200.0, "hello"]}', 'b') = ['-100', '200.0', '"hello"']' +``` + [Original article](https://clickhouse.yandex/docs/en/query_language/functions/json_functions/) diff --git a/docs/en/query_language/functions/other_functions.md b/docs/en/query_language/functions/other_functions.md index f6139741849..394cd78c0f3 100644 --- a/docs/en/query_language/functions/other_functions.md +++ b/docs/en/query_language/functions/other_functions.md @@ -4,8 +4,39 @@ Returns a string with the name of the host that this function was performed on. For distributed processing, this is the name of the remote server host, if the function is performed on a remote server. -## FQDN(), fullHostName() -Returns the Fully qualified domain name aka [FQDN](https://en.wikipedia.org/wiki/Fully_qualified_domain_name). +## FQDN {#fqdn} + +Returns the fully qualified domain name. + +**Syntax** + +```sql +fqdn(); +``` + +This function is case-insensitive. + +**Returned value** + +- String with the fully qualified domain name. + +Type: `String`. + +**Example** + +Query: + +```sql +SELECT FQDN(); +``` + +Result: + +```text +┌─FQDN()──────────────────────────┐ +│ clickhouse.ru-central1.internal │ +└─────────────────────────────────┘ +``` ## basename diff --git a/docs/en/query_language/functions/rounding_functions.md b/docs/en/query_language/functions/rounding_functions.md index ea2d899b13c..3fe58a05c46 100644 --- a/docs/en/query_language/functions/rounding_functions.md +++ b/docs/en/query_language/functions/rounding_functions.md @@ -2,25 +2,29 @@ ## floor(x\[, N\]) -Returns the largest round number that is less than or equal to x. A round number is a multiple of 1/10N, or the nearest number of the appropriate data type if 1 / 10N isn't exact. +Returns the largest round number that is less than or equal to `x`. A round number is a multiple of 1/10N, or the nearest number of the appropriate data type if 1 / 10N isn't exact. 'N' is an integer constant, optional parameter. By default it is zero, which means to round to an integer. 'N' may be negative. Examples: `floor(123.45, 1) = 123.4, floor(123.45, -1) = 120.` `x` is any numeric type. The result is a number of the same type. -For integer arguments, it makes sense to round with a negative 'N' value (for non-negative 'N', the function doesn't do anything). +For integer arguments, it makes sense to round with a negative `N` value (for non-negative `N`, the function doesn't do anything). If rounding causes overflow (for example, floor(-128, -1)), an implementation-specific result is returned. ## ceil(x\[, N\]), ceiling(x\[, N\]) -Returns the smallest round number that is greater than or equal to 'x'. In every other way, it is the same as the 'floor' function (see above). +Returns the smallest round number that is greater than or equal to `x`. In every other way, it is the same as the `floor` function (see above). -## round(x[, N]) {#rounding_functions-round} +## trunc(x\[, N\]), truncate(x\[, N\]) + +Returns the round number with largest absolute value that has an absolute value less than or equal to `x`'s. In every other way, it is the same as the 'floor' function (see above). + +## round(x\[, N\]) {#rounding_functions-round} Rounds a value to a specified number of decimal places. -The function returns the nearest number of the specified order. In case when given number has equal distance to surrounding numbers the function returns the number having the nearest even digit (banker's rounding). +The function returns the nearest number of the specified order. In case when given number has equal distance to surrounding numbers, the function uses banker's rounding for float number types and rounds away from zero for the other number types. ```sql round(expression [, decimal_places]) @@ -91,3 +95,9 @@ Accepts a number. If the number is less than 18, it returns 0. Otherwise, it rou Accept a number, round it down to an element in the specified array. If the value is less than the lowest bound, the lowest bound is returned. [Original article](https://clickhouse.yandex/docs/en/query_language/functions/rounding_functions/) + +## roundBankers(x\[, N\]) + +Rounds a value to a specified number of decimal places. + +The function returns the nearest number of the specified order. In case when given number has equal distance to surrounding numbers, the function always return the number having the nearest even digit (banker's rounding). diff --git a/docs/en/query_language/functions/string_functions.md b/docs/en/query_language/functions/string_functions.md index 32186bfb74e..33e5700f355 100644 --- a/docs/en/query_language/functions/string_functions.md +++ b/docs/en/query_language/functions/string_functions.md @@ -85,6 +85,42 @@ SELECT toValidUTF8('\x61\xF0\x80\x80\x80b') └───────────────────────┘ ``` +## repeat {#repeat} + +Repeats a string as many times as specified and concatenates the replicated values as a single string. + +**Syntax** + +```sql +repeat(s, n) +``` + +**Parameters** + +- `s` — The string to repeat. [String](../../data_types/string.md). +- `n` — The number of times to repeat the string. [UInt](../../data_types/int_uint.md). + +**Returned value** + +The single string, which contains the string `s` repeated `n` times. If `n` < 1, the function returns empty string. + +Type: `String`. + +**Example** + +Query: + +```sql +SELECT repeat('abc', 10) +``` + +Result: + +```text +┌─repeat('abc', 10)──────────────┐ +│ abcabcabcabcabcabcabcabcabcabc │ +└────────────────────────────────┘ +``` ## reverse @@ -181,17 +217,119 @@ Result: └───────────────────────────────────┘ ``` -## trimLeft(s) +## trimLeft {#trimleft} -Returns a string that removes the whitespace characters on left side. +Removes all consecutive occurrences of common whitespace (ASCII character 32) from the beginning of a string. It doesn't remove other kinds of whitespace characters (tab, no-break space, etc.). -## trimRight(s) +**Syntax** -Returns a string that removes the whitespace characters on right side. +```sql +trimLeft() +``` -## trimBoth(s) +Alias: `ltrim`. -Returns a string that removes the whitespace characters on either side. +**Parameters** + +- `string` — string to trim. [String](../../data_types/string.md). + +**Returned value** + +A string without leading common whitespaces. + +Type: `String`. + +**Example** + +Query: + +```sql +SELECT trimLeft(' Hello, world! ') +``` + +Result: + +```text +┌─trimLeft(' Hello, world! ')─┐ +│ Hello, world! │ +└─────────────────────────────────────┘ +``` + +## trimRight {#trimright} + +Removes all consecutive occurrences of common whitespace (ASCII character 32) from the end of a string. It doesn't remove other kinds of whitespace characters (tab, no-break space, etc.). + +**Syntax** + +```sql +trimRight() +``` + +Alias: `rtrim`. + +**Parameters** + +- `string` — string to trim. [String](../../data_types/string.md). + +**Returned value** + +A string without trailing common whitespaces. + +Type: `String`. + +**Example** + +Query: + +```sql +SELECT trimRight(' Hello, world! ') +``` + +Result: + +```text +┌─trimRight(' Hello, world! ')─┐ +│ Hello, world! │ +└──────────────────────────────────────┘ +``` + +## trimBoth {#trimboth} + +Removes all consecutive occurrences of common whitespace (ASCII character 32) from both ends of a string. It doesn't remove other kinds of whitespace characters (tab, no-break space, etc.). + +**Syntax** + +```sql +trimBoth() +``` + +Alias: `trim`. + +**Parameters** + +- `string` — string to trim. [String](../../data_types/string.md). + +**Returned value** + +A string without leading and trailing common whitespaces. + +Type: `String`. + +**Example** + +Query: + +```sql +SELECT trimBoth(' Hello, world! ') +``` + +Result: + +```text +┌─trimBoth(' Hello, world! ')─┐ +│ Hello, world! │ +└─────────────────────────────────────┘ +``` ## CRC32(s) diff --git a/docs/en/query_language/functions/type_conversion_functions.md b/docs/en/query_language/functions/type_conversion_functions.md index f37130ecf33..ec0ddbe3e9d 100644 --- a/docs/en/query_language/functions/type_conversion_functions.md +++ b/docs/en/query_language/functions/type_conversion_functions.md @@ -40,8 +40,36 @@ SELECT toInt64(nan), toInt32(32), toInt16('16'), toInt8(8.8) ## toInt(8|16|32|64)OrZero +It takes an argument of type String and tries to parse it into Int (8 | 16 | 32 | 64). If failed, returns 0. + +**Example** + +```sql +select toInt64OrZero('123123'), toInt8OrZero('123qwe123') +``` +```text +┌─toInt64OrZero('123123')─┬─toInt8OrZero('123qwe123')─┐ +│ 123123 │ 0 │ +└─────────────────────────┴───────────────────────────┘ +``` + + ## toInt(8|16|32|64)OrNull +It takes an argument of type String and tries to parse it into Int (8 | 16 | 32 | 64). If failed, returns NULL. + +**Example** + +```sql +select toInt64OrNull('123123'), toInt8OrNull('123qwe123') +``` +```text +┌─toInt64OrNull('123123')─┬─toInt8OrNull('123qwe123')─┐ +│ 123123 │ ᴺᵁᴸᴸ │ +└─────────────────────────┴───────────────────────────┘ +``` + + ## toUInt(8|16|32|64) Converts an input value to the [UInt](../../data_types/int_uint.md) data type. This function family includes: @@ -321,10 +349,32 @@ SELECT toTypeName(CAST(x, 'Nullable(UInt16)')) FROM t_null └─────────────────────────────────────────┘ ``` -## toInterval(Year|Quarter|Month|Week|Day|Hour|Minute|Second) +## toInterval(Year|Quarter|Month|Week|Day|Hour|Minute|Second) {#function-tointerval} -Converts a Number type argument to a Interval type (duration). -The interval type is actually very useful, you can use this type of data to perform arithmetic operations directly with Date or DateTime. At the same time, ClickHouse provides a more convenient syntax for declaring Interval type data. For example: +Converts a Number type argument to an [Interval](../../data_types/special_data_types/interval.md) data type. + +**Syntax** + +```sql +toIntervalSecond(number) +toIntervalMinute(number) +toIntervalHour(number) +toIntervalDay(number) +toIntervalWeek(number) +toIntervalMonth(number) +toIntervalQuarter(number) +toIntervalYear(number) +``` + +**Parameters** + +- `number` — Duration of interval. Positive integer number. + +**Returned values** + +- The value in `Interval` data type. + +**Example** ```sql WITH diff --git a/docs/en/query_language/insert_into.md b/docs/en/query_language/insert_into.md index 2b361fd5a18..3383e90620e 100644 --- a/docs/en/query_language/insert_into.md +++ b/docs/en/query_language/insert_into.md @@ -61,10 +61,10 @@ However, you can delete old data using `ALTER TABLE ... DROP PARTITION`. ### Performance Considerations -`INSERT` sorts the input data by primary key and splits them into partitions by month. If you insert data for mixed months, it can significantly reduce the performance of the `INSERT` query. To avoid this: +`INSERT` sorts the input data by primary key and splits them into partitions by a partition key. If you insert data into several partitions at once, it can significantly reduce the performance of the `INSERT` query. To avoid this: - Add data in fairly large batches, such as 100,000 rows at a time. -- Group data by month before uploading it to ClickHouse. +- Group data by a partition key before uploading it to ClickHouse. Performance will not decrease if: diff --git a/docs/en/query_language/misc.md b/docs/en/query_language/misc.md index 9bcac5cdbfd..a9d8e7339e9 100644 --- a/docs/en/query_language/misc.md +++ b/docs/en/query_language/misc.md @@ -101,10 +101,17 @@ DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] Deletes the table. If `IF EXISTS` is specified, it doesn't return an error if the table doesn't exist or the database doesn't exist. +``` +DROP DICTIONARY [IF EXISTS] [db.]name +``` + +Delets the dictionary. +If `IF EXISTS` is specified, it doesn't return an error if the table doesn't exist or the database doesn't exist. + ## EXISTS ```sql -EXISTS [TEMPORARY] TABLE [db.]name [INTO OUTFILE filename] [FORMAT format] +EXISTS [TEMPORARY] [TABLE|DICTIONARY] [db.]name [INTO OUTFILE filename] [FORMAT format] ``` Returns a single `UInt8`-type column, which contains the single value `0` if the table or database doesn't exist, or `1` if the table exists in the specified database. @@ -174,7 +181,7 @@ Changes already made by the mutation are not rolled back. ## OPTIMIZE {#misc_operations-optimize} ```sql -OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION ID 'partition_id'] [FINAL] +OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION ID 'partition_id'] [FINAL] [DEDUPLICATE] ``` This query tries to initialize an unscheduled merge of data parts for tables with a table engine from the [MergeTree](../operations/table_engines/mergetree.md) family. Other kinds of table engines aren't supported. @@ -184,6 +191,7 @@ When `OPTIMIZE` is used with the [ReplicatedMergeTree](../operations/table_engin - If `OPTIMIZE` doesn't perform a merge for any reason, it doesn't notify the client. To enable notifications, use the [optimize_throw_if_noop](../operations/settings/settings.md#setting-optimize_throw_if_noop) setting. - If you specify a `PARTITION`, only the specified partition is optimized. [How to set partition expression](alter.md#alter-how-to-specify-part-expr). - If you specify `FINAL`, optimization is performed even when all the data is already in one part. +- If you specify `DEDUPLICATE`, then completely identical rows will be deduplicated (all columns are compared), it makes sense only for the MergeTree engine. !!! warning "Warning" `OPTIMIZE` can't fix the "Too many parts" error. diff --git a/docs/en/query_language/operators.md b/docs/en/query_language/operators.md index 0a7a81550a2..571685e61d0 100644 --- a/docs/en/query_language/operators.md +++ b/docs/en/query_language/operators.md @@ -1,71 +1,73 @@ # Operators -All operators are transformed to the corresponding functions at the query parsing stage, in accordance with their precedence and associativity. +All operators are transformed to their corresponding functions at the query parsing stage in accordance with their precedence and associativity. Groups of operators are listed in order of priority (the higher it is in the list, the earlier the operator is connected to its arguments). ## Access Operators -`a[N]` Access to an element of an array; ` arrayElement(a, N) function`. +`a[N]` – Access to an element of an array. The `arrayElement(a, N)` function. -`a.N` – Access to a tuble element; `tupleElement(a, N)` function. +`a.N` – Access to a tuple element. The `tupleElement(a, N)` function. ## Numeric Negation Operator -`-a` – The `negate (a)` function. +`-a` – The `negate (a)` function. ## Multiplication and Division Operators -`a * b` – The `multiply (a, b) function.` +`a * b` – The `multiply (a, b)` function. -`a / b` – The ` divide(a, b) function.` +`a / b` – The `divide(a, b)` function. -`a % b` – The `modulo(a, b) function.` +`a % b` – The `modulo(a, b)` function. ## Addition and Subtraction Operators -`a + b` – The `plus(a, b) function.` +`a + b` – The `plus(a, b)` function. -`a - b` – The `minus(a, b) function.` +`a - b` – The `minus(a, b)` function. ## Comparison Operators -`a = b` – The `equals(a, b) function.` +`a = b` – The `equals(a, b)` function. -`a == b` – The ` equals(a, b) function.` +`a == b` – The `equals(a, b)` function. -`a != b` – The `notEquals(a, b) function.` +`a != b` – The `notEquals(a, b)` function. -`a <> b` – The `notEquals(a, b) function.` +`a <> b` – The `notEquals(a, b)` function. -`a <= b` – The `lessOrEquals(a, b) function.` +`a <= b` – The `lessOrEquals(a, b)` function. -`a >= b` – The `greaterOrEquals(a, b) function.` +`a >= b` – The `greaterOrEquals(a, b)` function. -`a < b` – The `less(a, b) function.` +`a < b` – The `less(a, b)` function. -`a > b` – The `greater(a, b) function.` +`a > b` – The `greater(a, b)` function. -`a LIKE s` – The `like(a, b) function.` +`a LIKE s` – The `like(a, b)` function. -`a NOT LIKE s` – The `notLike(a, b) function.` +`a NOT LIKE s` – The `notLike(a, b)` function. -`a BETWEEN b AND c` – The same as `a >= b AND a <= c.` +`a BETWEEN b AND c` – The same as `a >= b AND a <= c`. -`a NOT BETWEEN b AND c` – The same as `a < b OR a > c.` +`a NOT BETWEEN b AND c` – The same as `a < b OR a > c`. ## Operators for Working With Data Sets -*See the section [IN operators](select.md#select-in-operators).* +*See [IN operators](select.md#select-in-operators).* -`a IN ...` – The `in(a, b) function` +`a IN ...` – The `in(a, b)` function. -`a NOT IN ...` – The `notIn(a, b) function.` +`a NOT IN ...` – The `notIn(a, b)` function. -`a GLOBAL IN ...` – The `globalIn(a, b) function.` +`a GLOBAL IN ...` – The `globalIn(a, b)` function. -`a GLOBAL NOT IN ...` – The `globalNotIn(a, b) function.` +`a GLOBAL NOT IN ...` – The `globalNotIn(a, b)` function. -## Operator for Working With Dates and Times {#operators-datetime} +## Operators for Working with Dates and Times {#operators-datetime} + +### EXTRACT {#operator-extract} ```sql EXTRACT(part FROM date); @@ -120,7 +122,6 @@ SELECT FROM test.Orders; ``` ```text - ┌─OrderYear─┬─OrderMonth─┬─OrderDay─┬─OrderHour─┬─OrderMinute─┬─OrderSecond─┐ │ 2008 │ 10 │ 11 │ 13 │ 23 │ 44 │ └───────────┴────────────┴──────────┴───────────┴─────────────┴─────────────┘ @@ -128,21 +129,54 @@ FROM test.Orders; You can see more examples in [tests](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00619_extract.sql). +### INTERVAL {#operator-interval} + +Creates an [Interval](../data_types/special_data_types/interval.md)-type value that should be used in arithmetical operations with [Date](../data_types/date.md) and [DateTime](../data_types/datetime.md)-type values. + +Types of intervals: +- `SECOND` +- `MINUTE` +- `HOUR` +- `DAY` +- `WEEK` +- `MONTH` +- `QUARTER` +- `YEAR` + +!!! warning "Warning" + Intervals with different types can't be combined. You can't use expressions like `INTERVAL 4 DAY 1 HOUR`. Express intervals in units that are smaller or equal the the smallest unit of the interval, for example `INTERVAL 25 HOUR`. You can use consequtive operations like in the example below. + +Example: + +```sql +SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL 3 HOUR +``` +```text +┌───current_date_time─┬─plus(plus(now(), toIntervalDay(4)), toIntervalHour(3))─┐ +│ 2019-10-23 11:16:28 │ 2019-10-27 14:16:28 │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +``` + +**See Also** + +- [Interval](../data_types/special_data_types/interval.md) data type +- [toInterval](functions/type_conversion_functions.md#function-tointerval) type convertion functions + ## Logical Negation Operator -`NOT a` The `not(a) function.` +`NOT a` – The `not(a)` function. ## Logical AND Operator -`a AND b` – The`and(a, b) function.` +`a AND b` – The`and(a, b)` function. ## Logical OR Operator -`a OR b` – The `or(a, b) function.` +`a OR b` – The `or(a, b)` function. ## Conditional Operator -`a ? b : c` – The `if(a, b, c) function.` +`a ? b : c` – The `if(a, b, c)` function. Note: diff --git a/docs/en/query_language/select.md b/docs/en/query_language/select.md index 8ffc4b8efdc..cfa3a1e4dc7 100644 --- a/docs/en/query_language/select.md +++ b/docs/en/query_language/select.md @@ -112,8 +112,20 @@ In contrast to standard SQL, a synonym does not need to be specified after a sub To execute a query, all the columns listed in the query are extracted from the appropriate table. Any columns not needed for the external query are thrown out of the subqueries. If a query does not list any columns (for example, `SELECT count() FROM t`), some column is extracted from the table anyway (the smallest one is preferred), in order to calculate the number of rows. -The `FINAL` modifier can be used in the `SELECT` select query for engines from the [MergeTree](../operations/table_engines/mergetree.md) family. When you specify `FINAL`, data is selected fully "merged". Keep in mind that using `FINAL` leads to reading columns related to the primary key, in addition to the columns specified in the query. Additionally, the query will be executed in a single thread, and data will be merged during query execution. This means that when using `FINAL`, the query is processed slowly. In the most cases, avoid using `FINAL`. -The `FINAL` modifier can be applied for all engines of MergeTree family that do data transformations in background merges (except GraphiteMergeTree). +#### FINAL Modifier {#select-from-final} + +Appliable when selecting data from tables of the [MergeTree](../operations/table_engines/mergetree.md)-engine family, except `GraphiteMergeTree`. When `FINAL` is specified, ClickHouse fully merges data before returning the result and thus performs all data transformations that are supposed to happen during merges for given table engine. + +Also supported for: +- [Replicated](../operations/table_engines/replication.md) versions of `MergeTree` engines. +- [View](../operations/table_engines/view.md), [Buffer](../operations/table_engines/buffer.md), [Distributed](../operations/table_engines/distributed.md), [MaterializedView](../operations/table_engines/materializedview.md) engines that operate over other engines, if they created over `MergeTree`-engine tables. + +The queries that use `FINAL` are executed slower than similar queries that don't, because: + +- Query is executed in a single thread, and data is merged during query execution. +- Queries with `FINAL` read primary key columns additionally to the columns specified in the query. + +In the most cases, avoid using `FINAL`. ### SAMPLE Clause {#select-sample-clause} @@ -964,11 +976,11 @@ External sorting works much less effectively than sorting in RAM. ### SELECT Clause {#select-select} -[Expressions](syntax.md#syntax-expressions) that specified in the `SELECT` clause are analyzed after the calculations for all the clauses listed above are completed. More specifically, expressions are analyzed that are above the aggregate functions, if there are any aggregate functions. The aggregate functions and everything below them are calculated during aggregation (`GROUP BY`). These expressions work as if they are applied to separate rows in the result. +[Expressions](syntax.md#syntax-expressions) specified in the `SELECT` clause are calculated after all the operations in the clauses described above are finished. These expressions work as if they apply to separate rows in the result. If expressions in the `SELECT` clause contain aggregate functions, then ClickHouse processes aggregate functions and expressions used as their arguments during the [GROUP BY](#select-group-by-clause) aggregation. -If you want to get all columns in the result, use the asterisk (`*`) symbol. For example, `SELECT * FROM ...`. +If you want to include all columns in the result, use the asterisk (`*`) symbol. For example, `SELECT * FROM ...`. -To match some columns in the result by a [re2](https://en.wikipedia.org/wiki/RE2_(software)) regular expression, you can use the `COLUMNS` expression. +To match some columns in the result with a [re2](https://en.wikipedia.org/wiki/RE2_(software)) regular expression, you can use the `COLUMNS` expression. ```sql COLUMNS('regexp') @@ -991,7 +1003,9 @@ SELECT COLUMNS('a') FROM col_names └────┴────┘ ``` -You can use multiple `COLUMNS` expressions in a query, also you can apply functions to it. +The selected columns are returned not in the alphabetical order. + +You can use multiple `COLUMNS` expressions in a query and apply functions to them. For example: @@ -1004,7 +1018,7 @@ SELECT COLUMNS('a'), COLUMNS('c'), toTypeName(COLUMNS('c')) FROM col_names └────┴────┴────┴────────────────┘ ``` -Be careful when using functions because the `COLUMN` expression returns variable number of columns, and, if a function doesn't support this number of arguments, ClickHouse throws an exception. +Each column returned by the `COLUMNS` expression is passed to the function as a separate argument. Also you can pass other arguments to the function if it supports them. Be careful when using functions. If a function doesn't support the number of arguments you have passed to it, ClickHouse throws an exception. For example: @@ -1016,9 +1030,9 @@ Received exception from server (version 19.14.1): Code: 42. DB::Exception: Received from localhost:9000. DB::Exception: Number of arguments for function plus doesn't match: passed 3, should be 2. ``` -In this example, `COLUMNS('a')` returns two columns `aa`, `ab`, and `COLUMNS('c')` returns the `bc` column. The `+` operator can't apply to 3 arguments, so ClickHouse throws an exception with the message about it. +In this example, `COLUMNS('a')` returns two columns: `aa` and `ab`. `COLUMNS('c')` returns the `bc` column. The `+` operator can't apply to 3 arguments, so ClickHouse throws an exception with the relevant message. -Columns that matched by the `COLUMNS` expression can be in different types. If `COLUMNS` doesn't match any columns and it is the single expression in `SELECT`, ClickHouse throws an exception. +Columns that matched the `COLUMNS` expression can have different data types. If `COLUMNS` doesn't match any columns and is the only expression in `SELECT`, ClickHouse throws an exception. ### DISTINCT Clause {#select-distinct} diff --git a/docs/en/query_language/show.md b/docs/en/query_language/show.md index 04f373a31a9..5c03185ffa9 100644 --- a/docs/en/query_language/show.md +++ b/docs/en/query_language/show.md @@ -3,10 +3,10 @@ ## SHOW CREATE TABLE ```sql -SHOW CREATE [TEMPORARY] TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] +SHOW CREATE [TEMPORARY] [TABLE|DICTIONARY] [db.]table [INTO OUTFILE filename] [FORMAT format] ``` -Returns a single `String`-type 'statement' column, which contains a single value – the `CREATE` query used for creating the specified table. +Returns a single `String`-type 'statement' column, which contains a single value – the `CREATE` query used for creating the specified object. ## SHOW DATABASES {#show-databases} @@ -17,33 +17,15 @@ SHOW DATABASES [INTO OUTFILE filename] [FORMAT format] Prints a list of all databases. This query is identical to `SELECT name FROM system.databases [INTO OUTFILE filename] [FORMAT format]`. -See also the section "Formats". - ## SHOW PROCESSLIST ```sql SHOW PROCESSLIST [INTO OUTFILE filename] [FORMAT format] ``` -Outputs a list of queries currently being processed, other than `SHOW PROCESSLIST` queries. +Outputs the content of the [system.processes](../operations/system_tables.md#system_tables-processes) table, that contains a list of queries that is being processed at the moment, excepting `SHOW PROCESSLIST` queries. -Prints a table containing the columns: - -**user** – The user who made the query. Keep in mind that for distributed processing, queries are sent to remote servers under the 'default' user. SHOW PROCESSLIST shows the username for a specific query, not for a query that this query initiated. - -**address** – The name of the host that the query was sent from. For distributed processing, on remote servers, this is the name of the query requestor host. To track where a distributed query was originally made from, look at SHOW PROCESSLIST on the query requestor server. - -**elapsed** – The execution time, in seconds. Queries are output in order of decreasing execution time. - -**rows_read**, **bytes_read** – How many rows and bytes of uncompressed data were read when processing the query. For distributed processing, data is totaled from all the remote servers. This is the data used for restrictions and quotas. - -**memory_usage** – Current RAM usage in bytes. See the setting 'max_memory_usage'. - -**query** – The query itself. In INSERT queries, the data for insertion is not output. - -**query_id** – The query identifier. Non-empty only if it was explicitly defined by the user. For distributed processing, the query ID is not passed to remote servers. - -This query is nearly identical to: `SELECT * FROM system.processes`. The difference is that the `SHOW PROCESSLIST` query does not show itself in a list, when the `SELECT .. FROM system.processes` query does. +The `SELECT * FROM system.processes` query returns data about all the current queries. Tip (execute in the console): @@ -61,7 +43,7 @@ SHOW [TEMPORARY] TABLES [FROM ] [LIKE ''] [LIMIT ] [INTO OUTFILE If the `FROM` clause is not specified, the query returns the list of tables from the current database. -The same result as the `SHOW TABLES` query returns, you can get by the following way: +You can get the same results as the `SHOW TABLES` query in the following way: ```sql SELECT name FROM system.tables WHERE database = [AND name LIKE ] [LIMIT ] [INTO OUTFILE ] [FORMAT ] @@ -80,3 +62,33 @@ SHOW TABLES FROM system LIKE '%co%' LIMIT 2 │ collations │ └────────────────────────────────┘ ``` + +## SHOW DICTIONARIES + +Displays a list of [external dictionaries](dicts/external_dicts.md). + +```sql +SHOW DICTIONARIES [FROM ] [LIKE ''] [LIMIT ] [INTO OUTFILE ] [FORMAT ] +``` + +If the `FROM` clause is not specified, the query returns the list of dictionaries from the current database. + +You can get the same results as the `SHOW DICTIONARIES` query in the following way: + +```sql +SELECT name FROM system.dictionaries WHERE database = [AND name LIKE ] [LIMIT ] [INTO OUTFILE ] [FORMAT ] +``` + +**Example** + +The following query selects the first two rows from the list of tables in the `system` database, whose names contain `co`. + +```sql +SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2 +``` +```text +┌─name─────────┐ +│ regions │ +│ region_names │ +└──────────────┘ +``` diff --git a/docs/en/query_language/system.md b/docs/en/query_language/system.md index 3ef504e46b3..0b08183afe8 100644 --- a/docs/en/query_language/system.md +++ b/docs/en/query_language/system.md @@ -3,7 +3,7 @@ - [RELOAD DICTIONARIES](#query_language-system-reload-dictionaries) - [RELOAD DICTIONARY](#query_language-system-reload-dictionary) - [DROP DNS CACHE](#query_language-system-drop-dns-cache) -- [DROP MARKS CACHE](#query_language-system-drop-marks-cache) +- [DROP MARK CACHE](#query_language-system-drop-mark-cache) - [FLUSH LOGS](#query_language-system-flush_logs) - [RELOAD CONFIG](#query_language-system-reload-config) - [SHUTDOWN](#query_language-system-shutdown) @@ -11,6 +11,8 @@ - [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends) - [FLUSH DISTRIBUTED](#query_language-system-flush-distributed) - [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) +- [STOP MERGES](#query_language-system-stop-merges) +- [START MERGES](#query_language-system-start-merges) ## RELOAD DICTIONARIES {#query_language-system-reload-dictionaries} @@ -34,7 +36,7 @@ Resets ClickHouse's internal DNS cache. Sometimes (for old ClickHouse versions) For more convenient (automatic) cache management, see disable_internal_dns_cache, dns_cache_update_period parameters. -## DROP MARKS CACHE {#query_language-system-drop-marks-cache} +## DROP MARK CACHE {#query_language-system-drop-mark-cache} Resets the mark cache. Used in development of ClickHouse and performance tests. @@ -85,4 +87,24 @@ Enables background data distribution when inserting data into distributed tables SYSTEM START DISTRIBUTED SENDS [db.] ``` + +### STOP MERGES {#query_language-system-stop-merges} + +Provides possibility to stop background merges for tables in the MergeTree family: + +```sql +SYSTEM STOP MERGES [[db.]merge_tree_family_table_name] +``` +!!! note "Note" + `DETACH / ATTACH` table will start background merges for the table even in case when merges have been stopped for all MergeTree tables before. + + +### START MERGES {#query_language-system-start-merges} + +Provides possibility to start background merges for tables in the MergeTree family: + +```sql +SYSTEM START MERGES [[db.]merge_tree_family_table_name] +``` + [Original article](https://clickhouse.yandex/docs/en/query_language/system/) diff --git a/docs/en/security_changelog.md b/docs/en/security_changelog.md index 83eb55237fc..0847300cc19 100644 --- a/docs/en/security_changelog.md +++ b/docs/en/security_changelog.md @@ -1,3 +1,10 @@ +## Fixed in ClickHouse Release 19.13.6.1, 2019-09-20 + +### CVE-2019-18657 +Table function `url` had the vulnerability allowed the attacker to inject arbitrary HTTP headers in the request. + +Credits: [Nikita Tikhomirov](https://github.com/NSTikhomirov) + ## Fixed in ClickHouse Release 18.12.13, 2018-09-10 ### CVE-2018-14672 diff --git a/docs/fa/data_types/special_data_types/interval.md b/docs/fa/data_types/special_data_types/interval.md new file mode 120000 index 00000000000..6829f5ced00 --- /dev/null +++ b/docs/fa/data_types/special_data_types/interval.md @@ -0,0 +1 @@ +../../../en/data_types/special_data_types/interval.md \ No newline at end of file diff --git a/docs/fa/development/developer_instruction.md b/docs/fa/development/developer_instruction.md new file mode 120000 index 00000000000..bdfa9047aa2 --- /dev/null +++ b/docs/fa/development/developer_instruction.md @@ -0,0 +1 @@ +../../en/development/developer_instruction.md \ No newline at end of file diff --git a/docs/fa/getting_started/index.md b/docs/fa/getting_started/index.md index 778393aed91..57496c474e2 100644 --- a/docs/fa/getting_started/index.md +++ b/docs/fa/getting_started/index.md @@ -1,197 +1,11 @@
+# ﻥﺪﺷ ﻉﻭﺮﺷ -# شروع به کار +ﻖﯾﺮﻃ ﺯﺍ ﺪﯾﺎﺑ ﻪﻤﻫ ﺯﺍ ﻝﻭﺍ ، ﺪﯿﻨﮐ ﺱﺎﺴﺣﺍ ﺍﺭ ﻥﺁ ﺩﺮﮑﻠﻤﻋ ﺪﯿﻫﺍﻮﺧ ﯽﻣ ﻭ ﺪﯿﺘﺴﻫ ﺩﺭﺍﻭ ﻩﺯﺎﺗ[ﺐﺼﻧ ﻞﺣﺍﺮﻣ](install.md). +ﺪﯿﻨﮐ ﺏﺎﺨﺘﻧﺍ ﺍﺭ ﺮﯾﺯ ﯼﺎﻫ ﻪﻨﯾﺰﮔ ﺯﺍ ﯽﮑﯾ ﺪﯿﻧﺍﻮﺗ ﯽﻣ ﻥﺁ ﺯﺍ ﺲﭘ: -## نیازمندی های سیستم - -این یک سیستم چند سکویی (Cross-Platform) نمی باشد. این ابزار نیاز به Linux Ubuntu Precise (12.04) یا جدیدتر، با معماری x86\_64 و پشتیبانی از SSE 4.2 می باشد. برای چک کردن SSE 4.2 خروجی دستور زیر را بررسی کنید: +* [ﺪﯿﻨﮐ ﯽﻃ ﺍﺭ ﻞﺼﻔﻣ ﺵﺯﻮﻣﺁ](tutorial.md) +* [ﺪﯿﻨﮐ ﺶﯾﺎﻣﺯﺁ ﻪﻧﻮﻤﻧ ﯼﺎﻫ ﻩﺩﺍﺩ ﺎﺑ](example_datasets/ontime.md) +[ﯽﻠﺻﺍ ﻪﻟﺎﻘﻣ](https://clickhouse.yandex/docs/fa/getting_started/)
- -```bash -grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" -``` - -
- -پیشنهاد می کنیم از Ubuntu TrustyT، Ubuntu Xenial یا Ubuntu Precise استفاده کنید. ترمینال باید از UTF-8 پشتیبانی کند. (به صورت پیش فرض در Ubuntu پشتیبانی می شود). - -## نصب - -### نصب از طریق پکیج های Debian/Ubuntu - -در فایل `/etc/apt/sources.list` (یا در یک فایل جدا `/etc/apt/sources.list.d/clickhouse.list`)، Repo زیر را اضافه کنید: - -
- -``` -deb http://repo.yandex.ru/clickhouse/deb/stable/ main/ -``` - -
- -اگر شما میخوایید جدیدترین نسخه ی تست را استفاده کنید، 'stable' رو به 'testing' تغییر بدید. - -سپس دستورات زیر را اجرا کنید: - -
- -```bash -sudo apt-get install dirmngr # optional -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 # optional -sudo apt-get update -sudo apt-get install clickhouse-client clickhouse-server -``` - -
- -شما همچنین می توانید از طریق لینک زیر پکیج ClickHouse را به صورت دستی دانلود و نصب کنید: . - -ClickHouse دارای تنظیمات محدودیت دسترسی می باشد. این تنظیمات در فایل 'users.xml' (کنار 'config.xml') می باشد. به صورت پیش فرض دسترسی برای کاربر 'default' از همه جا بدون نیاز به پسورد وجود دارد. 'user/default/networks' را مشاهده کنید. برای اطلاعات بیشتر قسمت "تنظیمات فایل ها" را مشاهده کنید. - - RPM ﯼﺎﻫ ﻪﺘﺴﺑ ﺯﺍ ### - -.ﺪﻨﮐ ﯽﻣ ﻪﯿﺻﻮﺗ ﺲﮐﻮﻨﯿﻟ ﺮﺑ ﯽﻨﺘﺒﻣ rpm ﺮﺑ ﯽﻨﺘﺒﻣ ﯼﺎﻫ ﻊﯾﺯﻮﺗ ﺮﯾﺎﺳ ﻭ CentOS ، RedHat ﯼﺍ - - :ﺪﯿﻨﮐ ﻪﻓﺎﺿﺍ ﺍﺭ ﯽﻤﺳﺭ ﻥﺰﺨﻣ ﺪﯾﺎﺑ ﺍﺪﺘﺑﺍ - -```bash -sudo yum install yum-utils -sudo rpm --import https://repo.yandex.ru/clickhouse/CLICKHOUSE-KEY.GPG -sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/x86_64 -``` - -.(ﺩﻮﺷ ﯽﻣ ﻪﯿﺻﻮﺗ ﺎﻤﺷ ﺶﯾﺎﻣﺯﺁ ﯼﺎﻫ ﻂﯿﺤﻣ ﯼﺍﺮﺑ ﻦﯾﺍ) ﺪﯿﻨﮐ ﻦﯾﺰﮕﯾﺎﺟ "ﺖﺴﺗ" ﺎﺑ ﺍﺭ "ﺭﺍﺪﯾﺎﭘ" - - :ﺪﯿﻨﮐ ﺐﺼﻧ ﺍﺭ ﺎﻫ ﻪﺘﺴﺑ ﻊﻗﺍﻭ ﺭﺩ ﺎﺗ ﺪﯿﻨﮐ ﺍﺮﺟﺍ ﺍﺭ ﺕﺍﺭﻮﺘﺳﺩ ﻦﯾﺍ ﺲﭙﺳ - -```bash -sudo yum install clickhouse-server clickhouse-client -``` - -. :ﺪﯿﻨﮐ ﺐﺼﻧ ﻭ ﯼﺮﯿﮔﺭﺎﺑ ﺎﺠﻨ - - Docker Image ﺯﺍ ### - -.ﺪﻨﻨﮐ ﯽﻣ ﻩﺩﺎﻔﺘﺳﺍ ﻞﺧﺍﺩ ﺭﺩ "deb" ﯽﻤﺳﺭ ﯼﺎﻫ ﻪﺘﺴﺑ ﺯﺍ ﺮﯾﻭﺎﺼﺗ ﻦﯾﺍ .ﺪﯿﻨﮐ ﻝﺎﺒﻧﺩ ﺍﺭ (/ht - - -### نصب از طریق Source - -برای Compile، دستورالعمل های فایل build.md را دنبال کنید: - -شما میتوانید پکیج را compile و نصب کنید. شما همچنین می توانید بدون نصب پکیج از برنامه ها استفاده کنید. - -
- -``` -Client: dbms/programs/clickhouse-client -Server: dbms/programs/clickhouse-server -``` - -
- -برای سرور، یک کاتالوگ با دیتا بسازید، مانند - -
- -``` -/opt/clickhouse/data/default/ -/opt/clickhouse/metadata/default/ -``` - -
- -(قابل تنظیم در تنظیمات سرور). 'chown' را برای کاربر دلخواه اجرا کنید. - -به مسیر لاگ ها در تنظیمات سرور توجه کنید (src/dbms/programs/config.xml). - -### روش های دیگر نصب - -Docker image: - -پکیج RPM برای CentOS یا RHEL: - -Gentoo: `emerge clickhouse` - -## راه اندازی - -برای استارت سرور (به صورت daemon)، دستور زیر را اجرا کنید: - -
- -```bash -sudo service clickhouse-server start -``` - -
- -لاگ های دایرکتوری `/var/log/clickhouse-server/` directory. را مشاهده کنید. - -اگر سرور استارت نشد، فایل تنظیمات را بررسی کنید `/etc/clickhouse-server/config.xml.` - -شما همچنین می توانید سرور را از طریق کنسول راه اندازی کنید: - -
- -```bash -clickhouse-server --config-file=/etc/clickhouse-server/config.xml -``` - -
- -در این مورد که مناسب زمان توسعه می باشد، لاگ ها در کنسول پرینت می شوند. اگر فایل تنظیمات در دایرکتوری جاری باشد، نیازی به مشخص کردن '--config-file' نمی باشد. به صورت پیش فرض از './config.xml' استفاده می شود. - -شما می توانید از کلاینت command-line برای اتصال به سرور استفاده کنید: - -
- -```bash -clickhouse-client -``` - -
- -پارامترهای پیش فرض، نشان از اتصال به localhost:9000 از طرف کاربر 'default' بدون پسورد را می دهد. از کلاینت میتوان برای اتصال به یک سرور remote استفاده کرد. مثال: - -
- -```bash -clickhouse-client --host=example.com -``` - -
- -برای اطلاعات بیشتر، بخش "کلاینت Command-line" را مشاهده کنید. - -چک کردن سیستم: - -
- -```bash -milovidov@hostname:~/work/metrica/src/dbms/src/Client$ ./clickhouse-client -ClickHouse client version 0.0.18749. -Connecting to localhost:9000. -Connected to ClickHouse server version 0.0.18749. - -:) SELECT 1 - -SELECT 1 - -┌─1─┐ -│ 1 │ -└───┘ - -1 rows in set. Elapsed: 0.003 sec. - -:) -``` - -
- -**تبریک میگم، سیستم کار می کنه!** - -برای ادامه آزمایشات، شما میتوانید دیتاست های تستی را دریافت و امتحان کنید. - -
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/getting_started/) diff --git a/docs/fa/getting_started/install.md b/docs/fa/getting_started/install.md new file mode 100644 index 00000000000..790c9381007 --- /dev/null +++ b/docs/fa/getting_started/install.md @@ -0,0 +1,199 @@ +
+ +# ﯼﺯﺍﺪﻧﺍ ﻩﺍﺭ ﻭ ﺐﺼﻧ + +## نیازمندی های سیستم + +ClickHouse ﺲﮐﻮﻨﯿﻟ ﻉﻮﻧ ﺮﻫ ﯼﻭﺭ ﺮﺑ ﺪﻧﺍﻮﺗ ﯽﻣ ، FreeBSD ﺎﯾ Mac OS X ﯼﺭﺎﻤﻌﻣ ﺎﺑ CPU x + +:ﺖﺳﺍ ﻩﺪﻣﺁ ، ﺪﻨﮐ ﯽﻣ ﯽﻧﺎﺒﯿﺘﺸﭘ SSE 4.2 ﺯﺍ ﯽﻠﻌﻓ CPU ﺎﯾﺁ ﻪﮑﻨﯾﺍ ﯽﺳﺭﺮﺑ ﯼﺍﺮﺑ ﺭﻮﺘﺳﺩ ﻦﯾﺍ + +
+ +```bash +grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" +``` + +
+ +ﺪﯾﺎﺑ ، ﺪﻧﺭﺍﺪﻧ PowerPC64LE ﺎﯾ AArch64 ﯼﺭﺎﻤﻌﻣ ﺎﯾ ﺪﻨﻨﮐ ﯽﻤﻧ ﯽﻧﺎﺒﯿﺘﺸﭘ SSE 4.2 ﺯﺍ ﻪﮐ[ClickHouse ﺪﯿﻨﮐ ﺩﺎﺠﯾﺍ ﻊﺑﺎﻨﻣ ﺯﺍ ﺍﺭ](#from-sources) ﺐﺳﺎﻨﻣ ﺕﺎﻤﯿﻈﻨﺗ ﺎﺑ + +##ﺩﻮﺟﻮﻣ ﺐﺼﻧ ﯼﺎﻫ ﻪﻨﯾﺰﮔ + +### نصب از طریق پکیج های Debian/Ubuntu {#from-deb-packages} + +در فایل `/etc/apt/sources.list` (یا در یک فایل جدا `/etc/apt/sources.list.d/clickhouse.list`)، Repo زیر را اضافه کنید: + +
+ +``` +deb http://repo.yandex.ru/clickhouse/deb/stable/ main/ +``` + +
+ +اگر شما میخوایید جدیدترین نسخه ی تست را استفاده کنید، 'stable' رو به 'testing' تغییر بدید. + +سپس دستورات زیر را اجرا کنید: + +
+ +```bash +sudo apt-get install dirmngr # optional +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 # optional +sudo apt-get update +sudo apt-get install clickhouse-client clickhouse-server +``` + +
+ +شما همچنین می توانید از طریق لینک زیر پکیج ClickHouse را به صورت دستی دانلود و نصب کنید: . + +ClickHouse دارای تنظیمات محدودیت دسترسی می باشد. این تنظیمات در فایل 'users.xml' (کنار 'config.xml') می باشد. به صورت پیش فرض دسترسی برای کاربر 'default' از همه جا بدون نیاز به پسورد وجود دارد. 'user/default/networks' را مشاهده کنید. برای اطلاعات بیشتر قسمت "تنظیمات فایل ها" را مشاهده کنید. + +### RPM ﯼﺎﻫ ﻪﺘﺴﺑ ﺯﺍ {#from-rpm-packages} + +.ﺪﻨﮐ ﯽﻣ ﻪﯿﺻﻮﺗ ﺲﮐﻮﻨﯿﻟ ﺮﺑ ﯽﻨﺘﺒﻣ rpm ﺮﺑ ﯽﻨﺘﺒﻣ ﯼﺎﻫ ﻊﯾﺯﻮﺗ ﺮﯾﺎﺳ ﻭ CentOS ، RedHat ﯼﺍ + + :ﺪﯿﻨﮐ ﻪﻓﺎﺿﺍ ﺍﺭ ﯽﻤﺳﺭ ﻥﺰﺨﻣ ﺪﯾﺎﺑ ﺍﺪﺘﺑﺍ + +```bash +sudo yum install yum-utils +sudo rpm --import https://repo.yandex.ru/clickhouse/CLICKHOUSE-KEY.GPG +sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/x86_64 +``` + +.(ﺩﻮﺷ ﯽﻣ ﻪﯿﺻﻮﺗ ﺎﻤﺷ ﺶﯾﺎﻣﺯﺁ ﯼﺎﻫ ﻂﯿﺤﻣ ﯼﺍﺮﺑ ﻦﯾﺍ) ﺪﯿﻨﮐ ﻦﯾﺰﮕﯾﺎﺟ "ﺖﺴﺗ" ﺎﺑ ﺍﺭ "ﺭﺍﺪﯾﺎﭘ" + + :ﺪﯿﻨﮐ ﺐﺼﻧ ﺍﺭ ﺎﻫ ﻪﺘﺴﺑ ﻊﻗﺍﻭ ﺭﺩ ﺎﺗ ﺪﯿﻨﮐ ﺍﺮﺟﺍ ﺍﺭ ﺕﺍﺭﻮﺘﺳﺩ ﻦﯾﺍ ﺲﭙﺳ + +```bash +sudo yum install clickhouse-server clickhouse-client +``` + +. :ﺪﯿﻨﮐ ﺐﺼﻧ ﻭ ﯼﺮﯿﮔﺭﺎﺑ ﺎﺠﻨ + + Docker Image ﺯﺍ ### + +.ﺪﻨﻨﮐ ﯽﻣ ﻩﺩﺎﻔﺘﺳﺍ ﻞﺧﺍﺩ ﺭﺩ "deb" ﯽﻤﺳﺭ ﯼﺎﻫ ﻪﺘﺴﺑ ﺯﺍ ﺮﯾﻭﺎﺼﺗ ﻦﯾﺍ .ﺪﯿﻨﮐ ﻝﺎﺒﻧﺩ ﺍﺭ (/ht + + +### نصب از طریق Source {#from-sources} + +برای Compile، دستورالعمل های فایل build.md را دنبال کنید: + +شما میتوانید پکیج را compile و نصب کنید. شما همچنین می توانید بدون نصب پکیج از برنامه ها استفاده کنید. + +
+ +``` +Client: dbms/programs/clickhouse-client +Server: dbms/programs/clickhouse-server +``` + +
+ +برای سرور، یک کاتالوگ با دیتا بسازید، مانند + +
+ +``` +/opt/clickhouse/data/default/ +/opt/clickhouse/metadata/default/ +``` + +
+ +(قابل تنظیم در تنظیمات سرور). 'chown' را برای کاربر دلخواه اجرا کنید. + +به مسیر لاگ ها در تنظیمات سرور توجه کنید (src/dbms/programs/config.xml). + +### روش های دیگر نصب {#from-docker-image} + +Docker image: + +پکیج RPM برای CentOS یا RHEL: + +Gentoo: `emerge clickhouse` + +## راه اندازی + +برای استارت سرور (به صورت daemon)، دستور زیر را اجرا کنید: + +
+ +```bash +sudo service clickhouse-server start +``` + +
+ +لاگ های دایرکتوری `/var/log/clickhouse-server/` directory. را مشاهده کنید. + +اگر سرور استارت نشد، فایل تنظیمات را بررسی کنید `/etc/clickhouse-server/config.xml.` + +شما همچنین می توانید سرور را از طریق کنسول راه اندازی کنید: + +
+ +```bash +clickhouse-server --config-file=/etc/clickhouse-server/config.xml +``` + +
+ +در این مورد که مناسب زمان توسعه می باشد، لاگ ها در کنسول پرینت می شوند. اگر فایل تنظیمات در دایرکتوری جاری باشد، نیازی به مشخص کردن '--config-file' نمی باشد. به صورت پیش فرض از './config.xml' استفاده می شود. + +شما می توانید از کلاینت command-line برای اتصال به سرور استفاده کنید: + +
+ +```bash +clickhouse-client +``` + +
+ +پارامترهای پیش فرض، نشان از اتصال به localhost:9000 از طرف کاربر 'default' بدون پسورد را می دهد. از کلاینت میتوان برای اتصال به یک سرور remote استفاده کرد. مثال: + +
+ +```bash +clickhouse-client --host=example.com +``` + +
+ +برای اطلاعات بیشتر، بخش "کلاینت Command-line" را مشاهده کنید. + +چک کردن سیستم: + +
+ +```bash +milovidov@hostname:~/work/metrica/src/dbms/src/Client$ ./clickhouse-client +ClickHouse client version 0.0.18749. +Connecting to localhost:9000. +Connected to ClickHouse server version 0.0.18749. + +:) SELECT 1 + +SELECT 1 + +┌─1─┐ +│ 1 │ +└───┘ + +1 rows in set. Elapsed: 0.003 sec. + +:) +``` + +
+ +**تبریک میگم، سیستم کار می کنه!** + +برای ادامه آزمایشات، شما میتوانید دیتاست های تستی را دریافت و امتحان کنید. + +
+[مقاله اصلی](https://clickhouse.yandex/docs/fa/getting_started/install/) diff --git a/docs/fa/getting_started/tutorial.md b/docs/fa/getting_started/tutorial.md new file mode 120000 index 00000000000..8bc40816ab2 --- /dev/null +++ b/docs/fa/getting_started/tutorial.md @@ -0,0 +1 @@ +../../en/getting_started/tutorial.md \ No newline at end of file diff --git a/docs/fa/interfaces/cli.md b/docs/fa/interfaces/cli.md index 8501f46ecd7..7680348aef6 100644 --- a/docs/fa/interfaces/cli.md +++ b/docs/fa/interfaces/cli.md @@ -96,13 +96,13 @@ command line برا پایه 'readline' (و 'history' یا 'libedit'، یه بد - `--vertical, -E` اگر مشخص شود، از فرمت Vertical برای نمایش خروجی استفاده می شود. این گزینه مشابه '--format=Vertical' می باشد. در این فرمت، هر مقدار در یک خط جدید چاپ می شود، که در هنگام نمایش جداول عریض مفید است. - `--time, -t` اگر مشخص شود، در حالت non-interactive زمان اجرای query در 'stderr' جاپ می شود. - `--stacktrace` – اگر مشخص شود stack trase مربوط به اجرای query در هنگام رخ دادن یک exception چاپ می شود. -- `-config-file` – نام فایل پیکربندی. +- `--config-file` – نام فایل پیکربندی. ### فایل های پیکربندی `clickhouse-client` به ترتیب اولویت زیر از اولین فایل موجود برای ست کردن تنظیمات استفاده می کند: -- مشخص شده در پارامتر `-config-file` +- مشخص شده در پارامتر `--config-file` - `./clickhouse-client.xml` - `\~/.clickhouse-client/config.xml` - `/etc/clickhouse-client/config.xml` diff --git a/docs/fa/interfaces/cpp.md b/docs/fa/interfaces/cpp.md new file mode 100644 index 00000000000..29d53571e94 --- /dev/null +++ b/docs/fa/interfaces/cpp.md @@ -0,0 +1,5 @@ +# C++ Client Library + +See README at [clickhouse-cpp](https://github.com/ClickHouse/clickhouse-cpp) repository. + +[Original article](https://clickhouse.yandex/docs/fa/interfaces/cpp/) diff --git a/docs/fa/interfaces/index.md b/docs/fa/interfaces/index.md index a3e2eb55a5e..57d8e673748 100644 --- a/docs/fa/interfaces/index.md +++ b/docs/fa/interfaces/index.md @@ -11,6 +11,7 @@ ClickHouse دو اینترفیس شبکه را فراهم می کند (هر دو * [خط فرمان خط](cli.md) * [راننده JDBC](jdbc.md) * [راننده ODBC](odbc.md) +* [C ++ کتابخانه مشتری](cpp.md) همچنین برای کار با ClickHouse طیف گسترده ای از کتابخانه های شخص ثالث وجود دارد: * [کتابخانه های مشتری](third-party/client_libraries.md) diff --git a/docs/fa/interfaces/third-party/client_libraries.md b/docs/fa/interfaces/third-party/client_libraries.md index 48034195fab..c31998191e5 100644 --- a/docs/fa/interfaces/third-party/client_libraries.md +++ b/docs/fa/interfaces/third-party/client_libraries.md @@ -27,7 +27,7 @@ - [HTTP-ClickHouse](https://metacpan.org/release/HTTP-ClickHouse) - [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse) - Ruby - - [clickhouse (Ruby)](https://github.com/archan937/clickhouse) + - [ClickHouse (Ruby)](https://github.com/shlima/click_house) - R - [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r) - [RClickhouse](https://github.com/IMSMWU/RClickhouse) @@ -40,8 +40,6 @@ - C# - [ClickHouse.Ado](https://github.com/killwort/ClickHouse-Net) - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) -- C++ - - [clickhouse-cpp](https://github.com/artpaul/clickhouse-cpp/) - Elixir - [clickhousex](https://github.com/appodeal/clickhousex/) - Nim diff --git a/docs/fa/introduction/ya_metrika_task.md b/docs/fa/introduction/history.md similarity index 99% rename from docs/fa/introduction/ya_metrika_task.md rename to docs/fa/introduction/history.md index 1ea434f248c..abde10aa6f3 100644 --- a/docs/fa/introduction/ya_metrika_task.md +++ b/docs/fa/introduction/history.md @@ -1,6 +1,6 @@
-# Yandex.Metrica use case +# ClickHouse ﻪﭽﺨﯾﺭﺎﺗ ClickHouse در ابتدا برای قدرت به Yandex.Metrica دومین بستر آنالیز وب در دنیا توسعه داده شد، و همچنان جز اصلی آن است. ClickHouse اجازه می دهند که با بیش از 13 تریلیون رکورد در دیتابیس و بیش از 20 میلیارد event در روز، گزارش های مستقیم (On the fly) از داده های non-aggregate تهیه کنیم. این مقاله پیشنیه ی تاریخی در ارتباط با اهداف اصلی ClickHouse قبل از آنکه به یک محصول open source تبدیل شود، می دهد. diff --git a/docs/fa/query_language/functions/introspection.md b/docs/fa/query_language/functions/introspection.md new file mode 120000 index 00000000000..b1a487e9c77 --- /dev/null +++ b/docs/fa/query_language/functions/introspection.md @@ -0,0 +1 @@ +../../../en/query_language/functions/introspection.md \ No newline at end of file diff --git a/docs/ja/changelog.md b/docs/ja/changelog.md new file mode 120000 index 00000000000..699cc9e7b7c --- /dev/null +++ b/docs/ja/changelog.md @@ -0,0 +1 @@ +../../CHANGELOG.md \ No newline at end of file diff --git a/docs/ja/data_types/array.md b/docs/ja/data_types/array.md new file mode 120000 index 00000000000..808c98bf91a --- /dev/null +++ b/docs/ja/data_types/array.md @@ -0,0 +1 @@ +../../en/data_types/array.md \ No newline at end of file diff --git a/docs/ja/data_types/boolean.md b/docs/ja/data_types/boolean.md new file mode 120000 index 00000000000..42e84f1e52a --- /dev/null +++ b/docs/ja/data_types/boolean.md @@ -0,0 +1 @@ +../../en/data_types/boolean.md \ No newline at end of file diff --git a/docs/ja/data_types/date.md b/docs/ja/data_types/date.md new file mode 120000 index 00000000000..d1ebc137e8f --- /dev/null +++ b/docs/ja/data_types/date.md @@ -0,0 +1 @@ +../../en/data_types/date.md \ No newline at end of file diff --git a/docs/ja/data_types/datetime.md b/docs/ja/data_types/datetime.md new file mode 120000 index 00000000000..2eb9f44e6eb --- /dev/null +++ b/docs/ja/data_types/datetime.md @@ -0,0 +1 @@ +../../en/data_types/datetime.md \ No newline at end of file diff --git a/docs/ja/data_types/decimal.md b/docs/ja/data_types/decimal.md new file mode 120000 index 00000000000..ccea440adfa --- /dev/null +++ b/docs/ja/data_types/decimal.md @@ -0,0 +1 @@ +../../en/data_types/decimal.md \ No newline at end of file diff --git a/docs/ja/data_types/domains/ipv4.md b/docs/ja/data_types/domains/ipv4.md new file mode 120000 index 00000000000..eb4cc7d57b5 --- /dev/null +++ b/docs/ja/data_types/domains/ipv4.md @@ -0,0 +1 @@ +../../../en/data_types/domains/ipv4.md \ No newline at end of file diff --git a/docs/ja/data_types/domains/ipv6.md b/docs/ja/data_types/domains/ipv6.md new file mode 120000 index 00000000000..cca37a22458 --- /dev/null +++ b/docs/ja/data_types/domains/ipv6.md @@ -0,0 +1 @@ +../../../en/data_types/domains/ipv6.md \ No newline at end of file diff --git a/docs/ja/data_types/domains/overview.md b/docs/ja/data_types/domains/overview.md new file mode 120000 index 00000000000..13465d655ee --- /dev/null +++ b/docs/ja/data_types/domains/overview.md @@ -0,0 +1 @@ +../../../en/data_types/domains/overview.md \ No newline at end of file diff --git a/docs/ja/data_types/enum.md b/docs/ja/data_types/enum.md new file mode 120000 index 00000000000..23ebe64773e --- /dev/null +++ b/docs/ja/data_types/enum.md @@ -0,0 +1 @@ +../../en/data_types/enum.md \ No newline at end of file diff --git a/docs/ja/data_types/fixedstring.md b/docs/ja/data_types/fixedstring.md new file mode 120000 index 00000000000..53092fcb884 --- /dev/null +++ b/docs/ja/data_types/fixedstring.md @@ -0,0 +1 @@ +../../en/data_types/fixedstring.md \ No newline at end of file diff --git a/docs/ja/data_types/float.md b/docs/ja/data_types/float.md new file mode 120000 index 00000000000..d2ae6bd11de --- /dev/null +++ b/docs/ja/data_types/float.md @@ -0,0 +1 @@ +../../en/data_types/float.md \ No newline at end of file diff --git a/docs/ja/data_types/index.md b/docs/ja/data_types/index.md new file mode 120000 index 00000000000..c9f29d637f3 --- /dev/null +++ b/docs/ja/data_types/index.md @@ -0,0 +1 @@ +../../en/data_types/index.md \ No newline at end of file diff --git a/docs/ja/data_types/int_uint.md b/docs/ja/data_types/int_uint.md new file mode 120000 index 00000000000..3a913c9328e --- /dev/null +++ b/docs/ja/data_types/int_uint.md @@ -0,0 +1 @@ +../../en/data_types/int_uint.md \ No newline at end of file diff --git a/docs/ja/data_types/nested_data_structures/aggregatefunction.md b/docs/ja/data_types/nested_data_structures/aggregatefunction.md new file mode 120000 index 00000000000..36544324d2b --- /dev/null +++ b/docs/ja/data_types/nested_data_structures/aggregatefunction.md @@ -0,0 +1 @@ +../../../en/data_types/nested_data_structures/aggregatefunction.md \ No newline at end of file diff --git a/docs/ja/data_types/nested_data_structures/index.md b/docs/ja/data_types/nested_data_structures/index.md new file mode 120000 index 00000000000..a5659a9c5cd --- /dev/null +++ b/docs/ja/data_types/nested_data_structures/index.md @@ -0,0 +1 @@ +../../../en/data_types/nested_data_structures/index.md \ No newline at end of file diff --git a/docs/ja/data_types/nested_data_structures/nested.md b/docs/ja/data_types/nested_data_structures/nested.md new file mode 120000 index 00000000000..653a1ce31c3 --- /dev/null +++ b/docs/ja/data_types/nested_data_structures/nested.md @@ -0,0 +1 @@ +../../../en/data_types/nested_data_structures/nested.md \ No newline at end of file diff --git a/docs/ja/data_types/nullable.md b/docs/ja/data_types/nullable.md new file mode 120000 index 00000000000..0233f91d954 --- /dev/null +++ b/docs/ja/data_types/nullable.md @@ -0,0 +1 @@ +../../en/data_types/nullable.md \ No newline at end of file diff --git a/docs/ja/data_types/special_data_types/expression.md b/docs/ja/data_types/special_data_types/expression.md new file mode 120000 index 00000000000..4cec632b416 --- /dev/null +++ b/docs/ja/data_types/special_data_types/expression.md @@ -0,0 +1 @@ +../../../en/data_types/special_data_types/expression.md \ No newline at end of file diff --git a/docs/ja/data_types/special_data_types/index.md b/docs/ja/data_types/special_data_types/index.md new file mode 120000 index 00000000000..f3ca4a47f98 --- /dev/null +++ b/docs/ja/data_types/special_data_types/index.md @@ -0,0 +1 @@ +../../../en/data_types/special_data_types/index.md \ No newline at end of file diff --git a/docs/ja/data_types/special_data_types/interval.md b/docs/ja/data_types/special_data_types/interval.md new file mode 120000 index 00000000000..6829f5ced00 --- /dev/null +++ b/docs/ja/data_types/special_data_types/interval.md @@ -0,0 +1 @@ +../../../en/data_types/special_data_types/interval.md \ No newline at end of file diff --git a/docs/ja/data_types/special_data_types/nothing.md b/docs/ja/data_types/special_data_types/nothing.md new file mode 120000 index 00000000000..197a752ce9c --- /dev/null +++ b/docs/ja/data_types/special_data_types/nothing.md @@ -0,0 +1 @@ +../../../en/data_types/special_data_types/nothing.md \ No newline at end of file diff --git a/docs/ja/data_types/special_data_types/set.md b/docs/ja/data_types/special_data_types/set.md new file mode 120000 index 00000000000..5beb14114d3 --- /dev/null +++ b/docs/ja/data_types/special_data_types/set.md @@ -0,0 +1 @@ +../../../en/data_types/special_data_types/set.md \ No newline at end of file diff --git a/docs/ja/data_types/string.md b/docs/ja/data_types/string.md new file mode 120000 index 00000000000..7bdd739398f --- /dev/null +++ b/docs/ja/data_types/string.md @@ -0,0 +1 @@ +../../en/data_types/string.md \ No newline at end of file diff --git a/docs/ja/data_types/tuple.md b/docs/ja/data_types/tuple.md new file mode 120000 index 00000000000..d30a8463aeb --- /dev/null +++ b/docs/ja/data_types/tuple.md @@ -0,0 +1 @@ +../../en/data_types/tuple.md \ No newline at end of file diff --git a/docs/ja/data_types/uuid.md b/docs/ja/data_types/uuid.md new file mode 120000 index 00000000000..aba05e889ac --- /dev/null +++ b/docs/ja/data_types/uuid.md @@ -0,0 +1 @@ +../../en/data_types/uuid.md \ No newline at end of file diff --git a/docs/ja/database_engines/index.md b/docs/ja/database_engines/index.md new file mode 120000 index 00000000000..bbdb762a4ad --- /dev/null +++ b/docs/ja/database_engines/index.md @@ -0,0 +1 @@ +../../en/database_engines/index.md \ No newline at end of file diff --git a/docs/ja/database_engines/lazy.md b/docs/ja/database_engines/lazy.md new file mode 120000 index 00000000000..66830dcdb2f --- /dev/null +++ b/docs/ja/database_engines/lazy.md @@ -0,0 +1 @@ +../../en/database_engines/lazy.md \ No newline at end of file diff --git a/docs/ja/database_engines/mysql.md b/docs/ja/database_engines/mysql.md new file mode 120000 index 00000000000..51ac4126e2d --- /dev/null +++ b/docs/ja/database_engines/mysql.md @@ -0,0 +1 @@ +../../en/database_engines/mysql.md \ No newline at end of file diff --git a/docs/ja/development/architecture.md b/docs/ja/development/architecture.md new file mode 120000 index 00000000000..abda4dd48a8 --- /dev/null +++ b/docs/ja/development/architecture.md @@ -0,0 +1 @@ +../../en/development/architecture.md \ No newline at end of file diff --git a/docs/ja/development/build.md b/docs/ja/development/build.md new file mode 120000 index 00000000000..480dbc2e9f5 --- /dev/null +++ b/docs/ja/development/build.md @@ -0,0 +1 @@ +../../en/development/build.md \ No newline at end of file diff --git a/docs/ja/development/build_cross_arm.md b/docs/ja/development/build_cross_arm.md new file mode 120000 index 00000000000..983a9872dc1 --- /dev/null +++ b/docs/ja/development/build_cross_arm.md @@ -0,0 +1 @@ +../../en/development/build_cross_arm.md \ No newline at end of file diff --git a/docs/ja/development/build_cross_osx.md b/docs/ja/development/build_cross_osx.md new file mode 120000 index 00000000000..72e64e8631f --- /dev/null +++ b/docs/ja/development/build_cross_osx.md @@ -0,0 +1 @@ +../../en/development/build_cross_osx.md \ No newline at end of file diff --git a/docs/ja/development/build_osx.md b/docs/ja/development/build_osx.md new file mode 120000 index 00000000000..f9adaf24584 --- /dev/null +++ b/docs/ja/development/build_osx.md @@ -0,0 +1 @@ +../../en/development/build_osx.md \ No newline at end of file diff --git a/docs/ja/development/contrib.md b/docs/ja/development/contrib.md new file mode 120000 index 00000000000..4749f95f9ef --- /dev/null +++ b/docs/ja/development/contrib.md @@ -0,0 +1 @@ +../../en/development/contrib.md \ No newline at end of file diff --git a/docs/ja/development/developer_instruction.md b/docs/ja/development/developer_instruction.md new file mode 120000 index 00000000000..bdfa9047aa2 --- /dev/null +++ b/docs/ja/development/developer_instruction.md @@ -0,0 +1 @@ +../../en/development/developer_instruction.md \ No newline at end of file diff --git a/docs/ja/development/index.md b/docs/ja/development/index.md new file mode 120000 index 00000000000..1e2ad97dcc5 --- /dev/null +++ b/docs/ja/development/index.md @@ -0,0 +1 @@ +../../en/development/index.md \ No newline at end of file diff --git a/docs/ja/development/style.md b/docs/ja/development/style.md new file mode 120000 index 00000000000..c1bbf11f421 --- /dev/null +++ b/docs/ja/development/style.md @@ -0,0 +1 @@ +../../en/development/style.md \ No newline at end of file diff --git a/docs/ja/development/tests.md b/docs/ja/development/tests.md new file mode 120000 index 00000000000..c03d36c3916 --- /dev/null +++ b/docs/ja/development/tests.md @@ -0,0 +1 @@ +../../en/development/tests.md \ No newline at end of file diff --git a/docs/ja/faq/general.md b/docs/ja/faq/general.md new file mode 120000 index 00000000000..bc267395b1b --- /dev/null +++ b/docs/ja/faq/general.md @@ -0,0 +1 @@ +../../en/faq/general.md \ No newline at end of file diff --git a/docs/ja/getting_started/example_datasets/amplab_benchmark.md b/docs/ja/getting_started/example_datasets/amplab_benchmark.md new file mode 120000 index 00000000000..78c93906bb0 --- /dev/null +++ b/docs/ja/getting_started/example_datasets/amplab_benchmark.md @@ -0,0 +1 @@ +../../../en/getting_started/example_datasets/amplab_benchmark.md \ No newline at end of file diff --git a/docs/ja/getting_started/example_datasets/criteo.md b/docs/ja/getting_started/example_datasets/criteo.md new file mode 120000 index 00000000000..507dc68cd62 --- /dev/null +++ b/docs/ja/getting_started/example_datasets/criteo.md @@ -0,0 +1 @@ +../../../en/getting_started/example_datasets/criteo.md \ No newline at end of file diff --git a/docs/ja/getting_started/example_datasets/metrica.md b/docs/ja/getting_started/example_datasets/metrica.md new file mode 120000 index 00000000000..984023973eb --- /dev/null +++ b/docs/ja/getting_started/example_datasets/metrica.md @@ -0,0 +1 @@ +../../../en/getting_started/example_datasets/metrica.md \ No newline at end of file diff --git a/docs/ja/getting_started/example_datasets/nyc_taxi.md b/docs/ja/getting_started/example_datasets/nyc_taxi.md new file mode 120000 index 00000000000..c47fc83a293 --- /dev/null +++ b/docs/ja/getting_started/example_datasets/nyc_taxi.md @@ -0,0 +1 @@ +../../../en/getting_started/example_datasets/nyc_taxi.md \ No newline at end of file diff --git a/docs/ja/getting_started/example_datasets/ontime.md b/docs/ja/getting_started/example_datasets/ontime.md new file mode 120000 index 00000000000..87cfbb8be91 --- /dev/null +++ b/docs/ja/getting_started/example_datasets/ontime.md @@ -0,0 +1 @@ +../../../en/getting_started/example_datasets/ontime.md \ No newline at end of file diff --git a/docs/ja/getting_started/example_datasets/star_schema.md b/docs/ja/getting_started/example_datasets/star_schema.md new file mode 120000 index 00000000000..1c26392dd23 --- /dev/null +++ b/docs/ja/getting_started/example_datasets/star_schema.md @@ -0,0 +1 @@ +../../../en/getting_started/example_datasets/star_schema.md \ No newline at end of file diff --git a/docs/ja/getting_started/example_datasets/wikistat.md b/docs/ja/getting_started/example_datasets/wikistat.md new file mode 120000 index 00000000000..bf6e780fb27 --- /dev/null +++ b/docs/ja/getting_started/example_datasets/wikistat.md @@ -0,0 +1 @@ +../../../en/getting_started/example_datasets/wikistat.md \ No newline at end of file diff --git a/docs/ja/getting_started/index.md b/docs/ja/getting_started/index.md new file mode 120000 index 00000000000..1acedb0f03e --- /dev/null +++ b/docs/ja/getting_started/index.md @@ -0,0 +1 @@ +../../en/getting_started/index.md \ No newline at end of file diff --git a/docs/ja/getting_started/install.md b/docs/ja/getting_started/install.md new file mode 120000 index 00000000000..60aa3fb93a4 --- /dev/null +++ b/docs/ja/getting_started/install.md @@ -0,0 +1 @@ +../../en/getting_started/install.md \ No newline at end of file diff --git a/docs/ja/getting_started/tutorial.md b/docs/ja/getting_started/tutorial.md new file mode 120000 index 00000000000..8bc40816ab2 --- /dev/null +++ b/docs/ja/getting_started/tutorial.md @@ -0,0 +1 @@ +../../en/getting_started/tutorial.md \ No newline at end of file diff --git a/docs/ja/guides/apply_catboost_model.md b/docs/ja/guides/apply_catboost_model.md new file mode 120000 index 00000000000..dd36e885974 --- /dev/null +++ b/docs/ja/guides/apply_catboost_model.md @@ -0,0 +1 @@ +../../en/guides/apply_catboost_model.md \ No newline at end of file diff --git a/docs/ja/guides/index.md b/docs/ja/guides/index.md new file mode 120000 index 00000000000..162dcbc3b8f --- /dev/null +++ b/docs/ja/guides/index.md @@ -0,0 +1 @@ +../../en/guides/index.md \ No newline at end of file diff --git a/docs/ja/images/column_oriented.gif b/docs/ja/images/column_oriented.gif new file mode 100644 index 00000000000..15f4b12e697 Binary files /dev/null and b/docs/ja/images/column_oriented.gif differ diff --git a/docs/ja/images/logo.svg b/docs/ja/images/logo.svg new file mode 100644 index 00000000000..70662da887e --- /dev/null +++ b/docs/ja/images/logo.svg @@ -0,0 +1,12 @@ + + + + + + + + + diff --git a/docs/ja/images/row_oriented.gif b/docs/ja/images/row_oriented.gif new file mode 100644 index 00000000000..53daa20f322 Binary files /dev/null and b/docs/ja/images/row_oriented.gif differ diff --git a/docs/ja/index.md b/docs/ja/index.md new file mode 100644 index 00000000000..a7f8681a2bf --- /dev/null +++ b/docs/ja/index.md @@ -0,0 +1,142 @@ +# ClickHouseとは? + +ClickHouseは、クエリのオンライン分析処理（OLAP）用の列指向のデータベース管理システム（DBMS）です。 + +「通常の」行指向のDBMSでは、データは次の順序で保存されます。 + +| Row | WatchID | JavaEnable | Title | GoodEvent | EventTime | +| ------ | ------------------- | ---------- | ------------------ | --------- | ------------------- | +| #0 | 89354350662 | 1 | Investor Relations | 1 | 2016-05-18 05:19:20 | +| #1 | 90329509958 | 0 | Contact us | 1 | 2016-05-18 08:10:20 | +| #2 | 89953706054 | 1 | Mission | 1 | 2016-05-18 07:38:00 | +| #N | ... | ... | ... | ... | ... | + +つまり、行に関連するすべての値は物理的に隣り合わせに格納されます。 + +行指向のDBMSの例：MySQL, Postgres および MS SQL Server +{: .grey } + +列指向のDBMSでは、データは次のように保存されます： + +| Row: | #0 | #1 | #2 | #N | +| ----------- | ------------------- | ------------------- | ------------------- | ------------------- | +| WatchID: | 89354350662 | 90329509958 | 89953706054 | ... | +| JavaEnable: | 1 | 0 | 1 | ... | +| Title: | Investor Relations | Contact us | Mission | ... | +| GoodEvent: | 1 | 1 | 1 | ... | +| EventTime: | 2016-05-18 05:19:20 | 2016-05-18 08:10:20 | 2016-05-18 07:38:00 | ... | + +これらの例は、データが配置される順序のみを示しています。 +異なる列の値は別々に保存され、同じ列のデータは一緒に保存されます。 + +列指向DBMSの例：Vertica, Paraccel (Actian Matrix and Amazon Redshift), Sybase IQ, Exasol, Infobright, InfiniDB, MonetDB (VectorWise and Actian Vector), LucidDB, SAP HANA, Google Dremel, Google PowerDrill, Druid および kdb+ +{: .grey } + +異なったデータ格納の順序は、異なったシナリオにより適します。 +データアクセスシナリオとは、クエリの実行内容、頻度、割合を指します。クエリで読み取られるの各種データの量（行、列、バイト）。データの読み取りと更新の関係。作業データのサイズとローカルでの使用方法。トランザクションが使用されるかどうか、およびそれらがどの程度分離されているか。データ複製と論理的整合性の要件。クエリの種類ごとの遅延とスループットの要件など。 + +システムの負荷が高いほど、使用シナリオの要件に一致するようにセットアップされたシステムをカスタマイズすることがより重要になり、このカスタマイズはより細かくなります。大きく異なるシナリオに等しく適したシステムはありません。システムがさまざまなシナリオに適応可能である場合、高負荷下では、システムはすべてのシナリオを同等に不十分に処理するか、1つまたはいくつかの可能なシナリオでうまく機能します。 + +## OLAPシナリオの主要なプロパティ + +- リクエストの大部分は読み取りアクセス用である。 +- データは、単一行ではなく、かなり大きなバッチ（> 1000行）で更新されます。または、まったく更新されない。 +- データはDBに追加されるが、変更されない。 +- 読み取りの場合、非常に多くの行がDBから抽出されるが、一部の列のみ。 +- テーブルは「幅が広く」、多数の列が含まれる。 +- クエリは比較的まれ（通常、サーバーあたり毎秒数百あるいはそれ以下の数のクエリ）。 +- 単純なクエリでは、約50ミリ秒の遅延が容認される。 +- 列の値はかなり小さく、数値や短い文字列（たとえば、URLごとに60バイト）。 +- 単一のクエリを処理する場合、高いスループットが必要（サーバーあたり毎秒最大数十億行）。 +- トランザクションは必要ない。 +- データの一貫性の要件が低い。 +- クエリごとに1つの大きなテーブルがある。 1つを除くすべてのテーブルは小さい。 +- クエリ結果は、ソースデータよりも大幅に小さくなる。つまり、データはフィルター処理または集計されるため、結果は単一サーバーのRAMに収まる。 + +OLAPシナリオは、他の一般的なシナリオ（OLTPやKey-Valueアクセスなど）とは非常に異なることが容易にわかります。 したがって、まともなパフォーマンスを得るには、OLTPまたはKey-Value DBを使用して分析クエリを処理しようとするのは無意味です。 たとえば、分析にMongoDBまたはRedisを使用しようとすると、OLAPデータベースに比べてパフォーマンスが非常に低下します。 + +## OLAPシナリオで列指向データベースがよりよく機能する理由 + +列指向データベースは、OLAPシナリオにより適しています。ほとんどのクエリの処理が少なくとも100倍高速です。 理由を以下に詳しく説明しますが、その根拠は視覚的に簡単に説明できます： + +**行指向DBMS** + +![Row-oriented](images/row_oriented.gif#) + +**列指向DBMS** + +![Column-oriented](images/column_oriented.gif#) + +違いがわかりましたか？ + +### Input/output + +1. 分析クエリでは、少数のテーブル列のみを読み取る必要があります。列指向のデータベースでは、必要なデータのみを読み取ることができます。たとえば、100のうち5つの列が必要な場合、I/Oが20倍削減されることが期待できます。 +2. データはパケットで読み取られるため、圧縮が容易です。列のデータも圧縮が簡単です。これにより、I/Oボリュームがさらに削減されます。 +3. I/Oの削減により、より多くのデータがシステムキャッシュに収まります。 + +たとえば、「各広告プラットフォームのレコード数をカウントする」クエリでは、1つの「広告プラットフォームID」列を読み取る必要がありますが、これは非圧縮では1バイトの領域を要します。トラフィックのほとんどが広告プラットフォームからのものではない場合、この列は少なくとも10倍の圧縮が期待できます。高速な圧縮アルゴリズムを使用すれば、1秒あたり少なくとも非圧縮データに換算して数ギガバイトの速度でデータを展開できます。つまり、このクエリは、単一のサーバーで1秒あたり約数十億行の速度で処理できます。この速度はまさに実際に達成されます。 + +
Example +``` +$ clickhouse-client +ClickHouse client version 0.0.52053. +Connecting to localhost:9000. +Connected to ClickHouse server version 0.0.52053. + +:) SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 + +SELECT +CounterID, +count() +FROM hits +GROUP BY CounterID +ORDER BY count() DESC +LIMIT 20 + +┌─CounterID─┬──count()─┐ +│ 114208 │ 56057344 │ +│ 115080 │ 51619590 │ +│ 3228 │ 44658301 │ +│ 38230 │ 42045932 │ +│ 145263 │ 42042158 │ +│ 91244 │ 38297270 │ +│ 154139 │ 26647572 │ +│ 150748 │ 24112755 │ +│ 242232 │ 21302571 │ +│ 338158 │ 13507087 │ +│ 62180 │ 12229491 │ +│ 82264 │ 12187441 │ +│ 232261 │ 12148031 │ +│ 146272 │ 11438516 │ +│ 168777 │ 11403636 │ +│ 4120072 │ 11227824 │ +│ 10938808 │ 10519739 │ +│ 74088 │ 9047015 │ +│ 115079 │ 8837972 │ +│ 337234 │ 8205961 │ +└───────────┴──────────┘ + +20 rows in set. Elapsed: 0.153 sec. Processed 1.00 billion rows, 4.00 GB (6.53 billion rows/s., 26.10 GB/s.) + +:) +``` + +
+ +### CPU + +クエリを実行するには大量の行を処理する必要があるため、個別の行ではなくベクター全体のすべての操作をディスパッチするか、ディスパッチコストがほとんどないようにクエリエンジンを実装すると効率的です。 適切なディスクサブシステムでこれを行わないと、クエリインタープリターが必然的にCPUを失速させます。 +データを列に格納し、可能な場合は列ごとに処理することは理にかなっています。 + +これを行うには2つの方法があります: + +1. ベクトルエンジン。 すべての操作は、個別の値ではなく、ベクトルに対して記述されます。 これは、オペレーションを頻繁に呼び出す必要がなく、ディスパッチコストが無視できることを意味します。 操作コードには、最適化された内部サイクルが含まれています。 + +2. コード生成。 クエリ用に生成されたコードには、すべての間接的な呼び出しが含まれています。 + +これは、単純なクエリを実行する場合には意味がないため、「通常の」データベースでは実行されません。 ただし、例外があります。 たとえば、MemSQLはコード生成を使用して、SQLクエリを処理する際の遅延を減らします。 （比較のために、分析DBMSではレイテンシではなくスループットの最適化が必要です。） + +CPU効率のために、クエリ言語は宣言型（SQLまたはMDX）、または少なくともベクトル（J、K）でなければなりません。 クエリには、最適化を可能にする暗黙的なループのみを含める必要があります。 + +[Original article](https://clickhouse.yandex/docs/ja/) diff --git a/docs/ja/interfaces/cli.md b/docs/ja/interfaces/cli.md new file mode 120000 index 00000000000..04588066828 --- /dev/null +++ b/docs/ja/interfaces/cli.md @@ -0,0 +1 @@ +../../en/interfaces/cli.md \ No newline at end of file diff --git a/docs/ja/interfaces/cpp.md b/docs/ja/interfaces/cpp.md new file mode 120000 index 00000000000..581e50e774d --- /dev/null +++ b/docs/ja/interfaces/cpp.md @@ -0,0 +1 @@ +../../en/interfaces/cpp.md \ No newline at end of file diff --git a/docs/ja/interfaces/formats.md b/docs/ja/interfaces/formats.md new file mode 120000 index 00000000000..41a65ebe579 --- /dev/null +++ b/docs/ja/interfaces/formats.md @@ -0,0 +1 @@ +../../en/interfaces/formats.md \ No newline at end of file diff --git a/docs/ja/interfaces/http.md b/docs/ja/interfaces/http.md new file mode 120000 index 00000000000..fb293841d8b --- /dev/null +++ b/docs/ja/interfaces/http.md @@ -0,0 +1 @@ +../../en/interfaces/http.md \ No newline at end of file diff --git a/docs/ja/interfaces/index.md b/docs/ja/interfaces/index.md new file mode 120000 index 00000000000..61537763cac --- /dev/null +++ b/docs/ja/interfaces/index.md @@ -0,0 +1 @@ +../../en/interfaces/index.md \ No newline at end of file diff --git a/docs/ja/interfaces/jdbc.md b/docs/ja/interfaces/jdbc.md new file mode 120000 index 00000000000..27dfe0cfa5a --- /dev/null +++ b/docs/ja/interfaces/jdbc.md @@ -0,0 +1 @@ +../../en/interfaces/jdbc.md \ No newline at end of file diff --git a/docs/ja/interfaces/odbc.md b/docs/ja/interfaces/odbc.md new file mode 120000 index 00000000000..5ff7610e061 --- /dev/null +++ b/docs/ja/interfaces/odbc.md @@ -0,0 +1 @@ +../../en/interfaces/odbc.md \ No newline at end of file diff --git a/docs/ja/interfaces/tcp.md b/docs/ja/interfaces/tcp.md new file mode 120000 index 00000000000..a0529a856e4 --- /dev/null +++ b/docs/ja/interfaces/tcp.md @@ -0,0 +1 @@ +../../en/interfaces/tcp.md \ No newline at end of file diff --git a/docs/ja/interfaces/third-party/client_libraries.md b/docs/ja/interfaces/third-party/client_libraries.md new file mode 120000 index 00000000000..5320bbe1e16 --- /dev/null +++ b/docs/ja/interfaces/third-party/client_libraries.md @@ -0,0 +1 @@ +../../../en/interfaces/third-party/client_libraries.md \ No newline at end of file diff --git a/docs/ja/interfaces/third-party/gui.md b/docs/ja/interfaces/third-party/gui.md new file mode 120000 index 00000000000..ef7bc904197 --- /dev/null +++ b/docs/ja/interfaces/third-party/gui.md @@ -0,0 +1 @@ +../../../en/interfaces/third-party/gui.md \ No newline at end of file diff --git a/docs/ja/interfaces/third-party/integrations.md b/docs/ja/interfaces/third-party/integrations.md new file mode 120000 index 00000000000..9cd0a21e676 --- /dev/null +++ b/docs/ja/interfaces/third-party/integrations.md @@ -0,0 +1 @@ +../../../en/interfaces/third-party/integrations.md \ No newline at end of file diff --git a/docs/ja/interfaces/third-party/proxy.md b/docs/ja/interfaces/third-party/proxy.md new file mode 120000 index 00000000000..877f1b51dab --- /dev/null +++ b/docs/ja/interfaces/third-party/proxy.md @@ -0,0 +1 @@ +../../../en/interfaces/third-party/proxy.md \ No newline at end of file diff --git a/docs/ja/introduction/distinctive_features.md b/docs/ja/introduction/distinctive_features.md new file mode 120000 index 00000000000..9cf00a2a00f --- /dev/null +++ b/docs/ja/introduction/distinctive_features.md @@ -0,0 +1 @@ +../../en/introduction/distinctive_features.md \ No newline at end of file diff --git a/docs/ja/introduction/features_considered_disadvantages.md b/docs/ja/introduction/features_considered_disadvantages.md new file mode 120000 index 00000000000..45d3cdf563a --- /dev/null +++ b/docs/ja/introduction/features_considered_disadvantages.md @@ -0,0 +1 @@ +../../en/introduction/features_considered_disadvantages.md \ No newline at end of file diff --git a/docs/ja/introduction/history.md b/docs/ja/introduction/history.md new file mode 120000 index 00000000000..7004e990a59 --- /dev/null +++ b/docs/ja/introduction/history.md @@ -0,0 +1 @@ +../../en/introduction/history.md \ No newline at end of file diff --git a/docs/ja/introduction/performance.md b/docs/ja/introduction/performance.md new file mode 120000 index 00000000000..cb2912bcb81 --- /dev/null +++ b/docs/ja/introduction/performance.md @@ -0,0 +1 @@ +../../en/introduction/performance.md \ No newline at end of file diff --git a/docs/ja/operations/access_rights.md b/docs/ja/operations/access_rights.md new file mode 120000 index 00000000000..73463029569 --- /dev/null +++ b/docs/ja/operations/access_rights.md @@ -0,0 +1 @@ +../../en/operations/access_rights.md \ No newline at end of file diff --git a/docs/ja/operations/backup.md b/docs/ja/operations/backup.md new file mode 120000 index 00000000000..1003fb30e61 --- /dev/null +++ b/docs/ja/operations/backup.md @@ -0,0 +1 @@ +../../en/operations/backup.md \ No newline at end of file diff --git a/docs/ja/operations/configuration_files.md b/docs/ja/operations/configuration_files.md new file mode 120000 index 00000000000..a2d73dbaa25 --- /dev/null +++ b/docs/ja/operations/configuration_files.md @@ -0,0 +1 @@ +../../en/operations/configuration_files.md \ No newline at end of file diff --git a/docs/ja/operations/index.md b/docs/ja/operations/index.md new file mode 120000 index 00000000000..ce854687b86 --- /dev/null +++ b/docs/ja/operations/index.md @@ -0,0 +1 @@ +../../en/operations/index.md \ No newline at end of file diff --git a/docs/ja/operations/monitoring.md b/docs/ja/operations/monitoring.md new file mode 120000 index 00000000000..515ae8b4fff --- /dev/null +++ b/docs/ja/operations/monitoring.md @@ -0,0 +1 @@ +../../en/operations/monitoring.md \ No newline at end of file diff --git a/docs/ja/operations/quotas.md b/docs/ja/operations/quotas.md new file mode 120000 index 00000000000..1c52cdf1e91 --- /dev/null +++ b/docs/ja/operations/quotas.md @@ -0,0 +1 @@ +../../en/operations/quotas.md \ No newline at end of file diff --git a/docs/ja/operations/requirements.md b/docs/ja/operations/requirements.md new file mode 120000 index 00000000000..a71283af25c --- /dev/null +++ b/docs/ja/operations/requirements.md @@ -0,0 +1 @@ +../../en/operations/requirements.md \ No newline at end of file diff --git a/docs/ja/operations/server_settings/index.md b/docs/ja/operations/server_settings/index.md new file mode 120000 index 00000000000..1d1a0585a42 --- /dev/null +++ b/docs/ja/operations/server_settings/index.md @@ -0,0 +1 @@ +../../../en/operations/server_settings/index.md \ No newline at end of file diff --git a/docs/ja/operations/server_settings/settings.md b/docs/ja/operations/server_settings/settings.md new file mode 120000 index 00000000000..19cd2e82ce7 --- /dev/null +++ b/docs/ja/operations/server_settings/settings.md @@ -0,0 +1 @@ +../../../en/operations/server_settings/settings.md \ No newline at end of file diff --git a/docs/ja/operations/settings/constraints_on_settings.md b/docs/ja/operations/settings/constraints_on_settings.md new file mode 120000 index 00000000000..4dacf908662 --- /dev/null +++ b/docs/ja/operations/settings/constraints_on_settings.md @@ -0,0 +1 @@ +../../../en/operations/settings/constraints_on_settings.md \ No newline at end of file diff --git a/docs/ja/operations/settings/index.md b/docs/ja/operations/settings/index.md new file mode 120000 index 00000000000..fc3968d1f1e --- /dev/null +++ b/docs/ja/operations/settings/index.md @@ -0,0 +1 @@ +../../../en/operations/settings/index.md \ No newline at end of file diff --git a/docs/ja/operations/settings/permissions_for_queries.md b/docs/ja/operations/settings/permissions_for_queries.md new file mode 120000 index 00000000000..ce8473bf01c --- /dev/null +++ b/docs/ja/operations/settings/permissions_for_queries.md @@ -0,0 +1 @@ +../../../en/operations/settings/permissions_for_queries.md \ No newline at end of file diff --git a/docs/ja/operations/settings/query_complexity.md b/docs/ja/operations/settings/query_complexity.md new file mode 120000 index 00000000000..9a9c6d975a9 --- /dev/null +++ b/docs/ja/operations/settings/query_complexity.md @@ -0,0 +1 @@ +../../../en/operations/settings/query_complexity.md \ No newline at end of file diff --git a/docs/ja/operations/settings/settings.md b/docs/ja/operations/settings/settings.md new file mode 120000 index 00000000000..0c8df3cfc90 --- /dev/null +++ b/docs/ja/operations/settings/settings.md @@ -0,0 +1 @@ +../../../en/operations/settings/settings.md \ No newline at end of file diff --git a/docs/ja/operations/settings/settings_profiles.md b/docs/ja/operations/settings/settings_profiles.md new file mode 120000 index 00000000000..35d9747ad56 --- /dev/null +++ b/docs/ja/operations/settings/settings_profiles.md @@ -0,0 +1 @@ +../../../en/operations/settings/settings_profiles.md \ No newline at end of file diff --git a/docs/ja/operations/settings/settings_users.md b/docs/ja/operations/settings/settings_users.md new file mode 120000 index 00000000000..3a6a7cf6948 --- /dev/null +++ b/docs/ja/operations/settings/settings_users.md @@ -0,0 +1 @@ +../../../en/operations/settings/settings_users.md \ No newline at end of file diff --git a/docs/ja/operations/system_tables.md b/docs/ja/operations/system_tables.md new file mode 120000 index 00000000000..c5701190dca --- /dev/null +++ b/docs/ja/operations/system_tables.md @@ -0,0 +1 @@ +../../en/operations/system_tables.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/aggregatingmergetree.md b/docs/ja/operations/table_engines/aggregatingmergetree.md new file mode 120000 index 00000000000..907a073e0c8 --- /dev/null +++ b/docs/ja/operations/table_engines/aggregatingmergetree.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/aggregatingmergetree.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/buffer.md b/docs/ja/operations/table_engines/buffer.md new file mode 120000 index 00000000000..0a3c372fa67 --- /dev/null +++ b/docs/ja/operations/table_engines/buffer.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/buffer.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/collapsingmergetree.md b/docs/ja/operations/table_engines/collapsingmergetree.md new file mode 120000 index 00000000000..ef5cebb48d8 --- /dev/null +++ b/docs/ja/operations/table_engines/collapsingmergetree.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/collapsingmergetree.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/custom_partitioning_key.md b/docs/ja/operations/table_engines/custom_partitioning_key.md new file mode 120000 index 00000000000..a9d18cacb25 --- /dev/null +++ b/docs/ja/operations/table_engines/custom_partitioning_key.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/custom_partitioning_key.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/dictionary.md b/docs/ja/operations/table_engines/dictionary.md new file mode 120000 index 00000000000..2a95f4a669b --- /dev/null +++ b/docs/ja/operations/table_engines/dictionary.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/dictionary.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/distributed.md b/docs/ja/operations/table_engines/distributed.md new file mode 120000 index 00000000000..46994303c35 --- /dev/null +++ b/docs/ja/operations/table_engines/distributed.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/distributed.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/external_data.md b/docs/ja/operations/table_engines/external_data.md new file mode 120000 index 00000000000..27a7b6acec2 --- /dev/null +++ b/docs/ja/operations/table_engines/external_data.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/external_data.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/file.md b/docs/ja/operations/table_engines/file.md new file mode 120000 index 00000000000..27dffc8d78f --- /dev/null +++ b/docs/ja/operations/table_engines/file.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/file.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/graphitemergetree.md b/docs/ja/operations/table_engines/graphitemergetree.md new file mode 120000 index 00000000000..654425d050a --- /dev/null +++ b/docs/ja/operations/table_engines/graphitemergetree.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/graphitemergetree.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/hdfs.md b/docs/ja/operations/table_engines/hdfs.md new file mode 120000 index 00000000000..d4dbfa46e68 --- /dev/null +++ b/docs/ja/operations/table_engines/hdfs.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/hdfs.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/index.md b/docs/ja/operations/table_engines/index.md new file mode 120000 index 00000000000..994dff9b516 --- /dev/null +++ b/docs/ja/operations/table_engines/index.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/index.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/jdbc.md b/docs/ja/operations/table_engines/jdbc.md new file mode 120000 index 00000000000..5165d704b9a --- /dev/null +++ b/docs/ja/operations/table_engines/jdbc.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/jdbc.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/join.md b/docs/ja/operations/table_engines/join.md new file mode 120000 index 00000000000..0914ab950ed --- /dev/null +++ b/docs/ja/operations/table_engines/join.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/join.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/kafka.md b/docs/ja/operations/table_engines/kafka.md new file mode 120000 index 00000000000..cb7bd5dd0f8 --- /dev/null +++ b/docs/ja/operations/table_engines/kafka.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/kafka.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/log.md b/docs/ja/operations/table_engines/log.md new file mode 120000 index 00000000000..2c39ba68522 --- /dev/null +++ b/docs/ja/operations/table_engines/log.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/log.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/log_family.md b/docs/ja/operations/table_engines/log_family.md new file mode 120000 index 00000000000..8c5b5f0365b --- /dev/null +++ b/docs/ja/operations/table_engines/log_family.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/log_family.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/materializedview.md b/docs/ja/operations/table_engines/materializedview.md new file mode 120000 index 00000000000..e3b5deb73dc --- /dev/null +++ b/docs/ja/operations/table_engines/materializedview.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/materializedview.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/memory.md b/docs/ja/operations/table_engines/memory.md new file mode 120000 index 00000000000..eee940c7bd3 --- /dev/null +++ b/docs/ja/operations/table_engines/memory.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/memory.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/merge.md b/docs/ja/operations/table_engines/merge.md new file mode 120000 index 00000000000..9e17d9bb939 --- /dev/null +++ b/docs/ja/operations/table_engines/merge.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/merge.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/mergetree.md b/docs/ja/operations/table_engines/mergetree.md new file mode 120000 index 00000000000..cc6ac1e5297 --- /dev/null +++ b/docs/ja/operations/table_engines/mergetree.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/mergetree.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/mysql.md b/docs/ja/operations/table_engines/mysql.md new file mode 120000 index 00000000000..e4c268658cf --- /dev/null +++ b/docs/ja/operations/table_engines/mysql.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/mysql.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/null.md b/docs/ja/operations/table_engines/null.md new file mode 120000 index 00000000000..c7d9264571e --- /dev/null +++ b/docs/ja/operations/table_engines/null.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/null.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/odbc.md b/docs/ja/operations/table_engines/odbc.md new file mode 120000 index 00000000000..06091fd5377 --- /dev/null +++ b/docs/ja/operations/table_engines/odbc.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/odbc.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/replacingmergetree.md b/docs/ja/operations/table_engines/replacingmergetree.md new file mode 120000 index 00000000000..63ff25a4dd6 --- /dev/null +++ b/docs/ja/operations/table_engines/replacingmergetree.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/replacingmergetree.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/replication.md b/docs/ja/operations/table_engines/replication.md new file mode 120000 index 00000000000..b4b22ac708b --- /dev/null +++ b/docs/ja/operations/table_engines/replication.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/replication.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/set.md b/docs/ja/operations/table_engines/set.md new file mode 120000 index 00000000000..d37e659badd --- /dev/null +++ b/docs/ja/operations/table_engines/set.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/set.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/stripelog.md b/docs/ja/operations/table_engines/stripelog.md new file mode 120000 index 00000000000..f6521a41e3e --- /dev/null +++ b/docs/ja/operations/table_engines/stripelog.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/stripelog.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/summingmergetree.md b/docs/ja/operations/table_engines/summingmergetree.md new file mode 120000 index 00000000000..2b67e953d8a --- /dev/null +++ b/docs/ja/operations/table_engines/summingmergetree.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/summingmergetree.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/tinylog.md b/docs/ja/operations/table_engines/tinylog.md new file mode 120000 index 00000000000..bda90c7d5ce --- /dev/null +++ b/docs/ja/operations/table_engines/tinylog.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/tinylog.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/url.md b/docs/ja/operations/table_engines/url.md new file mode 120000 index 00000000000..d0de71dcf40 --- /dev/null +++ b/docs/ja/operations/table_engines/url.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/url.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/versionedcollapsingmergetree.md b/docs/ja/operations/table_engines/versionedcollapsingmergetree.md new file mode 120000 index 00000000000..5843fba70b8 --- /dev/null +++ b/docs/ja/operations/table_engines/versionedcollapsingmergetree.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/versionedcollapsingmergetree.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/view.md b/docs/ja/operations/table_engines/view.md new file mode 120000 index 00000000000..3f2164181a7 --- /dev/null +++ b/docs/ja/operations/table_engines/view.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/view.md \ No newline at end of file diff --git a/docs/ja/operations/tips.md b/docs/ja/operations/tips.md new file mode 120000 index 00000000000..9b3413bdbc3 --- /dev/null +++ b/docs/ja/operations/tips.md @@ -0,0 +1 @@ +../../en/operations/tips.md \ No newline at end of file diff --git a/docs/ja/operations/troubleshooting.md b/docs/ja/operations/troubleshooting.md new file mode 120000 index 00000000000..84f0ff34f41 --- /dev/null +++ b/docs/ja/operations/troubleshooting.md @@ -0,0 +1 @@ +../../en/operations/troubleshooting.md \ No newline at end of file diff --git a/docs/ja/operations/update.md b/docs/ja/operations/update.md new file mode 120000 index 00000000000..88a092c0dff --- /dev/null +++ b/docs/ja/operations/update.md @@ -0,0 +1 @@ +../../en/operations/update.md \ No newline at end of file diff --git a/docs/ja/operations/utils/clickhouse-copier.md b/docs/ja/operations/utils/clickhouse-copier.md new file mode 120000 index 00000000000..c9e89e33c7b --- /dev/null +++ b/docs/ja/operations/utils/clickhouse-copier.md @@ -0,0 +1 @@ +../../../en/operations/utils/clickhouse-copier.md \ No newline at end of file diff --git a/docs/ja/operations/utils/clickhouse-local.md b/docs/ja/operations/utils/clickhouse-local.md new file mode 120000 index 00000000000..032aaaa2b84 --- /dev/null +++ b/docs/ja/operations/utils/clickhouse-local.md @@ -0,0 +1 @@ +../../../en/operations/utils/clickhouse-local.md \ No newline at end of file diff --git a/docs/ja/operations/utils/index.md b/docs/ja/operations/utils/index.md new file mode 120000 index 00000000000..dd089d1ef4b --- /dev/null +++ b/docs/ja/operations/utils/index.md @@ -0,0 +1 @@ +../../../en/operations/utils/index.md \ No newline at end of file diff --git a/docs/ja/query_language/agg_functions/combinators.md b/docs/ja/query_language/agg_functions/combinators.md new file mode 120000 index 00000000000..2b914cebd15 --- /dev/null +++ b/docs/ja/query_language/agg_functions/combinators.md @@ -0,0 +1 @@ +../../../en/query_language/agg_functions/combinators.md \ No newline at end of file diff --git a/docs/ja/query_language/agg_functions/index.md b/docs/ja/query_language/agg_functions/index.md new file mode 120000 index 00000000000..2fcf67abdeb --- /dev/null +++ b/docs/ja/query_language/agg_functions/index.md @@ -0,0 +1 @@ +../../../en/query_language/agg_functions/index.md \ No newline at end of file diff --git a/docs/ja/query_language/agg_functions/parametric_functions.md b/docs/ja/query_language/agg_functions/parametric_functions.md new file mode 120000 index 00000000000..fd3ffafcc5b --- /dev/null +++ b/docs/ja/query_language/agg_functions/parametric_functions.md @@ -0,0 +1 @@ +../../../en/query_language/agg_functions/parametric_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/agg_functions/reference.md b/docs/ja/query_language/agg_functions/reference.md new file mode 120000 index 00000000000..c5651cb0793 --- /dev/null +++ b/docs/ja/query_language/agg_functions/reference.md @@ -0,0 +1 @@ +../../../en/query_language/agg_functions/reference.md \ No newline at end of file diff --git a/docs/ja/query_language/alter.md b/docs/ja/query_language/alter.md new file mode 120000 index 00000000000..44f4ecf9737 --- /dev/null +++ b/docs/ja/query_language/alter.md @@ -0,0 +1 @@ +../../en/query_language/alter.md \ No newline at end of file diff --git a/docs/ja/query_language/create.md b/docs/ja/query_language/create.md new file mode 120000 index 00000000000..a13304d176e --- /dev/null +++ b/docs/ja/query_language/create.md @@ -0,0 +1 @@ +../../en/query_language/create.md \ No newline at end of file diff --git a/docs/ja/query_language/dicts/external_dicts.md b/docs/ja/query_language/dicts/external_dicts.md new file mode 120000 index 00000000000..491b94bffe6 --- /dev/null +++ b/docs/ja/query_language/dicts/external_dicts.md @@ -0,0 +1 @@ +../../../en/query_language/dicts/external_dicts.md \ No newline at end of file diff --git a/docs/ja/query_language/dicts/external_dicts_dict.md b/docs/ja/query_language/dicts/external_dicts_dict.md new file mode 120000 index 00000000000..e27820fee60 --- /dev/null +++ b/docs/ja/query_language/dicts/external_dicts_dict.md @@ -0,0 +1 @@ +../../../en/query_language/dicts/external_dicts_dict.md \ No newline at end of file diff --git a/docs/ja/query_language/dicts/external_dicts_dict_layout.md b/docs/ja/query_language/dicts/external_dicts_dict_layout.md new file mode 120000 index 00000000000..e391c5be723 --- /dev/null +++ b/docs/ja/query_language/dicts/external_dicts_dict_layout.md @@ -0,0 +1 @@ +../../../en/query_language/dicts/external_dicts_dict_layout.md \ No newline at end of file diff --git a/docs/ja/query_language/dicts/external_dicts_dict_lifetime.md b/docs/ja/query_language/dicts/external_dicts_dict_lifetime.md new file mode 120000 index 00000000000..03b53c09077 --- /dev/null +++ b/docs/ja/query_language/dicts/external_dicts_dict_lifetime.md @@ -0,0 +1 @@ +../../../en/query_language/dicts/external_dicts_dict_lifetime.md \ No newline at end of file diff --git a/docs/ja/query_language/dicts/external_dicts_dict_sources.md b/docs/ja/query_language/dicts/external_dicts_dict_sources.md new file mode 120000 index 00000000000..d4f4bf8ef3e --- /dev/null +++ b/docs/ja/query_language/dicts/external_dicts_dict_sources.md @@ -0,0 +1 @@ +../../../en/query_language/dicts/external_dicts_dict_sources.md \ No newline at end of file diff --git a/docs/ja/query_language/dicts/external_dicts_dict_structure.md b/docs/ja/query_language/dicts/external_dicts_dict_structure.md new file mode 120000 index 00000000000..69ff759caea --- /dev/null +++ b/docs/ja/query_language/dicts/external_dicts_dict_structure.md @@ -0,0 +1 @@ +../../../en/query_language/dicts/external_dicts_dict_structure.md \ No newline at end of file diff --git a/docs/ja/query_language/dicts/index.md b/docs/ja/query_language/dicts/index.md new file mode 120000 index 00000000000..fdc188ca2a2 --- /dev/null +++ b/docs/ja/query_language/dicts/index.md @@ -0,0 +1 @@ +../../../en/query_language/dicts/index.md \ No newline at end of file diff --git a/docs/ja/query_language/dicts/internal_dicts.md b/docs/ja/query_language/dicts/internal_dicts.md new file mode 120000 index 00000000000..3f9408dcd45 --- /dev/null +++ b/docs/ja/query_language/dicts/internal_dicts.md @@ -0,0 +1 @@ +../../../en/query_language/dicts/internal_dicts.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/arithmetic_functions.md b/docs/ja/query_language/functions/arithmetic_functions.md new file mode 120000 index 00000000000..c22acb8c7f5 --- /dev/null +++ b/docs/ja/query_language/functions/arithmetic_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/arithmetic_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/array_functions.md b/docs/ja/query_language/functions/array_functions.md new file mode 120000 index 00000000000..268b2295a97 --- /dev/null +++ b/docs/ja/query_language/functions/array_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/array_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/array_join.md b/docs/ja/query_language/functions/array_join.md new file mode 120000 index 00000000000..b100dac784d --- /dev/null +++ b/docs/ja/query_language/functions/array_join.md @@ -0,0 +1 @@ +../../../en/query_language/functions/array_join.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/bit_functions.md b/docs/ja/query_language/functions/bit_functions.md new file mode 120000 index 00000000000..b5cccd0c56c --- /dev/null +++ b/docs/ja/query_language/functions/bit_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/bit_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/bitmap_functions.md b/docs/ja/query_language/functions/bitmap_functions.md new file mode 120000 index 00000000000..0a31d3d71d8 --- /dev/null +++ b/docs/ja/query_language/functions/bitmap_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/bitmap_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/comparison_functions.md b/docs/ja/query_language/functions/comparison_functions.md new file mode 120000 index 00000000000..417c589867c --- /dev/null +++ b/docs/ja/query_language/functions/comparison_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/comparison_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/conditional_functions.md b/docs/ja/query_language/functions/conditional_functions.md new file mode 120000 index 00000000000..ad0d775dbb5 --- /dev/null +++ b/docs/ja/query_language/functions/conditional_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/conditional_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/date_time_functions.md b/docs/ja/query_language/functions/date_time_functions.md new file mode 120000 index 00000000000..d11b9b8bb6b --- /dev/null +++ b/docs/ja/query_language/functions/date_time_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/date_time_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/encoding_functions.md b/docs/ja/query_language/functions/encoding_functions.md new file mode 120000 index 00000000000..b2e6be1405b --- /dev/null +++ b/docs/ja/query_language/functions/encoding_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/encoding_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/ext_dict_functions.md b/docs/ja/query_language/functions/ext_dict_functions.md new file mode 120000 index 00000000000..6318f900e4b --- /dev/null +++ b/docs/ja/query_language/functions/ext_dict_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/ext_dict_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/functions_for_nulls.md b/docs/ja/query_language/functions/functions_for_nulls.md new file mode 120000 index 00000000000..fa57e10ad15 --- /dev/null +++ b/docs/ja/query_language/functions/functions_for_nulls.md @@ -0,0 +1 @@ +../../../en/query_language/functions/functions_for_nulls.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/geo.md b/docs/ja/query_language/functions/geo.md new file mode 120000 index 00000000000..86fa3a85d34 --- /dev/null +++ b/docs/ja/query_language/functions/geo.md @@ -0,0 +1 @@ +../../../en/query_language/functions/geo.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/hash_functions.md b/docs/ja/query_language/functions/hash_functions.md new file mode 120000 index 00000000000..90de8ba97e7 --- /dev/null +++ b/docs/ja/query_language/functions/hash_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/hash_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/higher_order_functions.md b/docs/ja/query_language/functions/higher_order_functions.md new file mode 120000 index 00000000000..077feba2a3e --- /dev/null +++ b/docs/ja/query_language/functions/higher_order_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/higher_order_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/in_functions.md b/docs/ja/query_language/functions/in_functions.md new file mode 120000 index 00000000000..3ae5f24dbca --- /dev/null +++ b/docs/ja/query_language/functions/in_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/in_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/index.md b/docs/ja/query_language/functions/index.md new file mode 120000 index 00000000000..a4e9d619cc0 --- /dev/null +++ b/docs/ja/query_language/functions/index.md @@ -0,0 +1 @@ +../../../en/query_language/functions/index.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/introspection.md b/docs/ja/query_language/functions/introspection.md new file mode 120000 index 00000000000..b1a487e9c77 --- /dev/null +++ b/docs/ja/query_language/functions/introspection.md @@ -0,0 +1 @@ +../../../en/query_language/functions/introspection.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/ip_address_functions.md b/docs/ja/query_language/functions/ip_address_functions.md new file mode 120000 index 00000000000..b58175a7cdf --- /dev/null +++ b/docs/ja/query_language/functions/ip_address_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/ip_address_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/json_functions.md b/docs/ja/query_language/functions/json_functions.md new file mode 120000 index 00000000000..1b37184e006 --- /dev/null +++ b/docs/ja/query_language/functions/json_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/json_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/logical_functions.md b/docs/ja/query_language/functions/logical_functions.md new file mode 120000 index 00000000000..32015440e09 --- /dev/null +++ b/docs/ja/query_language/functions/logical_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/logical_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/machine_learning_functions.md b/docs/ja/query_language/functions/machine_learning_functions.md new file mode 120000 index 00000000000..4509602717e --- /dev/null +++ b/docs/ja/query_language/functions/machine_learning_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/machine_learning_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/math_functions.md b/docs/ja/query_language/functions/math_functions.md new file mode 120000 index 00000000000..e01674eca4d --- /dev/null +++ b/docs/ja/query_language/functions/math_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/math_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/other_functions.md b/docs/ja/query_language/functions/other_functions.md new file mode 120000 index 00000000000..65164784ced --- /dev/null +++ b/docs/ja/query_language/functions/other_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/other_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/random_functions.md b/docs/ja/query_language/functions/random_functions.md new file mode 120000 index 00000000000..b873e0c86ac --- /dev/null +++ b/docs/ja/query_language/functions/random_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/random_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/rounding_functions.md b/docs/ja/query_language/functions/rounding_functions.md new file mode 120000 index 00000000000..e1217e3b25a --- /dev/null +++ b/docs/ja/query_language/functions/rounding_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/rounding_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/splitting_merging_functions.md b/docs/ja/query_language/functions/splitting_merging_functions.md new file mode 120000 index 00000000000..5f8771abdec --- /dev/null +++ b/docs/ja/query_language/functions/splitting_merging_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/splitting_merging_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/string_functions.md b/docs/ja/query_language/functions/string_functions.md new file mode 120000 index 00000000000..cc4104aaf53 --- /dev/null +++ b/docs/ja/query_language/functions/string_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/string_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/string_replace_functions.md b/docs/ja/query_language/functions/string_replace_functions.md new file mode 120000 index 00000000000..4ec963ffd0f --- /dev/null +++ b/docs/ja/query_language/functions/string_replace_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/string_replace_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/string_search_functions.md b/docs/ja/query_language/functions/string_search_functions.md new file mode 120000 index 00000000000..0a2c7f4c4f1 --- /dev/null +++ b/docs/ja/query_language/functions/string_search_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/string_search_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/type_conversion_functions.md b/docs/ja/query_language/functions/type_conversion_functions.md new file mode 120000 index 00000000000..fcf51570d15 --- /dev/null +++ b/docs/ja/query_language/functions/type_conversion_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/type_conversion_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/url_functions.md b/docs/ja/query_language/functions/url_functions.md new file mode 120000 index 00000000000..529e4ffdd53 --- /dev/null +++ b/docs/ja/query_language/functions/url_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/url_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/uuid_functions.md b/docs/ja/query_language/functions/uuid_functions.md new file mode 120000 index 00000000000..95e3ded0477 --- /dev/null +++ b/docs/ja/query_language/functions/uuid_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/uuid_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/ym_dict_functions.md b/docs/ja/query_language/functions/ym_dict_functions.md new file mode 120000 index 00000000000..ec5ddc84479 --- /dev/null +++ b/docs/ja/query_language/functions/ym_dict_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/ym_dict_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/index.md b/docs/ja/query_language/index.md new file mode 120000 index 00000000000..44dfff9bb18 --- /dev/null +++ b/docs/ja/query_language/index.md @@ -0,0 +1 @@ +../../en/query_language/index.md \ No newline at end of file diff --git a/docs/ja/query_language/insert_into.md b/docs/ja/query_language/insert_into.md new file mode 120000 index 00000000000..29b47662b0d --- /dev/null +++ b/docs/ja/query_language/insert_into.md @@ -0,0 +1 @@ +../../en/query_language/insert_into.md \ No newline at end of file diff --git a/docs/ja/query_language/misc.md b/docs/ja/query_language/misc.md new file mode 120000 index 00000000000..3bd814f3568 --- /dev/null +++ b/docs/ja/query_language/misc.md @@ -0,0 +1 @@ +../../en/query_language/misc.md \ No newline at end of file diff --git a/docs/ja/query_language/operators.md b/docs/ja/query_language/operators.md new file mode 120000 index 00000000000..f94df928a82 --- /dev/null +++ b/docs/ja/query_language/operators.md @@ -0,0 +1 @@ +../../en/query_language/operators.md \ No newline at end of file diff --git a/docs/ja/query_language/select.md b/docs/ja/query_language/select.md new file mode 120000 index 00000000000..c8ec8369383 --- /dev/null +++ b/docs/ja/query_language/select.md @@ -0,0 +1 @@ +../../en/query_language/select.md \ No newline at end of file diff --git a/docs/ja/query_language/show.md b/docs/ja/query_language/show.md new file mode 120000 index 00000000000..4c2f4cf2c4f --- /dev/null +++ b/docs/ja/query_language/show.md @@ -0,0 +1 @@ +../../en/query_language/show.md \ No newline at end of file diff --git a/docs/ja/query_language/syntax.md b/docs/ja/query_language/syntax.md new file mode 120000 index 00000000000..5307fd51ae8 --- /dev/null +++ b/docs/ja/query_language/syntax.md @@ -0,0 +1 @@ +../../en/query_language/syntax.md \ No newline at end of file diff --git a/docs/ja/query_language/system.md b/docs/ja/query_language/system.md new file mode 120000 index 00000000000..6061858c3f2 --- /dev/null +++ b/docs/ja/query_language/system.md @@ -0,0 +1 @@ +../../en/query_language/system.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/file.md b/docs/ja/query_language/table_functions/file.md new file mode 120000 index 00000000000..a514547109a --- /dev/null +++ b/docs/ja/query_language/table_functions/file.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/file.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/hdfs.md b/docs/ja/query_language/table_functions/hdfs.md new file mode 120000 index 00000000000..2616e737eb6 --- /dev/null +++ b/docs/ja/query_language/table_functions/hdfs.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/hdfs.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/index.md b/docs/ja/query_language/table_functions/index.md new file mode 120000 index 00000000000..89b22522859 --- /dev/null +++ b/docs/ja/query_language/table_functions/index.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/index.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/input.md b/docs/ja/query_language/table_functions/input.md new file mode 120000 index 00000000000..f23cc8ee673 --- /dev/null +++ b/docs/ja/query_language/table_functions/input.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/input.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/jdbc.md b/docs/ja/query_language/table_functions/jdbc.md new file mode 120000 index 00000000000..73bec80ca58 --- /dev/null +++ b/docs/ja/query_language/table_functions/jdbc.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/jdbc.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/merge.md b/docs/ja/query_language/table_functions/merge.md new file mode 120000 index 00000000000..383f6c88331 --- /dev/null +++ b/docs/ja/query_language/table_functions/merge.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/merge.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/mysql.md b/docs/ja/query_language/table_functions/mysql.md new file mode 120000 index 00000000000..75c032cc63f --- /dev/null +++ b/docs/ja/query_language/table_functions/mysql.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/mysql.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/numbers.md b/docs/ja/query_language/table_functions/numbers.md new file mode 120000 index 00000000000..a679b915669 --- /dev/null +++ b/docs/ja/query_language/table_functions/numbers.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/numbers.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/odbc.md b/docs/ja/query_language/table_functions/odbc.md new file mode 120000 index 00000000000..7620f920494 --- /dev/null +++ b/docs/ja/query_language/table_functions/odbc.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/odbc.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/remote.md b/docs/ja/query_language/table_functions/remote.md new file mode 120000 index 00000000000..b157c4076d3 --- /dev/null +++ b/docs/ja/query_language/table_functions/remote.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/remote.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/url.md b/docs/ja/query_language/table_functions/url.md new file mode 120000 index 00000000000..038e08f7ba9 --- /dev/null +++ b/docs/ja/query_language/table_functions/url.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/url.md \ No newline at end of file diff --git a/docs/ja/roadmap.md b/docs/ja/roadmap.md new file mode 120000 index 00000000000..24df86352b3 --- /dev/null +++ b/docs/ja/roadmap.md @@ -0,0 +1 @@ +../en/roadmap.md \ No newline at end of file diff --git a/docs/ja/security_changelog.md b/docs/ja/security_changelog.md new file mode 120000 index 00000000000..101a4f4e48c --- /dev/null +++ b/docs/ja/security_changelog.md @@ -0,0 +1 @@ +../en/security_changelog.md \ No newline at end of file diff --git a/docs/redirects.txt b/docs/redirects.txt index 0ff077b660c..b38f6d242f2 100644 --- a/docs/redirects.txt +++ b/docs/redirects.txt @@ -1,3 +1,4 @@ +introduction/ya_metrika_task.md introduction/history.md system_tables.md operations/system_tables.md system_tables/system.asynchronous_metrics.md operations/system_tables.md system_tables/system.clusters.md operations/system_tables.md diff --git a/docs/ru/data_types/special_data_types/interval.md b/docs/ru/data_types/special_data_types/interval.md new file mode 100644 index 00000000000..6762f9bc850 --- /dev/null +++ b/docs/ru/data_types/special_data_types/interval.md @@ -0,0 +1,74 @@ +# Interval {#data-type-interval} + +Семейство типов данных, представляющих интервалы дат и времени. Оператор [INTERVAL](../../query_language/operators.md#operator-interval) возвращает значения этих типов. + +!!! warning "Внимание" + Нельзя использовать типы данных `Interval` для хранения данных в таблице. + +Структура: + +- Интервал времени в виде положительного целого числа. +- Тип интервала. + +Поддержанные типы интервалов: + +- `SECOND` +- `MINUTE` +- `HOUR` +- `DAY` +- `WEEK` +- `MONTH` +- `QUARTER` +- `YEAR` + +Каждому типу интервала соответствует отдельный тип данных. Например, тип данных `IntervalDay` соответствует интервалу `DAY`: + +```sql +SELECT toTypeName(INTERVAL 4 DAY) +``` +```text +┌─toTypeName(toIntervalDay(4))─┐ +│ IntervalDay │ +└──────────────────────────────┘ +``` + +## Использование {#data-type-interval-usage-remarks} + +Значения типов `Interval` можно использовать в арифметических операциях со значениями типов [Date](../../data_types/date.md) и [DateTime](../../data_types/datetime.md). Например, можно добавить 4 дня к текущей дате: + +```sql +SELECT now() as current_date_time, current_date_time + INTERVAL 4 DAY +``` +```text +┌───current_date_time─┬─plus(now(), toIntervalDay(4))─┐ +│ 2019-10-23 10:58:45 │ 2019-10-27 10:58:45 │ +└─────────────────────┴───────────────────────────────┘ +``` + +Нельзя объединять интервалы различных типов. Нельзя использовать интервалы вида `4 DAY 1 HOUR`. Вместо этого выражайте интервал в единицах меньших или равных минимальной единице интервала, например, интервал "1 день и 1 час" можно выразить как `25 HOUR` или `90000 SECOND`. + +Арифметические операции со значениями типов `Interval` не доступны, однако можно последовательно добавлять различные интервалы к значениям типов `Date` и `DateTime`. Например: + +```sql +SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL 3 HOUR +``` +```text +┌───current_date_time─┬─plus(plus(now(), toIntervalDay(4)), toIntervalHour(3))─┐ +│ 2019-10-23 11:16:28 │ 2019-10-27 14:16:28 │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +``` + +Следующий запрос приведёт к генерированию исключения: + +```sql +select now() AS current_date_time, current_date_time + (INTERVAL 4 DAY + INTERVAL 3 HOUR) +``` +```text +Received exception from server (version 19.14.1): +Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Wrong argument types for function plus: if one argument is Interval, then another must be Date or DateTime.. +``` + +## Смотрите также + +- Оператор[INTERVAL](../../query_language/operators.md#operator-interval) +- Функция приведения типа [toInterval](../../query_language/functions/type_conversion_functions.md#function-tointerval) diff --git a/docs/ru/development/build_cross.md b/docs/ru/development/build_cross.md deleted file mode 120000 index f595f252de3..00000000000 --- a/docs/ru/development/build_cross.md +++ /dev/null @@ -1 +0,0 @@ -../../en/development/build_cross.md \ No newline at end of file diff --git a/docs/ru/development/build_cross_osx.md b/docs/ru/development/build_cross_osx.md new file mode 120000 index 00000000000..72e64e8631f --- /dev/null +++ b/docs/ru/development/build_cross_osx.md @@ -0,0 +1 @@ +../../en/development/build_cross_osx.md \ No newline at end of file diff --git a/docs/ru/development/contrib.md b/docs/ru/development/contrib.md index 3640f1f3a58..f51d9f94d93 100644 --- a/docs/ru/development/contrib.md +++ b/docs/ru/development/contrib.md @@ -10,6 +10,7 @@ | double-conversion | [BSD 3-Clause License](https://github.com/google/double-conversion/blob/cf2f0f3d547dc73b4612028a155b80536902ba02/LICENSE) | | FastMemcpy | [MIT](https://github.com/yandex/ClickHouse/blob/master/libs/libmemcpy/impl/LICENSE) | | googletest | [BSD 3-Clause License](https://github.com/google/googletest/blob/master/LICENSE) | +| h3 | [Apache License 2.0](https://github.com/uber/h3/blob/master/LICENSE) | hyperscan | [BSD 3-Clause License](https://github.com/intel/hyperscan/blob/master/LICENSE) | | libbtrie | [BSD 2-Clause License](https://github.com/yandex/ClickHouse/blob/master/contrib/libbtrie/LICENSE) | | libcxxabi | [BSD + MIT](https://github.com/yandex/ClickHouse/blob/master/libs/libglibc-compatibility/libcxxabi/LICENSE.TXT) | diff --git a/docs/ru/development/developer_instruction.md b/docs/ru/development/developer_instruction.md new file mode 100644 index 00000000000..61be36a7089 --- /dev/null +++ b/docs/ru/development/developer_instruction.md @@ -0,0 +1,276 @@ +Сборка ClickHouse поддерживается на Linux, FreeBSD, Mac OS X. + + +# Если вы используете Windows + +Если вы используете Windows, вам потребуется создать виртуальную машину с Ubuntu. Для работы с виртуальной машиной, установите VirtualBox. Скачать Ubuntu можно на сайте: https://www.ubuntu.com/#download Создайте виртуальную машину из полученного образа. Выделите для неё не менее 4 GB оперативной памяти. Для запуска терминала в Ubuntu, найдите в меню программу со словом terminal (gnome-terminal, konsole или что-то в этом роде) или нажмите Ctrl+Alt+T. + + +# Создание репозитория на GitHub + +Для работы с репозиторием ClickHouse, вам потребуется аккаунт на GitHub. Наверное, он у вас уже есть. + +Если аккаунта нет - зарегистрируйтесь на https://github.com/. Создайте ssh ключи, если их нет, и загрузите публичные ключи на GitHub. Это потребуется для отправки изменений. Для работы с GitHub можно использовать такие же ssh ключи, как и для работы с другими ssh серверами - скорее всего, они уже у вас есть. + +Создайте fork репозитория ClickHouse. Для этого, на странице https://github.com/ClickHouse/ClickHouse нажмите на кнопку "fork" в правом верхнем углу. Вы получите полную копию репозитория ClickHouse на своём аккаунте, которая называется "форк". Процесс разработки состоит в том, чтобы внести нужные изменения в свой форк репозитория, а затем создать "pull request" для принятия изменений в основной репозиторий. + +Для работы с git репозиториями, установите `git`. + +В Ubuntu выполните в терминале: +``` +sudo apt update +sudo apt install git +``` + +Краткое руководство по использованию Git: https://services.github.com/on-demand/downloads/github-git-cheat-sheet.pdf + +Подробное руководство по использованию Git: https://git-scm.com/book/ru/v2 + + +# Клонирование репозитория на рабочую машину + +Затем вам потребуется загрузить исходники для работы на свой компьютер. Это называется "клонирование репозитория", потому что создаёт на вашем компьютере локальную копию репозитория, с которой вы будете работать. + +Выполните в терминале: +``` +git clone --recursive git@github.com:yandex/ClickHouse.git +cd ClickHouse +``` +Замените *yandex* на имя вашего аккаунта на GitHub. + +Эта команда создаст директорию ClickHouse, содержащую рабочую копию проекта. + +Необходимо, чтобы путь к рабочей копии не содержал пробелы в именах директорий. Это может привести к проблемам в работе системы сборки. + +Обратите внимание, что репозиторий ClickHouse использует submodules. Так называются ссылки на дополнительные репозитории (например, внешние библиотеки, от которых зависит проект). Это значит, что при клонировании репозитория, следует указывать ключ `--recursive`, как в примере выше. Если репозиторий был клонирован без submodules, то для их скачивания, необходимо выполнить: +``` +git submodule init +git submodule update +``` +Проверить наличие submodules можно с помощью команды `git submodule status`. + +Если вы получили сообщение об ошибке: +``` +Permission denied (publickey). +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. +``` +Как правило это означает, что отсутствуют ssh ключи для соединения с GitHub. Ключи расположены в директории `~/.ssh`. В интерфейсе GitHub, в настройках, необходимо загрузить публичные ключи, чтобы он их понимал. + +Вы также можете клонировать репозиторий по протоколу https: +``` +git clone https://github.com/ClickHouse/ClickHouse.git +``` +Этот вариант не подходит для отправки изменений на сервер. Вы можете временно его использовать, а затем добавить ssh ключи и заменить адрес репозитория с помощью команды `git remote`. + +Вы можете также добавить для своего локального репозитория адрес оригинального репозитория Яндекса, чтобы притягивать оттуда обновления: +``` +git remote add upstream git@github.com:yandex/ClickHouse.git +``` +После этого, вы сможете добавлять в свой репозиторий обновления из репозитория Яндекса с помощью команды `git pull upstream master`. + + +# Система сборки + +ClickHouse использует систему сборки CMake и Ninja. + +CMake - генератор задач сборки. +Ninja - система запуска сборочных задач. + +Для установки на Ubuntu или Debian, Mint, выполните `sudo apt install cmake ninja-build`. + +Для установки на CentOS, RedHat, выполните `sudo yum install cmake ninja-build`. + +Если у вас Arch или Gentoo, то вы сами знаете, как установить CMake. + +Для установки CMake и Ninja на Mac OS X, сначала установите Homebrew, а затем, с помощью него, установите всё остальное. +``` +/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +brew install cmake ninja +``` + +Проверьте версию CMake: `cmake --version`. Если версия меньше 3.3, то установите новую версию с сайта https://cmake.org/download/ + + +# Необязательные внешние библиотеки + +ClickHouse использует для сборки некоторое количество внешних библиотек. Большинство из них не требуется отдельно устанавливать, так как они собираются вместе с ClickHouse, из исходников, которые расположены в submodules. Посмотреть набор этих библиотек можно в директории contrib. + +Одна библиотека не собирается из исходников, а используется из системы: Readline, и её рекомендуется установить. + +Ubuntu: `sudo apt install libreadline-dev` + +Mac OS X: `brew install readline` + +Впрочем, эти библиотеки не обязательны для работы и ClickHouse может быть собран без них. ICU используется для поддержки `COLLATE` в `ORDER BY` (например, для сортировки с учётом турецкого алфавита). Readline используется для более удобного набора команд в интерактивном режиме в clickhouse-client. + + +# Компилятор C++ + +В качестве компилятора C++ поддерживается GCC начиная с версии 9 или Clang начиная с версии 8. + +Официальные сборки от Яндекса, на данный момент, используют GCC, так как он генерирует слегка более производительный машинный код (разница в среднем до нескольких процентов по нашим бенчмаркам). Clang обычно более удобен для разработки. Впрочем, наша среда continuous integration проверяет около десятка вариантов сборки. + +Для установки GCC под Ubuntu, выполните: `sudo apt install gcc g++`. + +Проверьте версию gcc: `gcc --version`. Если версия меньше 9, то следуйте инструкции: https://clickhouse.yandex/docs/en/development/build/#install-gcc-9 + +Сборка под Mac OS X поддерживается только для компилятора Clang. Чтобы установить его выполните `brew install llvm` + +Если вы решили использовать Clang, вы также можете установить `libc++` и `lld`, если вы знаете, что это такое. При желании, установите `ccache`. + + +# Процесс сборки + +Теперь вы готовы к сборке ClickHouse. Для размещения собранных файлов, рекомендуется создать отдельную директорию build внутри директории ClickHouse: +``` +mkdir build +cd build +``` +Вы можете иметь несколько разных директорий (build_release, build_debug) для разных вариантов сборки. + +Находясь в директории build, выполните конфигурацию сборки с помощью CMake. +Перед первым запуском необходимо выставить переменные окружения, отвечающие за выбор компилятора (в данном примере это - gcc версии 9). + +Linux: +``` +export CC=gcc-9 CXX=g++-9 +cmake .. +``` + +Mac OS X: +``` +export CC=clang CXX=clang++ +cmake .. +``` +Переменная CC отвечает за компилятор C (сокращение от слов C Compiler), переменная CXX отвечает за выбор компилятора C++ (символ X - это как плюс, но положенный набок, ради того, чтобы превратить его в букву). + +Для более быстрой сборки, можно использовать debug вариант - сборку без оптимизаций. Для этого, укажите параметр `-D CMAKE_BUILD_TYPE=Debug`: +``` +cmake -D CMAKE_BUILD_TYPE=Debug .. +``` +Вы можете изменить вариант сборки, выполнив эту команду в директории build. + +Запустите ninja для сборки: +``` +ninja clickhouse-server clickhouse-client +``` +В этом примере собираются только нужные в первую очередь программы. + +Если вы хотите собрать все программы (утилиты и тесты), то запустите ninja без параметров: +``` +ninja +``` + +Для полной сборки требуется около 30 GB свободного места на диске или 15 GB для сборки только основных программ. + +При наличии небольшого количества оперативной памяти на компьютере, следует ограничить количество параллельных задач с помощью параметра `-j`: +``` +ninja -j 1 clickhouse-server clickhouse-client +``` +На машинах с 4 GB памяти, рекомендуется указывать значение 1, а если памяти до 8 GB, укажите значение 2. + +Если вы получили сообщение `ninja: error: loading 'build.ninja': No such file or directory`, значит конфигурация сборки прошла с ошибкой и вам необходимо посмотреть на сообщение об ошибке выше. + +В случае успешного запуска, вы увидите прогресс сборки - количество обработанных задач и общее количество задач. + +В процессе сборки могут появится сообщения `libprotobuf WARNING` про protobuf файлы в библиотеке libhdfs2. Это не имеет значения. + +При успешной сборке, вы получите готовый исполняемый файл `ClickHouse/build/dbms/programs/clickhouse`: +``` +ls -l dbms/programs/clickhouse +``` + + +# Запуск собранной версии ClickHouse + +Для запуска сервера из под текущего пользователя, с выводом логов в терминал и с использованием примеров конфигурационных файлов, расположенных в исходниках, перейдите в директорию `ClickHouse/dbms/programs/server/` (эта директория находится не в директории build) и выполните: + +``` +../../../build/dbms/programs/clickhouse server +``` + +В этом случае, ClickHouse будет использовать конфигурационные файлы, расположенные в текущей директории. Вы можете запустить `clickhouse server` из любой директории, передав ему путь к конфигурационному файлу в аргументе командной строки `--config-file`. + +Для подключения к ClickHouse с помощью clickhouse-client, в соседнем терминале, зайдите в директорию `ClickHouse/build/dbms/programs/` и выполните `clickhouse client`. + +Если вы получили сообщение `Connection refused` на Mac OS X или FreeBSD, то укажите для клиента 127.0.0.1 в качестве имени хоста: +``` +clickhouse client --host 127.0.0.1 +``` + +Вы можете заменить собранным вами ClickHouse продакшен версию, установленную в системе. Для этого, установите ClickHouse на свою машину по инструкции с официального сайта. Затем выполните: +``` +sudo service clickhouse-server stop +sudo cp ClickHouse/build/dbms/programs/clickhouse /usr/bin/ +sudo service clickhouse-server start +``` + +Обратите внимание, что `clickhouse-client`, `clickhouse-server` и другие, являеются симлинками на общий бинарник `clickhouse`. + +Также вы можете запустить собранный вами ClickHouse с конфигурационным файлом системного ClickHouse: +``` +sudo service clickhouse-server stop +sudo -u clickhouse ClickHouse/build/dbms/programs/clickhouse server --config-file /etc/clickhouse-server/config.xml +``` + + +# Среда разработки + +Если вы не знаете, какую среду разработки использовать, то рекомендуется использовать CLion. CLion является платным ПО, но его можно использовать бесплатно в течение пробного периода. Также он бесплатен для учащихся. CLion можно использовать как под Linux, так и под Mac OS X. + +Также в качестве среды разработки, вы можете использовать KDevelop или QTCreator. KDevelop - очень удобная, но нестабильная среда разработки. Если KDevelop вылетает через небольшое время после открытия проекта, вам следует нажать на кнопку "Stop All" как только он открыл список файлов проекта. После этого, KDevelop можно будет использовать. + +В качестве простых редакторов кода можно использовать Sublime Text или Visual Studio Code или Kate (все варианты доступны под Linux). + +На всякий случай заметим, что CLion самостоятельно создаёт свою build директорию, самостоятельно выбирает тип сборки debug по-умолчанию, для конфигурации использует встроенную в CLion версию CMake вместо установленного вами, а для запуска задач использует make вместо ninja. Это нормально, просто имейте это ввиду, чтобы не возникало путаницы. + + +# Написание кода + +Описание архитектуры ClickHouse: https://clickhouse.yandex/docs/ru/development/architecture/ + +Стиль кода: https://clickhouse.yandex/docs/ru/development/style/ + +Разработка тестов: https://clickhouse.yandex/docs/ru/development/tests/ + +Список задач: https://github.com/ClickHouse/ClickHouse/blob/master/dbms/tests/instructions/easy_tasks_sorted_ru.md + + +# Тестовые данные + +Разработка ClickHouse часто требует загрузки реалистичных наборов данных. Особенно это важно для тестирования производительности. Специально для вас мы подготовили набор данных, представляющий собой анонимизированные данные Яндекс.Метрики. Загрузка этих данных потребует ещё 3 GB места на диске. Для выполнения большинства задач разработки, загружать эти данные не обязательно. + +``` +sudo apt install wget xz-utils + +wget https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz +wget https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz + +xz -v -d hits_v1.tsv.xz +xz -v -d visits_v1.tsv.xz + +clickhouse-client + +CREATE TABLE test.hits ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree PARTITION BY toYYYYMM(EventDate) SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID), EventTime); + +CREATE TABLE test.visits ( CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8, VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32, Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String, EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32, SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32, SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16, UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16, FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8, Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), Params Array(String), `Goals.ID` Array(UInt32), `Goals.Serial` Array(UInt32), `Goals.EventTime` Array(DateTime), `Goals.Price` Array(Int64), `Goals.OrderID` Array(String), `Goals.CurrencyID` Array(UInt32), WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64, ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32, ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32, ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32, ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16, ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime, PredLastVisit Date, LastVisit Date, TotalVisits UInt32, `TraficSource.ID` Array(Int8), `TraficSource.SearchEngineID` Array(UInt16), `TraficSource.AdvEngineID` Array(UInt8), `TraficSource.PlaceID` Array(UInt16), `TraficSource.SocialSourceNetworkID` Array(UInt8), `TraficSource.Domain` Array(String), `TraficSource.SearchPhrase` Array(String), `TraficSource.SocialSourcePage` Array(String), Attendance FixedString(16), CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64, StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64, OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64, UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), `Market.Type` Array(UInt8), `Market.GoalID` Array(UInt32), `Market.OrderID` Array(String), `Market.OrderPrice` Array(Int64), `Market.PP` Array(UInt32), `Market.DirectPlaceID` Array(UInt32), `Market.DirectOrderID` Array(UInt32), `Market.DirectBannerID` Array(UInt32), `Market.GoodID` Array(String), `Market.GoodName` Array(String), `Market.GoodQuantity` Array(Int32), `Market.GoodPrice` Array(Int64), IslandID FixedString(16)) ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) SAMPLE BY intHash32(UserID) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID); + +clickhouse-client --max_insert_block_size 100000 --query "INSERT INTO test.hits FORMAT TSV" < hits_v1.tsv +clickhouse-client --max_insert_block_size 100000 --query "INSERT INTO test.visits FORMAT TSV" < visits_v1.tsv +``` + + +# Создание pull request + +Откройте свой форк репозитория в интерфейсе GitHub. Если вы вели разработку в бранче, выберите этот бранч. На странице будет доступна кнопка "Pull request". По сути, это означает "создать заявку на принятие моих изменений в основной репозиторий". + +Pull request можно создать, даже если работа над задачей ещё не завершена. В этом случае, добавьте в его название слово "WIP" (work in progress). Название можно будет изменить позже. Это полезно для совместного просмотра и обсуждения изменений, а также для запуска всех имеющихся тестов. Введите краткое описание изменений - впоследствии, оно будет использовано для релизных changelog. + +Тесты будут запущены, как только сотрудники Яндекса поставят для pull request тег "Can be tested". Результаты первых проверок (стиль кода) появятся уже через несколько минут. Результаты сборки появятся примерно через пол часа. Результаты основного набора тестов будут доступны в пределах часа. + +Система подготовит сборки ClickHouse специально для вашего pull request. Для их получения, нажмите на ссылку "Details" у проверки "Clickhouse build check". Там вы сможете найти прямые ссылки на собранные .deb пакеты ClickHouse, которые, при желании, вы даже сможете установить на свои продакшен серверы (если не страшно). + +Вероятнее всего, часть сборок не будет успешной с первого раза. Ведь мы проверяем сборку кода и gcc и clang, а при сборке с помощью clang включаются почти все существующие в природе warnings (всегда с флагом `-Werror`). На той же странице, вы сможете найти логи сборки - вам не обязательно самому собирать ClickHouse всеми возможными способами. diff --git a/docs/ru/extended_roadmap.md b/docs/ru/extended_roadmap.md new file mode 100644 index 00000000000..767f56d74f1 --- /dev/null +++ b/docs/ru/extended_roadmap.md @@ -0,0 +1,1720 @@ +# Планы разработки ClickHouse 2020. + +Здесь собраны важные задачи на 2020 год. Многие из них присутствуют в GitHub Issues. Данный текст следует рассматривать как рабочий черновик со сводкой и кратким описанием задач, ссылками и материалами для быстрого доступа на одной странице. Классификация задач условная. + +Так как ClickHouse - open-source продукт, мы хотим, чтобы рабочий процесс был также максимально открытым. В связи с этим, вам следует ожидать наличия на данной странице несколько большего количества деталей описания рабочего процесса, чем вы могли бы предположить - настолько близко, насколько рабочий процесс видят разработчики. Так как неотъемлимой частью процесса разработки является исправление недостатков продукта и улучшение качества кода, на данной странице вы найдёте весьма подробные описания таких деталей. Для других open-source продуктов такой подход к разработке обычно нехарактерен. Благодаря тому, что для каждой задачи указаны её зависимости, вы сможете понять, какие подготовительные работы требуются, что позволяет более точно понимать сроки реализации. + + +## 1. Хранение данных, индексация. + +### 1.1. Индексы по z-Order curve, normalized z-Order curve. + +Задача также относится к категории "17. Работа с географическими данными", так как geoHash - это частный случай z-Order curve. +Также связана с задачей 24.27 для нечёткого поиска полудубликатов строк, так как позволит индексировать min-hash. +Задача "normalized z-Order curve" в перспективе может быть полезна для БК и Метрики, так как позволяет смешивать OrderID и PageID и избежать дублирования данных. +В задаче также вводится способ индексации путём обращения функции нескольких аргументов на интервале, что имеет смысл для дальнейшего развития. + +Изначально делал [Андрей Чулков](https://github.com/achulkov2), ВШЭ, теперь доделывает [Ольга Хвостикова](https://github.com/stavrolia), но сроки немного сдвинуты из-за задачи 25.9. Будем надеятся на реализацию к концу ноября. Впрочем, [Андрей Чулков](https://github.com/achulkov2) скоро сможет помочь её доделать. + +### 1.2. Wait-free каталог баз данных. + +Делает [Александр Токмаков](https://github.com/tavplubix), первый рабочий вариант в декабре 2019. Нужно для DataLens и Яндекс.Метрики. + +Манипуляции с каталогом баз данных: запросы CREATE TABLE, DROP TABLE, RENAME TABLE и DATABASE, требуют синхронизации с помощью блокировок. Эта синхронизация становится весьма сложной, так как на неё полагается много внутренних структур данных. + +Предлагается реализовать альтернативный подход, в котором таблицы и базы данных являются всего лишь ссылками на persistent объекты. Подробное описание задачи: https://github.com/yandex/ClickHouse/issues/6787 + +### 1.3. Неблокирующие ALTER. + +И полностью immutable куски. Делает [Александр Сапин](https://github.com/alesapin). Готов приступить к задаче в конце ноября 2019. Нужно для Яндекс.Метрики. + +### 1.4. Нетранзитивные ALTER столбцов. + +Требует 1.3. Будет делать [Александр Сапин](https://github.com/alesapin). + +### 1.5. ALTER RENAME COLUMN. + +Требует 1.3. Будет делать [Александр Сапин](https://github.com/alesapin). + +### 1.6. Полиморфные куски данных. + +Делает [Антон Попов](https://github.com/CurtizJ), первый рабочий вариант в декабре. Пререквизит чтобы снизить сложность мелких INSERT, что в свою очередь нужно для 1.12, иначе задача 1.12 не сможет нормально работать. Особенно нужно для Яндекс.Облака. + +Данные в таблицах типа MergeTree в ClickHouse хранятся в виде набора независимых "кусков". Внутри куска, каждый столбец, а также индекс, хранится в отдельных файлах. Это сделано для возможности быстрых манипуляций со столбцами (пример - запрос ALTER DROP COLUMN). При вставке данных (INSERT), создаётся новый кусок. Для таблиц с большим количеством столбцов, запросы INSERT с маленьким количеством строк являются неэффективными, так как требуют создания большого количества файлов в файловой системе. Это является врождённой особенностью ClickHouse - одной из первой проблем, с которыми сталкиваются пользователи. Пользователям приходится буферизовывать данные и собирать их в более крупные пачки перед вставкой в ClickHouse. + +Для смягчения эффекта от этой проблемы, в ClickHouse существуют таблицы типа Buffer. Они накапливают данные в оперативке перед записью в другую таблицу. Впрочем, таблицы Buffer не являются полноценным решением проблемы из-за: - наличия блокировок при вставке; - переупорядочивание вставляемых данных; - неатомарность перекладывания данных из Buffer в результирующую таблицу. + +Вместо этого предлагается разрешить кускам таблиц типа MergeTree располагать данные в разных форматах. А именно: - в оперативной памяти; - на диске со всеми столбцами в одном файле; - на диске со столбцами в отдельных файлах: в зависимости от размера куска и прошедшего времени. Для размещения кусков в оперативной памяти, придётся также реализовать опциональную поддержку write-ahead log с настраиваемыми правилами по сбросу на диск. Это позволит избавиться от проблем с мелкими вставками для MergeTree таблиц. Для ReplicatedMergeTree таблиц, это решит проблему лишь частично. + +### 1.7. Буферизация и WAL в MergeTree. + +Требует 1.6. + +### 1.8. Перенос между разделами по TTL. + +Делает [Владимир Чеботарёв](https://github.com/excitoon), Altinity. Декабрь 2019. + +### 1.9. Использование TTL для прореживания данных. + +Будет делать Сорокин Николай, ВШЭ и Яндекс. + +Сейчас пользователь может задать в таблице выражение, которое определяет, сколько времени хранятся данные. Обычно это выражение задаётся относительно значения столбца с датой - например: удалять данные через три месяца. https://clickhouse.yandex/docs/ru/operations/table_engines/mergetree/#table_engine-mergetree-ttl + +Это может быть задано для всей таблицы (тогда строки целиком удаляются после указанного времени) или для отдельных столбцов (тогда данные столбца физически удаляются с диска, а строки в таблице остаются; при чтении значений столбца, они читаются как значения по-умолчанию). + +Но пользователи также хотят более продвинутый вариант этой функциональности: не удалять строки или столбцы целиком, а прореживать их - оставлять меньшее количество строк. + +И тут есть несколько вариантов: +1. По прошествии времени, оставлять каждую N-ую строку. +2. По прошествии времени, выполнять агрегацию данных, заменяя значения некоторых столбцов на значения агрегатных функций от множества значений в нескольких строках. + +Пункт 1 не представляет интереса, так как уже реализован с помощью TTL выражений для удаления данных. В качестве этого выражения можно прописать, например, cityHash64(*) % 10 = 0 ? now() : event_time + INTERVAL 3 MONTH. Правда как-то неудобно получается. + +А вот пункт 2 требуется продумать. Не очевидно даже, какой лучше использовать синтаксис для этого при создании таблицы. Но мы придумаем - сразу видно несколько вариантов. + +Частный случай такой задачи уже есть в https://clickhouse.yandex/docs/ru/operations/table_engines/graphitemergetree/ Но это было сделано для конкретной задачи. А надо обобщить. + +### 1.10. Пережатие старых данных в фоне. + +Будет делать Кирилл Барухов, ВШЭ, экспериментальная реализация к весне 2020. Нужно для Яндекс.Метрики. + +Алгоритмы сжатия типа LZ77 позволяют потратить больше времени на сжатие данных, чтобы сжать данные сильнее, но при этом без проигрыша по скорости разжатия данных. В частности, этим свойством обладает LZ4 и ZSTD, которые используются в ClickHouse. Это позволяет использовать свободные ресурсы CPU, когда сервер не нагружен, для пережатия данных, чтобы данные занимали меньше места на дисках, и при этом сохранить или даже улучшить скорость обработки запросов. + +В то же время, ClickHouse обычно используется для "импульсного" сценария нагрузки. Запрос от пользователя обрабатывается максимально быстро, используя все ресурсы CPU, но в среднем по времени, сервер недостаточно нагружен. + +Предлагается добавить в ClickHouse настройки по пережатию данных и фоновые потоки, выполняющие эту задачу. + +### 1.11. Виртуальная файловая система. + +Нужно для Яндекс.Облака. Делает Александр, Яндекс.Облако, а также Олег Ершов, ВШЭ и Яндекс. + +ClickHouse использует для хранения данных локальную файловую систему. Существует сценарий работы, в котором размещение старых (архивных) данных было бы выгодно на удалённой файловой системе. Если файловая система POSIX совместимая, то это не составляет проблем: ClickHouse успешно работает с Ceph, GlusterFS, MooseFS. Также востребованным является сценарий использования S3 (из-за доступности в облаке) или HDFS (для интеграции с Hadoop). Но эти файловые системы не являются POSIX совместимыми. Хотя для них существуют FUSE драйверы, но скорость работы сильно страдает и поддержка неполная. + +ClickHouse использует небольшое подмножество функций ФС, но в то же время, и некоторые специфические части: симлинки и хардлинки, O_DIRECT. Предлагается выделить всё взаимодействие с файловой системой в отдельный интерфейс. + +### 1.12. Экспериментальная реализация VFS поверх S3 и HDFS. + +Нужно для Яндекс.Облака. Требует 1.11. Желательно 1.6 и 1.18. +Делает Александр, Яндекс.Облако (сначала часть для S3), а также Олег Ершов, ВШЭ и Яндекс. + +### 1.13. Ускорение запросов с FINAL. + +Требует 2.1. Делает [Николай Кочетов](https://github.com/KochetovNicolai). Нужно для Яндекс.Метрики. + +### 1.14. Не писать столбцы, полностью состоящие из нулей. + +В очереди. Простая задача, является небольшим пререквизитом для потенциальной поддержки полуструктурированных данных. + +### 1.15. Возможность иметь разный первичный ключ в разных кусках. + +Сложная задача, только после 1.3. + +### 1.16. Несколько физических представлений для одного куска данных. + +Сложная задача, только после 1.3 и 1.6. Позволяет компенсировать 21.20. + +### 1.17. Несколько сортировок для одной таблицы. + +Сложная задача, только после 1.3 и 1.6. + +### 1.18. Отдельное хранение файлов кусков. + +Требует 1.3 и 1.6. Полная замена hard links на sym links, что будет лучше для 1.12. + + +## 2. Крупные рефакторинги. + +Для обоснования необходимости смотрите ссылки в описании других задач. + +### 2.1. Переделка конвейера выполнения запросов на Processors. + +Делает [Николай Кочетов](https://github.com/KochetovNicolai). Финальная стадия разработки. Включение по-умолчанию в конце декабря 2019. Удаление старого кода в начале 2020. + +### 2.2. Инфраструктура событий/метрик/ограничений/квот/трассировки. + +В очереди. https://gist.github.com/alexey-milovidov/d62d73222d83b9319dc519cbb13aeff6 + +### 2.3. Перенос столбцового ser/de из DataType в Column. + +В очереди. + +### 2.4. Перевод LowCardinality из DataType в Column. Добавление ColumnSparse. + +Требует 2.3. + +### 2.5. Версионирование состояний агрегатных функций. + +В очереди. + +### 2.6. Правая часть IN как тип данных. Выполнение IN в виде скалярного подзапроса. + +Требует 2.1. + +### 2.7. Нормализация Context. + +В очереди. + +### 2.8. Декларативный парсер запросов. + +Низкий приоритет. Задачу хочет сделать [Иван Лежанкин](https://github.com/abyss7) в свободное время, но пока ничего нет. + +### 2.9. Логгировние в format-стиле. + +Делает [Иван Лежанкин](https://github.com/abyss7). Низкий приоритет. + +### 2.10. Запрашивать у таблиц не столбцы, а срезы. + +В очереди. + +### 2.11. Разбирательство и нормализация функциональности для bitmap. + +В очереди. + +### 2.12. Декларативные сигнатуры функций. + +Задачу делает Алексей Миловидов. Прогресс 50% и разработка временно приостановлена. + +### 2.13. Каждая функция в отдельном файле. + +Задачу делает Алексей Миловидов. Прогресс 80%. Потребуется помощь других разработчиков. + +### 2.14. Все функции с состоянием переделать на FunctionBuilder. + +Долг [Николай Кочетов](https://github.com/KochetovNicolai). Сейчас код находится в переходном состоянии, что неприемлемо. + +### 2.15. Функция subscribe для IStorage. + +Для нормализации работы materialized views поверх Merge, Distributed, Kafka. + + +## 3. Документация. + +Здесь задачи только по инфраструктуре документации. + +### 3.1. Перенос документации по функциям в код. + +Требует 2.12 и 2.13. + +### 3.2. Перенос однородных частей документации в код. + +Требует 3.1. + +### 3.3. Исправить катастрофически отвратительно неприемлемый поиск по документации. + +[Иван Блинков](https://github.com/blinkov/) - очень хороший человек. Сам сайт документации основан на технологиях, не удовлетворяющих требованиям задачи, и эти технологии трудно исправить. + +### 3.4. + Добавить японский язык в документацию. + +Эту задачу сделает [Иван Блинков](https://github.com/blinkov/), до конца декабря 2019. Сделано. + + +## 4. Сетевое взаимодействие. + +### 4.1. Уменьшение числа потоков при распределённых запросах. + +[Никита Лапков](https://github.com/laplab), весна 2020. + +### 4.2. Спекулятивное выполнение запросов на нескольких репликах. + +[Никита Лапков](https://github.com/laplab), весна 2020. Нужно для Яндекс.Метрики. Требует 4.1. + +Если распределённый запрос затрагивает большое количество серверов, то время выполнения запросов часто становится большим из-за tail latencies - случайных редких замедлений отдельных серверов. Эту проблему можно избежать, отправляя один и тот же запрос сразу на несколько реплик, и используя данные с наиболее быстрой. + +Задача скрывает в себе много тонкостей, связанных с обработкой стадий выполнения запроса (соединение, обмен handshake, отправка запроса, получение заголовка результата, получение пакетов прогресса, получение данных), правильной возможностью настройки таймаутов, правильной отменой запросов. + +Сейчас для распределённых запросов используется по потоку на соединение. Это позволяет хорошо распараллелить вычисления над полученными данными и утилизировать сеть, но становится сильно избыточным для больших кластеров. Для примера, создание 1000 потоков для чтения данных из 1000 серверов кластера - лишь расходует ресурсы и увеличивает время выполнения запроса. Вместо этого необходимо использовать количество потоков не большее количества процессорных ядер, и мультиплексировать в одном потоке общение с серверами. Реализация нетривиальна, так как мультиплексировать необходимо каждую стадию общения по сети, включая установку соединения и обмен handshake. + +### 4.3. Ограничение числа одновременных скачиваний с реплик. + +Дмитрий Григорьев, ВШЭ. +Изначально делал Олег Алексеенков, но пока решение не готово, хотя там не так уж много доделывать. + +### 4.4. Ограничение сетевой полосы при репликации. + +Дмитрий Григорьев, ВШЭ. + +### 4.5. Возможность продолжить передачу куска данных при репликации после сбоя. + +Дмитрий Григорьев, ВШЭ. + +### 4.6. p2p передача для GLOBAL подзапросов. + +### 4.7. Ленивая загрузка множеств для IN и JOIN с помощью k/v запросов. + +### 4.8. Разделить background pool для fetch и merge. + +Дмитрий Григорьев, ВШЭ. +В очереди. Исправить проблему, что восстанавливающаяся реплика перестаёт мержить. Частично компенсируется 4.3. + + +## 5. Операции. + +### 5.1. Разделение задач на более мелкие куски в clickhouse-copier. + +Нужно для Метрики, в очереди, но исполнитель не назначен, есть шанс успеть в 2019. + +### 5.2. Автонастройка лимита на оперативку и размера кэшей. + +### 5.3. Встроенная ручка для Prometheus и, возможно, Solomon. + +Простая задача. https://github.com/Vdimir + +### 5.4. Opt-in сообщать в клиенте, если вышла новая версия. + +### 5.5. LTS релизы. + +Требует 7.5. Задачу хочет Метрика, Облако, БК, Маркет и Altinity. Первой LTS версией уже стала версия 19.14. + + +## 6. Инструментирование. + +### 6.1. Исправления сэмплирующего профайлера запросов. + +Михаил Филимонов, Altinity. Ноябрь 2019. Сделано. +Осталось ещё проверить работоспособность профайлера в первом потоке (что важно для INSERT). + +### 6.2. Добавление memory profiler. + +Сравнительно простая задача, но только для опытных разработчиков. Нужна всем. + +### 6.3. Учёт оперативки total расширить не только на запросы. + +Исправление долгоживущей проблемы с дрифтом учёта оперативки. Нужна для Метрики и БК. + +### 6.4. Поддержка perf events как метрик запроса. + +Делает Андрей Скобцов, ВШЭ. + +В Linux существует возможность получать в программе информацию о счётчиках производительности и событиях, относящихся к CPU и ядру ОС. Подробнее смотрите `man perf_event_open`. Предлагается добавить эти метрики в ClickHouse для инструментирования запросов. + +### 6.5. Эксперименты с LLVM X-Ray. + +Требует 2.2. + +### 6.6. Стек трейс для любых исключений. + +Сейчас есть стек трейс для почти всех, но не всех исключений. Требует 7.4. + +### 6.7. Таблица system.stack_trace. + +Сравнительно простая задача, но только для опытных разработчиков. + +### 6.8. Таблица system.crashes. + +Сравнительно простая задача, но только для опытных разработчиков. + +### 6.9. Отправлять информацию клиенту, если сервер падает по сигналу. + +### 6.10. Сбор общих системных метрик. + + +## 7. Сопровождение разработки. + +### 7.1. + ICU в submodules. + +Добавление в submodules также нужно для Аркадии (7.26). + +### 7.2. + LLVM в submodules. + +Сделал Алексей Миловидов. + +### 7.3. Обновление Poco. + +Алексанр Кузьменков. + +### 7.4. Включить libc++, libc++-abi при сборке с gcc. + +Сейчас включено только при сборке с clang, но продакшен сборка использует gcc. +Требует 7.2 и, возможно, 7.1 (только в случае новой версии ICU). + +### 7.5. Начать публиковать LTS релизы. + +[Александр Сапин](https://github.com/alesapin). + +### 7.6. Правильный статистический тест для comparison mode в clickhouse-performance-test. + +Задачу начал делать Дмитрий Рубашкин (ВШЭ). Сейчас продолжает [Александр Кузьменков](https://github.com/akuzm). + +### 7.7. Доделать тесты под MSan. + +Уже есть ASan, TSan, UBSan. Не хватает тестов под MSan. Они уже добавлены в CI, но не проходят. +[Александр Кузьменков](https://github.com/akuzm). + +### 7.8. Добавить clang-tidy. + +Уже есть PVS-Studio. Мы очень довольны, но этого недостаточно. + +### 7.9. Проверки на стиль имён с помощью clang-tidy. + +### 7.10. Включение UBSan и MSan в интеграционных тестах. + +UBSan включен в функциональных тестах, но не включен в интеграционных тестах. Требует 7.7. + +### 7.11. Включение *San в unit тестах. + +У нас мало unit тестов по сравнению с функциональными тестами и их использование не обязательно. Но они всё-равно важны и нет причин не запускать их под всеми видами sanitizers. + +### 7.12. Показывать тестовое покрытие нового кода в PR. + +Пока есть просто показ тестового покрытия всего кода. + +### 7.13. Включение аналога -Weverything в gcc. + +Мы используем -Wall -Wextra -Weverything -Werror. +При сборке с clang, -Weverything уже включено. Но в gcc есть уникальные warning-и, отсутствующие в clang. +Wolf Kreuzerkrieg. Возможно, его уже не интересует эта задача. +Низкий приоритет. Возможно, будет отменено. + +### 7.14. Альтернатива для readline и libedit. + +Тагир Кускаров, ВШЭ. Посмотрим на https://github.com/AmokHuginnsson/replxx + +Для ввода запросов в интерактивном режиме в клиенте командной строки clickhouse-client используется библиотека readline или libedit. + +Библиотеки readline и libedit обладает следующими недостатками: +- (исправлено в новых версиях readline) Очень низкая производительность вставки больших кусков текста. Вставка каждого следующего символа имеет сложность O(n = количество предыдущих символов) и при вставке 1 МБ текста, скорость падает до десятков байт в секунду. +- Крайне сложно или невозможно реализовать подсветку синтаксиса по мере набора текста, а также autocomplete без нажатия дополнительных клавиш для вызова. +- Лицензия GPL (для readline) препятствует её включению в кодовую базу продукта. +- Плохо работает навигация по истории, если история вкючает запросы, не помещающиеся на экран. +- История сохраняется лишь при завершении работы клиента. +- При параллельной работе нескольких клиентов с одним файлом истории, сохраняется история только одного из клиентов. +- Плохо работает история для многострочных запросов. +- Излишняя экономия пересылаемых данных, что часто приводит к остаткам мусора в терминале. + +Кроме того, имеются следующие сложно достижимые достоинства: +- Поддержка right-to-left текста; +- Поддержка editrc конфигураций. + +В качестве альтернатив можно рассмотреть следующие варианты: +- Linenoise от Salvatore Sanfilippo. Достоинства: простота и компактность кода; высокая скорость работы. Недостатки: отсутствует поддержка Unicode; отсутствует автоматический перенос текста, что затрудняет работу с многострочными запросами. +- Linenoise с патчами для поддержки Unicode. Недостаток: теряется преимущество по скорости работы. +- Fish shell. Не является библиотекой, но представляет собой отличный пример, как можно реализовать подстветку синтаксиса и удобный autocomplete. Поддерживает Unicode, но работает весьма медленно. +- Python Prompt Toolkit. Не является подходящим решением для интеграции в C++ проект. Хорошие возможности по подсветке синтаксиса и autocomplete. + +Вместо этого предлагается в качестве примера изучить прототип текстового редактора Kilo: https://viewsourcecode.org/snaptoken/kilo/ и реализовать всю необходимую функциональность. + + +### 7.15. + Замена libressl обратно на openssl. + +Поводом использования libressl послужило желание нашего хорошего друга из известной компании несколько лет назад. Но сейчас ситуация состоит в том, что openssl продолжает развиваться, а libressl не особо, и можно спокойно менять обратно. + +Нужно для Яндекс.Облака для поддержки TLS 1.3. + +### 7.16. tzdata внутри бинарника. + +Как в Аркадии, но только в качестве fallback. + +### 7.17. Доделать tgz пакеты. + +Уже давно собираются универсальные tgz пакеты, но по нелепой случайности из них исчез install скрипт. +[Александр Сапин](https://github.com/alesapin). Может делегировать эту задачу кому угодно. + +### 7.18.1. Доделать бинарники под Mac. + +Уже есть автосборка бинарников под Mac на каждый коммит и PR, но с недостатками. +[Иван Лежанкин](https://github.com/abyss7). Требует 7.1, 7.2. Рекомендуется 7.14. Сейчас не хватает по крайней мере SSL и ICU. Нужно для Яндекс.Облака. + +### 7.18. Поместить ссылку на собранные бинарники под Mac на сайт. + +Сейчас людям приходится делать несколько кликов, чтобы их скачать. +[Иван Лежанкин](https://github.com/abyss7) или [Александр Сапин](https://github.com/alesapin). + +### 7.19. + Доделать (проверить) автосборку под AArch64. + +https://github.com/ClickHouse/ClickHouse/issues/8027#issuecomment-566670282 +Проверили на настоящем сервере Huawei, а также в специальном Docker контейнере, который содержит внутри qemu-user-static. +Также можно проверить на Cavium, на Raspberry Pi а также на твоём Android телефоне. + + +### 7.20. Автосборка для FreeBSD x86_64. + +[Иван Лежанкин](https://github.com/abyss7). + +### 7.21. Автосборка для Linux ppc64. + +[Иван Лежанкин](https://github.com/abyss7). + +### 7.22. Дэшборд для pull requests. + +Дарья Петрова, УрФУ. + +Над ClickHouse одновременно работает большое количество разработчиков, которые оформляют свои изменения в виде pull requests. Когда непомерженных pull requests много, то возникает сложность с организацией работы - непонятно, на какой pull request смотреть в первую очередь. + +Предлагается реализовать простое одностраничное веб-приложение, в котором отображается список pull requests со следующей информацией: +- размер diff - количество изменённых строк; +- как давно было последнее обновление; +- типы изменённых файлов: C++, документация, скрипты сборки; +- наличие добавленных тестов; +- есть ли описание для changelog; +- изменены ли submodules; +- был ли разрешён запуск проверок CI; +- статусы проверок CI; +- количество approve от ревьюеров; + +Статусы проверок - наиболее важная часть. Так как для каждого PR выполняется несколько десятков проверок и наиболее медленные работают до нескольких часов, придётся: +- отображать сразу все проверки для каждого PR в виде красивой разноцветной матрицы с информацией по наведению мыши; +- отсортировать проверки по важности: например, если у внешнего разработчика проходят все проверки кроме стиля кода, то мы можем взять это в работу сами; +- если для предыдущего коммита проверка была завершена, а для последнего коммита ещё только идёт - то можно отображать в таблице статус предыдущей проверки более блёклым цветом. + +Предлагается реализовать несколько вариантов сортировок. Очевидное - по времени обновления, более интересно - некое ранжирование с целью выяснить, "что лучше взять в работу прямо сейчас". + +Похожие продукты уже есть, например: http://prs.mozilla.io/yandex:ClickHouse К сожалению, этот продукт заброшен, да и делает не совсем то, что нужно. По своему усмотрению, можно взять из него что-нибудь полезное. + +### 7.23. Функции для fuzzing. + +Андрей Некрашевич, ВШЭ. + +Fuzzing тестирование - это тестирование случайными данными. Мы рассмотрим несколько подходов к этой задачи: + +1. Добавление в SQL диалект ClickHouse функций для генерации случайных данных (пример - случайные бинарные строки заданной длины, случайные валидные UTF-8 строки) и "порчи" данных (например, поменять значения случайных бит с заданной частотой). Это будет использовано для тестирования SQL-функций ClickHouse. + +Можно добавить функции: +`randomString(length)` +`randomFixedString(length)` + - строка заданной длины с равномерно распределёнными случайными байтами; +`randomStringASCII(length)` +`randomStringUTF8(length)` + +`fuzzBits(s, inverse_probability)` - изменить каждый бит строки на противоположный с заданной вероятностью; +`fuzzBytes(s, inverse_probability)` - изменить каждый байт строки на равномерно случайный с заданной вероятностью; + +У каждой функции опциональный аргумент против склейки одинаковых выражений в запросе. + +Также можно сделать функции с детерминированным генератором случайных чисел (аргументом передаётся seed) для воспроизводимости тестовых кейсов. + +### 7.24. Fuzzing лексера и парсера запросов; кодеков и форматов. + +Андрей Некрашевич, ВШЭ. + +Продолжение 7.23. + +2. Использование AFL или LibFuzzer для тестирования отдельных частей кодовой базы ClickHouse. + +3. Генерация и выполнение случайных синтаксически корректных запросов на случайных данных. + +### 7.25. Синхронизация релизов в Аркадию. + +Изначально занимался Олег Алексеенков. Сейчас он перешёл работать в дружественный отдел, но обещает продолжать синхронизацию. +Затем, возможно, [Иван Лежанкин](https://github.com/abyss7). Но сейчас приостановлено, так как Максим из YT должен исправить регрессию производительности в анализе индекса. + +### 7.26. Побайтовая идентичность репозитория с Аркадией. + +Команда DevTools. Прогресс по задаче под вопросом. + +### 7.27. Запуск автотестов в Аркадии. + +Требует 7.26. + +### 7.29. Опции clickhouse install, stop, start вместо postinst, init.d, systemd скриптов. + +Низкий приоритет. + +### 7.30. Возможность переключения бинарных файлов на продакшене без выкладки пакетов. + +Низкий приоритет. + +### 7.31. Зеркалирование нагрузки между серверами. + +В очереди. Нужно для Яндекс.Метрики. + +### 7.32. Обфускация продакшен запросов. + +Роман Ильговский. Нужно для Яндекс.Метрики. + +Имея SQL запрос, требуется вывести структуру таблиц, на которых этот запрос будет выполнен, и заполнить эти таблицы случайными данными, такими, что результат этого запроса зависит от выбора подмножества данных. + +Для примера, если есть запрос `SELECT SearchPhrase, count(*) FROM table WHERE CounterID = 34 AND SearchPhrase LIKE '%ClickHouse%'`, то мы можем сделать вывод, что CounterID имеет числовой тип, а SearchPhrase - строковый. Заполнить таблицу данными, на которых отдельные условия `CounterID = 34` и `SearchPhrase LIKE '%ClickHouse%'` для некоторых строк выполнены, а для некоторых строк не выполнены. + +Обфускация запросов: имея секретные запросы и структуру таблиц, заменить имена полей и константы, чтобы запросы можно было использовать в качестве публично доступных тестов. + +### 7.33. Выкладывать патч релизы в репозиторий автоматически. + +[Александр Сапин](https://github.com/alesapin). Может делегировать эту задачу кому угодно. + +### 7.34. Бэкпортировать bugfix автоматически. + +[Александр Сапин](https://github.com/alesapin). Может делегировать эту задачу кому угодно. + +### 7.35. Начальные правила для авто-merge. + +Зелёные проверки и два ревью. +[Александр Сапин](https://github.com/alesapin). Может делегировать эту задачу кому угодно. + +### 7.36. Понятие доверенных контрибьюторов. + +Контрибьюторы, у которых есть 5 померженных PR. Для их новых PR автотесты запускаются сразу. +[Александр Сапин](https://github.com/alesapin). Может делегировать эту задачу кому угодно. + +### 7.37. Разобраться с repo.yandex.ru. + +Есть жалобы на скорость загрузки и неудобство maintenance, operations, visibility. + + +## 8. Интеграция с внешними системами. + +### 8.1. Поддержка ALTER MODIFY SETTING для Kafka. + +Altinity. + +### 8.2. Поддержка Mongo Atlas URI. + +[Александр Кузьменков](https://github.com/akuzm). + +### 8.3. Доработки globs (правильная поддержка диапазонов, уменьшение числа одновременных stream-ов). + +[Ольга Хвостикова](https://github.com/stavrolia). + +### 8.4. Унификация File, HDFS, S3 под URL. + +### 8.5. + Аутентификация в S3. + +[Владимир Чеботарёв](https://github.com/excitoon), Altinity. + +### 8.6. Kerberos аутентификация для HDFS и Kafka. + +Андрей Коняев, ArenaData. + +### 8.7. + Исправление мелочи HDFS на очень старых ядрах Linux. + +В ядрах 2.6 отсутствует один системный вызов, который библиотека hdfs3 использует без необходимости. +Сделал Amos Bird. + +### 8.8. Поддержка виртуальных столбцов с именем файла и путём. + +[Ольга Хвостикова](https://github.com/stavrolia). + +### 8.9. + Поддержка сжатых файлов (gz, bz) на чтение и запись. + +Сделал [Andrey Bodrov](https://github.com/apbodrov) + +### 8.10. Запись в табличную функцию ODBC. + +Артемий Бобровский, ВШЭ + +### 8.11. Движок таблиц для чтения из Mongo. + +Артемий Бобровский, ВШЭ + +### 8.12. Пропуск столбцов в форматах Parquet, ORC. + +Артемий Бобровский, ВШЭ + +### 8.13. Поддержка массивов в Parquet, ORC. + +Артемий Бобровский, ВШЭ + +### 8.14. Запись данных в ORC. + +Возможно, Андрей Коняев, ArenaData (зависит от желания). + +### 8.15. Запись данных в CapNProto. + +### 8.16. Поддержка формата Avro. + +Павел Круглов, ВШЭ и Яндекс. + +Формат Apache Avro является компактным структурированным построчным бинарным форматом данных с внешней схемой. Этот формат часто используется совместно с Kafka и поддержка его в качестве одного из форматов ввода-вывода в ClickHouse является востребованной пользователями. + +### 8.17. ClickHouse как MySQL реплика. + +Ильяс Адюгамов, ВШЭ. + +Реализовать возможность подписаться на row-based репликацию MySQL и сохранять полученные данные в CollapsingMergeTree или ReplacingMergeTree таблицы. Сторонние решения для этой задачи уже существуют: https://www.altinity.com/blog/2018/6/30/realtime-mysql-clickhouse-replication-in-practice Также существует стороннее решение для PostgreSQL: https://github.com/mkabilov/pg2ch + +Встроенная в ClickHouse возможность работать в качестве реплики MySQL даст преимущества для дальнейшего развития. + +### 8.18. + ClickHouse как Federated MySQL. + +Maxim Fedotov, Wargaming + Yuri Baranov, Яндекс. + +### 8.19. Интеграция с RabbitMQ. + +Ксения Сумарокова, ВШЭ. + +В ClickHouse часто используется потоковый импорт данных из распределённой очереди. Наиболее популярно использование совместно с Kafka. Эта возможность уже есть. + +Следующей по востребованности является система очередей RabbitMQ. Её поддержка в ClickHouse отсутствует. + +### 8.20. Интеграция с SQS. + +Низкий приоритет. + +### 8.21. Поддержка произвольного количества языков для имён регионов. + +Нужно для БК. Декабрь 2019. + +### 8.22. Поддержка синтаксиса для переменных в стиле MySQL. + +При парсинге запроса преобразовывать синтаксис вида `@@version_full` в вызов функции `getGlobalVariable('version_full')`. Поддержать популярные MySQL переменные. Может быть поможет Юрий Баранов, если будет энтузиазм. + +### 8.23. Подписка для импорта обновляемых и ротируемых логов в ФС. + +Желательно 2.15. + + +## 9. Безопасность. + +### 9.1. + Ограничение на хосты в запросах ко внешним системам. + +Михаил Коротов. + +### 9.2. Преднастроенные именованные соединения к внешним БД. + +Валерий Батурин, ВШЭ. + +ClickHouse предоставляет возможность обратиться к внешней базе данных из языка запросов. Это реализовано в виде табличных функций. В параметрах к табличной функции указывается адрес удалённой базы данных (хост, порт), а также аутентификационные данные (имя пользователя, пароль). Аутентификационные данные указываются в запросе в открытом виде и, таким образом, попадают в историю запросов и в логи, что компрометирует безопасность системы. + +Вместо этого предлагается описывать необходимые данные в конфигурационном файле сервера или в отдельном сервисе и ссылаться на них по именам. + +### 9.3. Поддержка TLS для ZooKeeper. + + +## 10. Внешние словари. + +### 10.1. Исправление зависания в библиотеке доступа к YT. + +Библиотека для доступа к YT не переживает учения. +Нужно для БК и Метрики. Поиск причин - [Александр Сапин](https://github.com/alesapin). Дальшейшее исправление возможно на стороне YT. + +### 10.2. Исправление SIGILL в библиотеке доступа к YT. + +Код YT использует SIGILL вместо abort. Это, опять же, происходит при учениях. +Нужно для БК и Метрики. Поиск причин - [Александр Сапин](https://github.com/alesapin). Дальшейшее исправление возможно на стороне YT. + +### 10.3. Возможность чтения данных из статических таблиц в YT словарях. + +Нужно для БК и Метрики. + +### 10.4. Словарь из YDB (KikiMR). + +Нужно для Метрики, а делать будет таинственный незнакомец из команды KikiMR (под вопросом). + +### 10.5. Закрытие соединений и уменьшение числа соединений для MySQL и ODBC. + +Нужно для Метрики. + +### 10.6. Словари из Cassandra и Couchbase. + +### 10.7. Поддержка Nullable в словарях. + +Артём Стрельцов, Николай Дегтеринский, Наталия Михненко, ВШЭ. + +### 10.8. Поддержка массивов в словарях. + +Артём Стрельцов, Николай Дегтеринский, Наталия Михненко, ВШЭ. + +### 10.9. Уменьшение блокировок для cache словарей за счёт одновременных запросов одного и того же. + +Нужно для БК, но мотивация задачи находится под вопросом, так как есть рабочее предположение о том, что данная задача не устраняет причину проблемы. + +### 10.10. Возможность использования старых значений из cache словаря пока они перезапрашиваются. + +Нужно для БК и Метрики. + +### 10.11. Возможность исключительно асинхронных запросов в cache словарях. + +Нужно для БК и Метрики. Требует 10.10. + +### 10.12. Layout direct для словарей. + +Артём Стрельцов, Николай Дегтеринский, Наталия Михненко, ВШЭ. + +### 10.13. Использование Join как generic layout для словарей. + +Артём Стрельцов, Николай Дегтеринский, Наталия Михненко, ВШЭ. + +### 10.14. Поддержка всех типов в функции transform. + +### 10.15. Использование словарей как специализированного layout для Join. + +### 10.16. Словари на локальном SSD. + +Никита Васильев, ВШЭ и Яндекс. + +Реализовать в ClickHouse специализированный движок таблиц, подходящий для быстрых key-value запросов и оптимизированный для расположения данных на SSD. Это может быть: реализация на основе RocksDB; сериализованные RowBinary данные с индексом в оперативке; секретная очень эффективная структура данных, о которой я расскажу. + +Использовать эту структуру данных как отдельный вид словарей, как источник для cache словарей или как дополнительный уровень кэширования для cache словарей. + +### 10.17. Локальный дамп состояния словаря для быстрого старта сервера. + +### 10.18. Таблица Join или словарь на удалённом сервере как key-value БД для cache словаря. + +### 10.19. Возможность зарегистрировать некоторые функции, использующие словари, под пользовательскими именами. + + +## 11. Интерфейсы. + +### 11.1. Вставка состояний агрегатных функций в виде кортежа аргументов или массива кортежей аргументов. + +### 11.2. Возможность использовать движок JDBC из коробки. + +Нужно разобраться, как упаковывать Java в статический бинарник, возможно AppImage. Или предоставить максимально простую инструкцию по установке jdbc-bridge. Может быть будет заинтересован Александр Крашенинников, Badoo, так как он разработал jdbc-bridge. + +### 11.3. Интеграционные тесты ODBC драйвера путём подключения ClickHouse к самому себе через ODBC. + +Денис Глазачев, Altinity. + +### 11.4. Исправление упячек с типами Date и Decimal в clickhouse-cpp. + +### 11.5. Поддержка TLS в clickhouse-cpp. + +А знаете ли вы, что библиотеку clickhouse-cpp разрабатывал один хороший человек в свободное время? + +### 11.6. Интеграционные тесты clickhouse-cpp. + +### 11.7. Интерактивный режим работы программы clickhouse-local. + +### 11.8. Поддержка протокола PostgreSQL. + +Элбакян Мовсес Андраникович, ВШЭ. + +В ClickHouse в прошлом году добавили поддержку wire-протокола MySQL. PostgreSQL, так же как MySQL, использует несложный протокол общения между клиентом и сервером, но свой собственный. Поддержка этого протокола является востребованной и откроет новые возможности для ClickHouse. + +### 11.9. Доработки ODBC драйвера. + +Денис Глазачев, Altinity. Хороший прогресс по этой задаче. + +### 11.10. Преднастроенные HTTP handlers для запросов. + +zhang2014 + +Возможность описать в конфигурационном файле handler (путь в URL) для HTTP запросов к серверу, которому соответствует некоторый параметризованный запрос. Пользователь может вызвать этот обработчик и не должен передавать SQL запрос. + + +## 12. Управление пользователями и доступом. + +### 12.1. Role Based Access Control. + +[Виталий Баранов](https://github.com/vitlibar). Финальная стадия разработки, рабочая версия в декабре 2019. + +### 12.2. Управление пользователями и правами доступа с помощью SQL запросов. + +[Виталий Баранов](https://github.com/vitlibar). Финальная стадия разработки, рабочая версия в декабре 2019. + +### 12.3. Подключение справочника пользователей и прав доступа из LDAP. + +[Виталий Баранов](https://github.com/vitlibar). Требует 12.1. + +### 12.4. Подключение IDM системы Яндекса как справочника пользователей и прав доступа. + +Пока низкий приоритет. Нужно для Метрики. Требует 12.3. + +### 12.5. Pluggable аутентификация с помощью Kerberos (возможно, подключение GSASL). + +[Виталий Баранов](https://github.com/vitlibar). Требует 12.1. + +### 12.6. Информация о пользователях и квотах в системной таблице. + +[Виталий Баранов](https://github.com/vitlibar). Требует 12.1. + + +## 13. Разделение ресурсов, multi-tenancy. + +### 13.1. Overcommit запросов по памяти и вытеснение. + +Требует 2.1. Способ реализации обсуждается. + +### 13.2. Общий конвейер выполнения на сервер. + +Требует 2.1. [Николай Кочетов](https://github.com/KochetovNicolai). + +### 13.3. Пулы ресурсов. + +Требует 13.2 или сможем сделать более неудобную реализацию раньше. + + +## 14. Диалект SQL. + +### 14.1. Исправление семантики CAST для Nullable. + +Нужно для DataLens. А также для внедрения в BI инструмент Looker. + +### 14.2. Поддержка WITH для подзапросов. + +### 14.3. Поддержка подстановок для множеств в правой части IN. + +### 14.4. Поддержка подстановок для идентификаторов (имён) в SQL запросе. + +zhang2014 + +### 14.5. Поддержка задания множества как массива в правой части секции IN. + +Василий Немков, Altinity, делал эту задачу, но временно приостановил работу над ней в пользу других задач. + +### 14.6. Глобальный scope для WITH. + +### 14.7. Nullable для WITH ROLLUP, WITH CUBE, WITH TOTALS. + +Простая задача. + +### 14.8. Модификаторы DISTINCT, ORDER BY для агрегатных функций. + +Софья Борзенкова, ВШЭ. + +В ClickHouse поддерживается вычисление COUNT(DISTINCT x). Предлагается добавить возможность использования модификатора DISTINCT для всех агрегатных функций. Например, AVG(DISTINCT x) - вычислить среднее значение для всех различных значений x. Под вопросом вариант, в котором фильтрация уникальных значений выполняется по одному выражению, а агрегация по другому. + +Результат некоторых агрегатных функций зависит от порядка данных. Предлагается реализовать модификатор ORDER BY, задающий порядок явно. Пример: groupArray(x ORDER BY y, z). + +### 14.9. Поддержка запроса EXPLAIN. + +Требует 2.1. [Николай Кочетов](https://github.com/KochetovNicolai). + +### 14.10. arrayReduce как функция высшего порядка. + +### 14.11. Функции для grouping sets. + +### 14.12. Функции обработки временных рядов. + +Сложная задача, так как вводит новый класс функций и требует его обработку в оптимизаторе запросов. + +В time-series СУБД нужны функции, которые зависят от последовательности значений. Или даже от последовательности значений и их меток времени. Примеры: moving average, exponential smoothing, derivative, Holt-Winters forecast. Вычисление таких функций поддерживается в ClickHouse лишь частично. Так, ClickHouse поддерживает тип данных "массив" и позволяет реализовать эти функции как функции, принимающие массивы. Но гораздо удобнее для пользователя было бы иметь возможность применить такие функции к таблице (промежуточному результату запроса после сортировки). + +Это требует введение нового класса функций (помимо обычных и агрегатных функций) - такие функции будут иметь в коде ClickHouse свой собственный интерфейс, и их вычисление придётся отдельно учитывать в конвейере выполнения запросов. Для примера, вычисление обычных функций тривиально распараллеливается по процессорным ядрам и по серверам; вычисление агрегатных функций распараллеливается с некоторыми особенностями (работа с промежуточными состояниями вычислений, операция merge); а для функций по обработке временных рядов этот вопрос остаётся открытым - возможно, их придётся вычислять на одном сервере и в одном потоке. + +### 14.13. Применимость функций высшего порядка для кортежей и Nested. + +### 14.14. Неявные преобразования типов констант. + +Требует 2.12. + +### 14.15. Неявные преобразования типов под настройкой. + +Требует 2.12. Для внедрения в BI инструмент Looker. + +### 14.16. Синонимы для функций из MySQL. + +### 14.17. Ввести понятие stateful функций. + +zhang2014. +Для runningDifference, neighbour - их учёт в оптимизаторе запросов. +В интерфейсе уже сделано. Надо проверить, что учитывается в нужных местах (например, что работает predicate pushdown сквозь ORDER BY, если таких функций нет). + +### 14.18. UNION DISTINCT и возможность включить его по-умолчанию. + +Для BI систем. + +### 14.19. Совместимость парсера типов данных с SQL. + +Для BI систем. + +### 14.20. Позиционные аргументы для GROUP BY и ORDER BY. + +Тривиально и используется многими системами, но не входит в стандарт SQL. + +### 14.21. Приведение типов для IN (подзапрос) и для JOIN. + + +## 15. Улучшение поддержки JOIN. + +### 15.1. Доведение merge JOIN до продакшена. + +Артём Зуйков. Сейчас merge JOIN включается вручную опцией и всегда замедляет запросы. Хотим, чтобы он замедлял запросы только когда это неизбежно. +Кстати, смысл merge JOIN появляется только совместно с 15.2 и 15.3. + +### 15.2. Прокидывание условий в OUTER JOIN. + +Возможно, Артём Зуйков, но задача ещё не продумана до конца. Возможно, требует 2.1. + +### 15.3. Логический вывод для цепочек вида ON t1.x = t2.y WHERE t1.x = 10 + +Возможно, Артём Зуйков. Для полноценной работы 15.2. + +### 15.4. Distributed JOIN с помощью перераспределения данных. + +Артём Зуйков. + +### 15.5. Использование ключа таблицы для оптимизации merge JOIN. + +### 15.6. + SEMI и ANTI JOIN. + +Артём Зуйков. + + +## 16. Типы данных и функции. + +### 16.1. DateTime64. + +Василий Немков, Altinity, декабрь 2019. + +### 16.2. Тип данных для JSON. + +zhang2014 + +### 16.3. Поддержка неконстантных аргументов с регулярными выражениями в функциях. + +Данила Кутенин, но только после секретного изменения в работе. + +### 16.4. Функция rowNumberForKey. + +### 16.5. Функции для XML и HTML escape. + +### 16.6. Функции нормализации и хэширования SQL запросов. + + +## 17. Работа с географическими данными. + +### 17.1. Гео-словари для определения региона по координатам. + +[Андрей Чулков](https://github.com/achulkov2), Антон Кваша, Артур Петуховский, ВШЭ. +Будет основано на коде от Арслана Урташева. + +ClickHouse не является geospatial СУБД. Тем не менее, в ClickHouse есть несколько функций для таких задач. Например, функция `pointInPolygon` позволяет быстро проверить попадание точек в полигон на плоскости. При этом, полигон задаётся в явном виде и должен быть константным для вызова функции (то есть - проверяется принадлежность многих точек одному полигону). Эта функциональность нужна, например, для рекламного таргетинга мобильных устройств по координатам. + +Похожая, но более сложная задача, которую ClickHouse пока не умеет решать - определение полигона среди множества полигонов, в которые попадают точки. Для примера: определение района города по географическим координатам. Для решения этой задачи нужно будет реализовать поддержку словарей с полигонами, в которых данные проиндексированы для быстрого поиска. + +### 17.2. GIS типы данных и операции. + +Алексей Коряков, Алексей Илюхов, ВШЭ, Яндекс.Карты. + +Реализовать в ClickHouse типы данных для задач обработки геоинформационных данных: Point, Line, MultiLine, Polygon и операции над ними - проверка вхождения, пересечения. Вариантом минимум будет реализация этих операций в евклидовой системе координат. Дополнительно - на сфере и WGS84. + +### 17.3. + Ускорение greatCircleDistance. + +[Ольга Хвостикова](https://github.com/stavrolia), основано на коде Андрея Аксёнова, получено разрешение на использование кода. + +### 17.4. Ускорение geohash с помощью библиотеки из Аркадии. + +Предположительно, [Андрей Чулков](https://github.com/achulkov2). Получено одобрение от руководства. + +### 17.5. Проверки в функции pointInPolygon. + +[Николай Кочетов](https://github.com/KochetovNicolai). Сейчас функция тихо не работает в случае полигонов с самопересечениями, надо кидать исключение. + + +## 18. Машинное обучение и статистика. + +### 18.1. Инкрементальная кластеризация данных. + +Александр Кожихов, Максим Кузнецов. Обнаружена фундаментальная проблема в реализации, доделывает предположительно [Николай Кочетов](https://github.com/KochetovNicolai). Он может делегировать задачу кому угодно. + +### 18.2. Агрегатные функции для статистических тестов. + +Артём Цыганов, Руденский Константин Игоревич, Семёнов Денис, ВШЭ. + +Предлагается реализовать в ClickHouse статистические тесты (Analysis of Variance, тесты нормальности распределения и т. п.) в виде агрегатных функций. Пример: `welchTTest(value, sample_idx)`. + +### 18.3. Инфраструктура для тренировки моделей в ClickHouse. + +В очереди. Возможно, Александр Кожихов. У него сначала идёт задача 24.26. + + +## 19. Улучшение работы кластера. + +### 19.1. Параллельные кворумные вставки без линеаризуемости. + +Александра Латышева, ВШЭ и Яндекс. + +Репликация данных в ClickHouse по-умолчанию является асинхронной без выделенного мастера. Это значит, что клиент, осуществляющий вставку данных, получает успешный ответ после того, как данные попали на один сервер; репликация данных по остальным серверам осуществляется в другой момент времени. Это ненадёжно, потому что допускает потерю только что вставленных данных при потере лишь одного сервера. + +Для решения этой проблемы, в ClickHouse есть возможность включить "кворумную" вставку. Это значит, что клиент, осуществляющий вставку данных, получает успешный ответ после того, как данные попали на несколько (кворум) серверов. Обеспечивается линеаризуемость: клиент, получает успешный ответ после того, как данные попали на несколько реплик, *которые содержат все предыдущие данные, вставленные с кворумом* (такие реплики можно называть "синхронными"), и при запросе SELECT можно выставить настройку, разрешающую только чтение с синхронных реплик. + +Если бы свойства линеаризуемости не было, то для трёх серверов A, B, C, значения кворума = 2, и для трёх вставок данных 1, 2, 3, возможна ситуация, что первая вставка прошла на серверы A и B, вторая прошла на серверы B и C, а третья - на серверы A и C, и теперь ни один из серверов не содержит полный набор данных 1, 2, 3. + +Как ни странно, такое свойство не нужно большинству пользователей. Оно запрещает параллельно выполняющиеся вставки. А пользователи хотят вставлять данные надёжно (на более чем одну реплику), но не важно, в каком порядке. Предлагается сделать опцию, которая отключает линеаризуемость. + +Иногда пользователь хочет реализовать кворумную вставку вручную: просто соединиться с несколькими репликами и вставть на них одинаковые данные (чтобы обеспечить надёжную вставку, не ориентируясь на то, как работает механизм репликации). Сейчас ожидания пользователя не оправдываются. В ClickHouse есть механизм дедупликации для обеспечения идемпотентности вставок. Вторая вставка с такими же данными (пусть даже на другую реплику) будет проигнорирована. Надо сделать так, чтобы вместо этого, вставка одинаковых данных на другую реплику, имела такой же эффект, как если бы эти данные были получены с помощью механизма репликации. + +### 19.2. Подключение Etcd или Consul как альтернативы ZooKeeper. + +Алексей Лёвушкин, ВШЭ и Яндекс. + +Для координации реплик в ClickHouse используется ZooKeeper. Многие пользователи ClickHouse хотели бы иметь возможность использовать для координации некоторые другие системы вместо ZooKeeper. Рассматриваемыми вариантами таких систем являются Etcd, Consul, FoundationDB. Это весьма проблематично, так как эти системы существенно отличаются по интерфейсам и возможностям. Тем не менее, для того, чтобы эта задача стала возможной, в ClickHouse обобщён интерфейс взаимодействия с ZooKeeper, и теперь на его место можно подставлять другие реализации. + +В прошлом году, Алексей добавил модельную реализацию (mock) интерфейса ZooKeeper для тестирования. Сейчас предлагается сделать реализацию поверх Etcd, а также расширить возможности тестовой реализации. + +### 19.3. Подключение YT Cypress или YDB как альтернативы ZooKeeper. + +Hold. Полезно для заказчиков внутри Яндекса, но есть риски. + +### 19.4. internal_replication = 'auto'. + +### 19.5. Реплицируемые базы данных. + +В очереди, возможно Валерий Батурин, ВШЭ. + +Репликация в ClickHouse работает на уровне отдельных таблиц. Это является очень гибким решением: на одном сервере одна из таблиц может быть не реплицирована, другая иметь двухкратную репликацию, а третья - реплицирована по всем серверам. Но если все таблицы в базе данных реплицированы одинаковым образом. то это затрудняет управление кластером. Например, при восстановлени сервера, требуется отдельно создавать реплику для каждой таблицы. + +Предлагается реализовать "движок баз данных", который осуществляет репликацию метаданных (множество имеющихся таблиц и лог DDL операций над ними: CREATE, DROP, RENAME, ALTER). Пользователь сможет создать реплицируемую базу данных; при её создании или восстановлении на другом сервере, все реплицируемые таблицы будут созданы автоматически. + +### 19.6. Одновременный выбор кусков для слияния многими репликами, отказ от leader election в ZK. + +### 19.7. Возможность записи данных при недоступности ZK и отказ от линейного порядка кусков в большинстве случаев. + +### 19.8. Отказ от хранения в ZK множества кусков для каждой реплики отдельно. + +### 19.9. Отказ от хранения в ZK лога вставок и мержей. Обмен данными о кусках напрямую. + +### 19.10. Облачные таблицы. + +Требует 1.6, 19.1, 19.6, 19.7, 19.8, 19.9. + + +## 20. Мутации данных. + +Пока все задачи по точечным UPDATE/DELETE имеют низкий приоритет, но ожидаем взять в работу в середине 2020. + +### 20.1. Поддержка DELETE путём запоминания множества затронутых кусков и ключей. + +### 20.2. Поддержка DELETE путём преобразования множества ключей в множество row_numbers на реплике, столбца флагов и индекса по диапазонам. + +### 20.3. Поддержка ленивых DELETE путём запоминания выражений и преобразования к множеству ключей в фоне. + +### 20.4. Поддержка UPDATE с помощью преобразования в DELETE и вставок. + + +## 21. Оптимизации производительности. + +### 21.1. + Параллельный парсинг форматов. + +Начинал Олег Ершов, доделывает Никита Михайлов, помогает [Александр Кузьменков](https://github.com/akuzm). Готово. + +### 21.2. Параллельное форматирование форматов. + +После 21.1, предположительно Никита Михайлов. Задача сильно проще чем 21.1. + +### 21.3. Исправление низкой производительности анализа индекса в случае большого множества в секции IN. + +Нужно всем (Zen, БК, DataLens...) Пока ещё не выбран исполнитель. + +### 21.4. Использование ORDER BY ключа для оптимизации GROUP BY и DISTINCT. + +Дмитрий Рубашкин, ВШЭ. Помогает Антон Попов. + +Если таблица имеет ключ сортировки, то возможно эффективное чтение упорядоченных данных. Если запрос содержит операцию GROUP BY, содержащую по крайней мере префикс от ключа сортировки таблицы, либо инъективные функции от него, то возможно более эффективное выполнение GROUP BY: промежуточный результат агрегации финализируется и отправляется клиенту как только в потоке данных при чтении из таблицы встретился следующий ключ. + +Аналогичную оптимизацию следует реализовать для DISTINCT и LIMIT BY. + +В прошлом году, аналогичное решение сделали для операции ORDER BY. + +### 21.5. Распараллеливание INSERT при INSERT SELECT, если это необходимо. + +[Vxider](https://github.com/Vxider), ICT + +### 21.6. Уменьшение числа потоков для SELECT в случае тривиального INSERT SELECT. + +### 21.7. Кэш результатов запросов. + +[Achimbab](https://github.com/achimbab). + +### 21.8. Взаимная интеграция аллокатора и кэша. + +Михаил Кот, ВШЭ. Задача сложная и рискованная. + +Для выделения памяти, аллокаторы запрашивают её у операционной системы (`mmap`). Это возможно только для достаточно крупных кусков памяти является довольно медленной операцией. Поэтому, современные аллокаторы кэшируют крупные куски памяти в программе. При вызове free, кусок памяти, как правило, не отдаётся ОС, а остаётся для последующего переиспользования. Для выделения мелких кусков памяти, крупные куски разбиваются с помощью специальных структур данных (free-list, heap, bitmap). Для уменьшения contention в многопоточных программах, эти структуры также делаются thread-локальными. + +Часто в программе есть кэши некоторых данных. Например - кэш данных после разжатия, использующийся чтобы сэкономить на повторных запросах одних и тех же данных. При вытеснении из кэша, блок данных освобождается (`free`) и данные, бывшие в кэше, становятся недоступными для переиспользования. Но если принимать во внимание то, как работает аллокатор памяти, то оказывается, что после освобождения памяти, данные всё ещё остаются доступными в программе. И если этот кусок памяти не будет выделен аллокатором снова, его можно было бы продолжить использовать в качестве кэша. Иными словами, в программе есть domain-specific кэш, а аллокатор имеет свой кэш, и они не знают друг о друге. + +Для domain-specific кэшей (как например, кэш разжатых данных) выгодно, чтобы они использовали как можно больший объём свободной памяти. Но в этом случае, памяти может не хватить для других структур данных в программе. Если аллокатор памяти знает про кэш, то выделение памяти можно было бы делать путём вытеснения данных из кэша. + +### 21.8.1. Отдельный аллокатор для кэшей с ASLR. + +В прошлом году задачу пытался сделать Данила Кутенин с помощью lfalloc из Аркадии и mimalloc из Microsoft, но оба решения не были квалифицированы для использования в продакшене. Успешная реализация задачи 21.8 отменит необходимость в этой задаче, поэтому холд. + +### 21.9. Исправить push-down выражений с помощью Processors. + +[Николай Кочетов](https://github.com/KochetovNicolai). Требует 2.1. + +### 21.10. + Улучшение эвристики PREWHERE. + +Amos Bird. + +### 21.11. Peephole оптимизации запросов. + +Руслан Камалов, Михаил Малафеев, Виктор Гришанин, ВШЭ + +Реализовать в ClickHouse оптимизации запросов, основанные на упрощении отдельных небольших кусков выражений (так называемые "peephole" оптимизации). Примеры: +- Замена цепочек if на multiIf. +- Удаление min/max/any-агрегатов от выражений от ключей GROUP BY. +- Вынесение арифметических операций из агрегатных функций; +- Вынесение любых функций наружу any, anyLast. +- При GROUP BY по transform или if по строкам, замена строк на Enum. + +### 21.12. Алгебраические оптимизации запросов. + +Руслан Камалов, Михаил Малафеев, Виктор Гришанин, ВШЭ + +Реализовать в ClickHouse оптимизации запросов, основанные на алгебраических свойствах функций. Примеры: +- Обращение инъективных функций в сравнениях на равенство. +- Вынесение инъективных функцию наружу uniq. +- Удаление монотонных функций из ORDER BY. +- Удаление избыточных выражений из ORDER BY. +- Удаление из GROUP BY функций от других ключей GROUP BY. +- Удаление дублирующихся DISTINCT, ORDER BY из подзапросов. + +### 21.13. Fusion агрегатных функций. + +После или совместно с 21.11. + +### 21.14. Оптимизация запросов с помощью constraints. + +Мария Нефедова, ВШЭ. + +Constraints позволяют задать выражение, истинность которого проверяется при вставке данных в таблицу. Предположение о том, что выражение истинно, может использоваться и для оптимизации запросов. Например, встретив в запросе точно такое же выражение, можно заменить его на константу 1. + +Если выражение содержит равенство, то встретив в запросе одну из частей равенства, её можно заменить на другую часть равенства, если это сделает проще чтение данных или вычисление выражения. Например, задан constraint: `URLDomain = domain(URL)`. Значит, выражение `domain(URL)` можно заменить на `URLDomain`. + +### 21.15. Многоступенчатое чтение данных вместо PREWHERE. + +Требует 2.1 и 21.10. + +### 21.16. Оптимизация GROUP BY с большим количеством агрегатных функций путём вычисления в два прохода. + +Нужно для БК. + +### 21.17. Оптимизация GROUP BY при наличии ORDER BY по тем же ключам с LIMIT. + +Нужно для БК. + +### 21.18. Внутренняя параллелизация мержа больших состояний агрегатных функций. + +### 21.19. Оптимизация сортировки. + +Василий Морозов, Арслан Гумеров, Альберт Кидрачев, ВШЭ. +В прошлом году задачу начинал делать другой человек, но не добился достаточного прогресса. + +1. Оптимизация top sort. + +В ClickHouse используется неоптимальный вариант top sort. Суть его в том, что из каждого блока достаётся top N записей, а затем, все блоки мержатся. Но доставание top N записей у каждого следующего блока бессмысленно, если мы знаем, что из них в глобальный top N войдёт меньше. Конечно нужно реализовать вариацию на тему priority queue (heap) с быстрым пропуском целых блоков, если ни одна строка не попадёт в накопленный top. + +2. Рекурсивный вариант сортировки по кортежам. + +Для сортировки по кортежам используется обычная сортировка с компаратором, который в цикле по элементам кортежа делает виртуальные вызовы `IColumn::compareAt`. Это неоптимально - как из-за короткого цикла по неизвестному в compile-time количеству элементов, так и из-за виртуальных вызовов. Чтобы обойтись без виртуальных вызовов, есть метод `IColumn::getPermutation`. Он используется в случае сортировки по одному столбцу. Есть вариант, что в случае сортировки по кортежу, что-то похожее тоже можно применить... например, сделать метод `updatePermutation`, принимающий аргументы offset и limit, и допереставляющий перестановку в диапазоне значений, в которых предыдущий столбец имел равные значения. + +3. RadixSort для сортировки. + +Один наш знакомый начал делать задачу по попытке использования RadixSort для сортировки столбцов. Был сделан вариант indirect сортировки (для `getPermutation`), но не оптимизирован до конца - есть лишние ненужные перекладывания элементов. Для того, чтобы его оптимизировать, придётся добавить немного шаблонной магии (на последнем шаге что-то не копировать, вместо перекладывания индексов - складывать их в готовое место). Также этот человек добавил метод MSD Radix Sort для реализации radix partial sort. Но даже не проверил производительность. + +Наиболее содержательная часть задачи может состоять в применении Radix Sort для сортировки кортежей, расположенных в оперативке в виде Structure Of Arrays неизвестного в compile-time размера. Это может работать хуже, чем то, что описано в пункте 2... Но попробовать не помешает. + +4. Three-way comparison sort. + +Виртуальный метод `compareAt` возвращает -1, 0, 1. Но алгоритмы сортировки сравнениями обычно рассчитаны на `operator<` и не могут получить преимущества от three-way comparison. А можно ли написать так, чтобы преимущество было? + +5. pdq partial sort + +Хороший алгоритм сортировки сравнениями `pdqsort` не имеет варианта partial sort. Заметим, что на практике, почти все сортировки в запросах ClickHouse являются partial_sort, так как `ORDER BY` почти всегда идёт с `LIMIT`. Кстати, Данила Кутенин уже попробовал это и показал, что в тривиальном случае преимущества нет. Но не очевидно, что нельзя сделать лучше. + +### 21.20. Использование материализованных представлений для оптимизации запросов. + +В ByteDance есть готовая реализация, но они её боятся из-за, возможно, низкого качества кода. + +### 21.21. Чтение больших файлов с помощью mmap. + +Тривиально, почти всё готово. + +### 21.22. Userspace page cache. + +Требует 21.8. + +### 21.23. Ускорение работы с вторичными индексами. + +zhang2014. + + +## 22. Долги и недоделанные возможности. + +### 22.1. + Исправление неработающих таймаутов, если используется TLS. + +Нужно для Яндекс.Облака. Сделал Алексей Миловидов. + +### 22.2. Убрать возможность изменить настройки в native протоколе в случае readonly. + +Алексей Миловидов или [Виталий Баранов](https://github.com/vitlibar). + +### 22.3. Защита от абсурдно заданных пользователем кодеков. + +В очереди, скорее всего [Ольга Хвостикова](https://github.com/stavrolia). + +### 22.4. Исправление оставшихся deadlocks в табличных RWLock-ах. + +Александр Казаков. Нужно для Яндекс.Метрики и Datalens. + +### 22.5. Исправление редких срабатываний TSan в stress тестах в CI. + +Александр Казаков. + +### 22.6. Изменение только DEFAULT в ALTER TABLE может поменять тип столбца. + +### 22.7. Row-Level Security не работает в случае наличия в запросе IN подзапросов. + +[Виталий Баранов](https://github.com/vitlibar). Нужно для Метрики. + +### 22.8. Исправить десериализацию параметров для параметризованных запросов. + +Хотел исправить Василий Немков, Altinity, но есть маленькие затруднения, наверное переделает Алексей Миловидов. + +### 22.9. Разобраться с десериализацией массивов со значениями по-умолчанию в Protobuf формате в случае protobuf 3. + +[Виталий Баранов](https://github.com/vitlibar). Возможно, это - фундаментальная проблема и следует её только документировать. + +### 22.10. Исправление дрифта при отслеживании потребления памяти запросами. + +Требует 6.3., но можно улучшить отдельными хаками. Нужно Метрике и БК. + +### 22.11. + Более простая ser/de настроек запросов. + +И пропуск неизвестных настроек. Важно для Метрики для упрощения апгрейда без изменения конфига. +[Виталий Баранов](https://github.com/vitlibar), готово. + +### 22.12. + Исправление низкой производительности чтения из Kafka. + +Для ClickHouse нехарактерно наличие кода, обладающего столь низкой производительностью. Практики разработки не подразумевают, что такой код должен попасть в продакшен без надлежащего тестирования производительности. + +Изначально было назначено на [Ивана Лежанкина](https://github.com/abyss7), но по неизвестной причине было не сделано в течение нескольких месяцев. +Сделал Михаил Филимонов, Altinity. + +### 22.13. + Посмотреть, почему не работают некоторые collations. + +Изначально было назначено на [Ивана Лежанкина](https://github.com/abyss7), но в результате сделал Александр Сапин. + +### 22.14. Посмотреть, почему не работает StorageSet для MergeTree таблиц при некоторых условиях. + +Вроде бы сделал Никита Михайлов - проверить существующие issues на эту тему. + + +### 22.15. Нормализация коммитов в Kafka и идемпотентности операций. + +Altinity. + +### 22.16. Исправление низкой производительности кодека DoubleDelta. + +Василий Немков, Altinity - в процессе. + +Мы считаем важным, что код в ClickHouse содержит разумные оптимизации, основанные на анализе производительности. Но иногда бывают досадные исключения. + +### 22.17. Консистентно работающий POPULATE для MaterializedView. + +### 22.18. Исправление заметного падения производительности форматов после добавления доменов типов. + +Василий Немков, Altinity. + +### 22.19. + Одновременное использование SAMPLE и PREWHERE. + +Нужно для Метрики. [Николай Кочетов](https://github.com/KochetovNicolai), ноябрь 2019. + +### 22.20. + Неправильная работа PREWHERE при некоторых условиях. + +[Николай Кочетов](https://github.com/KochetovNicolai), декабрь 2019. + +### 22.21. Неправильное поведение DateTime в районе начала unix epoch. + +Алексей Миловидов. + +### 22.22. Nullable в функции transform и в CASE по множеству значений. + +После 10.14. + +https://github.com/ClickHouse/ClickHouse/issues/7237 +https://github.com/ClickHouse/ClickHouse/issues/2655 + +### 22.23. Правильная обработка Nullable в функциях, которые кидают исключение на default значении: modulo, intDiv. + +### 22.24. Излишняя фильтрация ODBC connection string. + +Нужно для Метрики. Алексей Миловидов. + +### 22.25. Избавиться от библиотеки btrie. + +Алексей Миловидов. Низкий приоритет. + +### 22.26. Плохая производительность quantileTDigest. + +Алексей Миловидов или будет переназначено. + +### 22.27. Проверить несколько PR, которые были закрыты zhang2014 и sundy-li. + +Алексей Миловидов. + +### 22.28. Изучить и исправить поведение работы с Kafka при ребалансировке. + +Altinity. + +### 22.29. Уязвимость DDL для словарей executable. + +[Александр Сапин](https://github.com/alesapin) + + +## 23. Default Festival. + +### 23.1. Включение minimalistic_part_header в ZooKeeper. + +Сильно уменьшает объём данных в ZooKeeper. Уже год в продакшене в Яндекс.Метрике. +Алексей Миловидов, ноябрь 2019. + +### 23.2. Включение distributed_aggregation_memory_efficient. + +Есть риски меньшей производительности лёгких запросов, хотя производительность тяжёлых запросов всегда увеличивается. + +### 23.3. Включение min_bytes_to_external_sort и min_bytes_to_external_group_by. + +Желательно 5.2. и 13.1. + +### 23.4. Включение синхронной записи в Distributed таблицы по-умолчанию. + +Есть гипотеза, что плохо работает на очень больших кластерах. + +### 23.5. Включение compile_expressions. + +Требует 7.2. Задачу изначально на 99% сделал Денис Скоробогатов, ВШЭ и Яндекс. Остальной процент доделывал Алексей Миловидов, а затем [Александр Сапин](https://github.com/alesapin). + +### 23.6. Включение учёта порядка столбцов в CSV. + +Просто аккуратно включить. + +### 23.7. Включение NULL as Default в CSV. + +Просто аккуратно включить. + +### 23.8. Включение оптимизации VALUES. + +Просто аккуратно включить. + +### 23.9. Включение Processors. + +[Николай Кочетов](https://github.com/KochetovNicolai). + +### 23.10. Включение mlock бинарника. + +Возможность mlock бинарника сделал Олег Алексеенков. Поможет, когда на серверах кроме ClickHouse работает много посторонних программ (мы иногда называем их в шутку "треш-программами"). + + +## 24. Экспериментальные задачи. + +### 24.1. Веб-интерфейс для просмотра состояния кластера и профилирования запросов. + +Антон Мамонов, УрФУ, Яндекс. + +Внутри ClickHouse есть богатые возможности по интроспекции и профилированию. Эти возможности доступны через системные таблицы и использовать их приходится путём формулирования SQL запросов. Это неудобно. + +Вместо этого предлагается сделать, чтобы ClickHouse отдавал HTML страницу, реализующую интерактивный web-интерфейс со следующими возможностями: +- отображение состояния кластеров (какие кластеры известны, статус каждого сервера); +- графики нагрузки текущего сервера или выбранного сервера кластера; +- обновляемый список запросов; +- просмотр лога запросов с наиболее востребованными фильтрациями по одной кнопке; +- просмотр лога на кластере, например - последние ошибки; +- просмотр метрик использования ресурсов, flame graph и pprof-граф для выбранных запросов; +- отчёт по использованию кластера (пример: количество ядер CPU по пользователям за сегодня). + +### 24.2. Экспериментальные алгоритмы сжатия. + +Анастасия Наумова, ВШЭ. + +ClickHouse поддерживает LZ4 и ZSTD для сжатия данных. Эти алгоритмы являются парето-оптимальными по соотношению скорости и коэффициентам сжатия среди достаточно известных. Тем не менее, существуют менее известные алгоритмы сжатия, которые могут превзойти их по какому-либо критерию. Из потенциально более быстрых по сравнимом коэффициенте сжатия: Lizard, LZSSE, density. Из более сильных: bsc и csc. Необходимо изучить эти алгоритмы, добавить их поддержку в ClickHouse и исследовать их работу на тестовых датасетах. + +### 24.3. Экспериментальные кодеки. + +Вероника Фалчикова, Лада Торчик, ВШЭ. + +Существуют специализированные алгоритмы кодирования числовых последовательностей: Group VarInt, MaskedVByte, PFOR. Необходимо изучить наиболее эффективные реализации этих алгоритмов. Примеры вы сможете найти на https://github.com/lemire и https://github.com/powturbo/ а также https://github.com/schizofreny/middle-out + +Внедрить их в ClickHouse в виде кодеков и изучить их работу на тестовых датасетах. + +### 24.4. Шифрование в ClickHouse на уровне кусков данных. + +Yuchen Dong, ICS. + +Данные в ClickHouse хранятся без шифрования. При наличии доступа к дискам, злоумышленник может прочитать данные. Предлагается реализовать два подхода к шифрованию: + +1. Шифрование блоков данных. +Шифрование данных столбцов на диске требуется реализовать в виде кодеков. Это позволит применять шифрование к отдельным столбцам; применять его после сжатия данных (эффективно, но менее безопасно) или без сжатия. Потребуется проработать работу с ключами: получение ключей из отдельного сервиса, правильная работа с ключами в оперативке. Отдельным вопросом стоит шифрование индексов. + +### 24.5. Поддержка функций шифрования для отдельных значений. + +Yuchen Dong, ICS. + +Смотрите также 24.5. + +2. Шифрование отдельных значений. +Для этого требуется реализовать функции шифрования и расшифрования, доступные из SQL. Для шифрования реализовать возможность добавления нужного количества случайных бит для исключения одинаковых зашифрованных значений на одинаковых данных. Это позволит реализовать возможность "забывания" данных без удаления строк таблицы: можно шифровать данные разных клиентов разными ключами, и для того, чтобы забыть данные одного клиента, потребуется всего лишь удалить ключ. + +### 24.6. Userspace RAID. + +Глеб Новиков, ВШЭ. + +RAID позволяет одновременно увеличить надёжность хранения данных на дисках и увеличить скорость работы дискового массива. Обычно RAID настраивается с помощью встроенных возможностей ядра Linux (mdraid) или с помощью hardware контроллера. У этого есть следующие ограничения: + +1. Иногда (в облачной инфраструктуре некоторых компаний) сервер предоставляется с отдельными дисками, подмонтированными в виде отдельных разделов (JBOD), без возможности создания RAID. + +2. В ClickHouse для обеспечения избыточности обычно используется репликация между серверами. Но при восстановлении одного из дисков RAID не используются данные с реплик, а в случае отказа одного из дисков в RAID-0, приходится передавать с реплики все данные, а не только данные, соответствующие одному из дисков. Это происходит, потому что RAID не интегрирован в ClickHouse и "не знает" про его особенности. + +3. Отсутствуют продвинутые варианты обеспечения избыточности, как например, LRC. + +Для преодоления этих ограничений, предлагается реализовать в ClickHouse встроенный алгоритм расположения данных на дисках. + +### 24.7. Вероятностные структуры данных для фильтрации по подзапросам. + +Рузель Ибрагимов, ВШЭ и Яндекс. + +Частой задачей является выполнение запроса с фильтрацией по множеству, полученному по подзапросу. Пример: найти пользователей, которые заходили на сайт сегодня и заходили неделю назад. Это выражается в виде запроса: `SELECT UserID FROM table WHERE EventDate = today() AND UserID IN (SELECT ...)`. При выполнении этого запроса, сначала выполняется подзапрос в правой части `IN` и формируется хэш-таблица в оперативке; затем эта хэш-таблица используется для фильтрации. + +Иногда объём данных достаточно большой, и хэш-таблица не помещается в оперативку. В этом случае можно рассмотреть в качестве варианта приближённый рассчёт: найти пользователей, которые заходили на сайт сегодня и наверное заходили неделю назад. Для этого можно вместо хэш-таблицы использовать Bloom Filter. Другая задача: найти пользователей, которые встречались, скорее всего, не менее некоторого количества раз. Для этого можно использовать Counting Bloom Filter. Также следует изучить структуры данных Quotient Filter и Cuckoo Filer, а ещё - секретный алгоритм Chaotic Map от Андрея Плахова. + +Предлагается реализовать это в языке запросов ClickHouse с помощью специального синтаксиса, например `x IN BLOOM FILTER (n, m) (SELECT ...)`. + +### 24.8. Специализация векторизованного кода для AVX/AVX2/AVX512 и ARM NEON. + +Дмитрий Ковальков, ВШЭ и Яндекс. + +Подавляющее большинство кода ClickHouse написана для x86_64 с набором инструкций до SSE 4.2 включительно. Лишь отдельные редкие функции поддерживают AVX/AVX2/AVX512 с динамической диспетчеризацией. + +В первой части задачи, следует добавить в ClickHouse реализации некоторых примитивов, оптимизированные под более новый набор инструкций. Например, AVX2 реализацию генератора случайных чисел pcg: https://github.com/lemire/simdpcg + +Во второй части задачи, предлагается адаптировать существующие куски кода, использующие SSE intrinsics на AVX/AVX2 и сравнить производительность. Также рассматривается оптимизация под ARM NEON. + +### 24.9. Общий подход к CPU dispatching в фабрике функций. + +Дмитрий Ковальков, ВШЭ и Яндекс. + +Продолжение 24.8. + +### 24.10. Поддержка типов half/bfloat16/unum. + +Рустам Гусейн-заде, ВШЭ. + +### 24.11. User Defined Functions. + +Игорь Минеев, ВШЭ. + +ClickHouse предоставляет достаточно богатый набор встроенных функций языка запросов, но не позволяет пользователю добавлять свои функции без редактировния исходников и перекомпиляции системы. Это мотивировано следующими потенциальными проблемами: + +1. ClickHouse является array-oriented системой, и все функции внутри кода принимают для обработки целые массивы, а не отдельные значения. Это усложняет внутренний интерфейс и делает его менее удобным для пользователя. +2. Предоставление возможности подключения UDF в виде shared библиотек, потребовало бы фиксировать этот интерфейс или поддерживать обратную совместимость, тогда как мы бы хотели, при разработке ClickHouse, менять этот интерфейс по своему усмотрению без оглядки. +3. Сложность внутренних структур данных повышает вероятность ошибок типа buffer overflow и повреждения памяти, что сильно затруднит сопровождение ClickHouse с пользовательскими функциями. + +Тем не менее, можно выбрать более аккуратный подход, избегающий непосредственной линковки с shared библиотеками. + +Сначала можно реализовать поддержку UDF в виде выражений, составленных из простых функций ClickHouse. В ClickHouse есть встроенная кодогенерация на LLVM, что позволит таким функциям работать весьма эффективно. Но этот подход весьма ограничен и поэтому не является исчерпывающим. + +Затем предлагается реализовать поддержку UDF в виде исходников на C++, которые компилируются в runtime, с использованием заголовочных файлов ClickHouse. Требование компиляции из исходников вместо shared библиотек, позволит ослабить необходимость в поддержке совместимости ABI. + +Для безопасности, потребуется исследовать возможность размещения буферов данных в shared memory для выполнения UDF в отдельных процессах с изоляцией по памяти. Возможно, для этого пригодится интеграция с Apache Arrow. + +Также рассматривается возможность написания UDF на Rust, а также использование Web Assembly. Отдельно можно рассмотреть подключение NumPy и R и других технологий, которые предоставляют операции над целыми массивами. + +### 24.12. GPU offloading. + +Риск состоит в том, что даже известные GPU базы, такие как OmniSci, работают медленнее, чем ClickHouse. +Преимущество возможно только на полной сортировке и JOIN. +Алексей Соловей, nVidia и Рита Коннова, ВШЭ. + +В компании nVidia сделали прототип offloading вычисления GROUP BY с некоторыми из агрегатных функций в ClickHouse и обещат предоставить исходники в публичный доступ для дальнейшего развития. Предлагается изучить этот прототип и расширить его применимость для более широкого сценария использования. В качестве альтернативы, предлагается изучить исходные коды системы `OmniSci` или `Alenka` или библиотеку `CUB` https://nvlabs.github.io/cub/ и применить некоторые из алгоритмов в ClickHouse. + +### 24.13. Stream запросы. + +Пререквизит для ClickHouse как CEP-системы. + +### 24.14. Window функции. + +Требует 2.1. + +### 24.15. Поддержка полуструктурированных данных. + +Требует 1.14 и 2.10. + +### 24.16. Улучшение эвристики слияний. + +В прошлом году исследование по этой задаче сделал Егор Соловьёв, ВШЭ и Яндекс.Такси. Его исследование показало, что алгоритм нельзя существенно улучшить путём изменения параметров. Но исследование лажовое, так как рассмотрен только уже использующийся алгоритм. То есть, задача остаётся открытой. + +### 24.17. Экспериментальные способы ускорения параллельного GROUP BY. + +Максим Серебряков + +### 24.18. Не TCP протокол передачи файлов при репликации. + +### 24.19. Промежуточное состояние GROUP BY как структура данных для key-value доступа. + +### 24.20. Short-circuit вычисления некоторых выражений. + +Два года назад задачу попробовала сделать Анастасия Царькова, ВШЭ и Яндекс, но реализация получилась слишком неудобной и её удалили. + +### 24.21. Реализация в ClickHouse протокола распределённого консенсуса. + +Имеет смысл только после 19.2. + +### 24.22. Вывод типов по блоку данных. Вывод формата данных по примеру. + +Эльмир Марданов, ВШЭ. + +ClickHouse является строго типизированной системой. Для того, чтобы прочитать данные в каком либо формате (например, CSV), требуется заранее указать типы данных. Если при чтении формата выясняется, что данные не могут быть прочитаны в рамках заданных типов, то кидается исключение. + +ClickHouse также может использоваться для быстрой аналитики по локальным файлам, без загрузки их в базу данных (программа `clickhouse-local`). В этом случае, его использование может заменить `awk`, `sed`, `grep`. Но остаётся неудобство - необходимость указания типов данных. + +Предлагается реализовать функциональность вывода типов по первому блоку данных путём применения эвристик и постепенного расширения типов. + +Другая экспериментальная задача - реализация эвристик для обработки данных в неизвестном построчном текстовом формате. Детектирование CSV, TSV, JSON, детектирование разделителей и форматов значений. + +### 24.23. Минимальная поддержка транзакций для множества вставок/чтений. + +Максим Кузнецов, ВШЭ. + +Таблицы типа MergeTree состоят из набора независимых неизменяемых "кусков" данных. При вставках данных (INSERT), формируются новые куски. При модификациях данных (слияние кусков), формируются новые куски, а старые - становятся неактивными и перестают использоваться следующими запросами. Чтение данных (SELECT) производится из снэпшота множества кусков на некоторый момент времени. Таким образом, чтения и вставки не блокируют друг друга. + +Если же выполняется несколько запросов SELECT, то чтение данных может осуществляться из снэпшотов по состоянию на несколько разных моментов времени и быть неконсистентным. Пример: пользователю отображается отчёт из нескольких графиков и таблиц, но из-за того, что между разными запросами, данные успели обновиться, отображаемые данные не соответствуют друг другу. + +Пример с другой стороны - пользователь хочет осуществить несколько вставок (INSERT) в одну или несколько таблиц, но так, чтобы данные появились в них атомарно с точки зрения других запросов (SELECT). + +Для решения этих проблем, предлагается ввести глобальные метки времени для кусков данных (сейчас уже есть инкрементальные номера кусков, но они выделяются в рамках одной таблицы). Первым шагом сделаем эти метки времени в рамках сервера. Вторым шагом сделаем метки времени в рамках всех серверов, но неточные на основе локальных часов. Третьим шагом сделаем метки времени, выдаваемые сервисом координации. + +### 24.24. Реализация алгоритмов differential privacy. + +Артём Вишняков, ВШЭ. + +https://github.com/yandex/ClickHouse/issues/6874 + +### 24.25. Интеграция в ClickHouse функциональности обработки HTTP User Agent. + +Есть хороший код в Яндекс.Метрике. Получено согласие от руководства. +Михаил Филитов, ВШЭ. + +### 24.26. Поддержка open tracing или аналогов. + +Александр Кожихов, ВШЭ и Яндекс.YT. + +### 24.27. Реализация алгоритмов min-hash, sim-hash для нечёткого поиска полудубликатов. + +ucasFL, ICS. + +Алгоритмы min-hash и sim-hash позволяют вычислить для текста несколько хэш-значений таких, что при небольшом изменении текста, по крайней мере один из хэшей не меняется. Вычисления можно реализовать на n-грамах и словарных шинглах. Предлагается добавить поддержку этих алгоритмов в виде функций в ClickHouse и изучить их применимость для задачи нечёткого поиска полудубликатов. + +### 24.28. Другой sketch для квантилей. + +Похоже на quantileTiming, но с логарифмическими корзинами. + +### 24.29. Поддержка Arrow Flight. + +### 24.30. ClickHouse как графовая СУБД. + +Amos Bird, но его решение слишком громоздкое и пока не open-source. + +### 24.31. Кореллированные подзапросы. + +Перепиывание в JOIN. Не раньше 21.11, 21.12, 21.9. Низкий приоритет. + +### 24.32. Поддержка GRPC. + +Мария Конькова, ВШЭ и Яндекс. +Также смотрите 24.29. + +В ClickHouse есть два основных протокола: родной протокол общения между серверами и HTTP/1.1 протокол. HTTP/1.1 протокол удобен для работы из самых разных языков программирования, но, в отличие от родного протокола, не поддерживает двусторонний обмен информацией во время запроса: +- передачу информации о прогрессе во время выполнения запроса; +- передачу логов во время выполнения запроса; +- отмену выполнения запроса в тот момент как данные ещё не начали передаваться; + +Рассматривается вариант - поддержка GRPC в ClickHouse. Здесь есть неочевидные моменты, такие как - эффективная передача массивов данных в column-oriented формате - насколько удобно будет обернуть это в GRPC. + + +## 25. DevRel + +### 25.1. + Перевод инструкции для начинающих разработчиков. + +Александр Казаков, ноябрь 2019. + +### 25.2. Вычитка и выкладка статьи про обфускацию данных на английском. + +Эми, Александр Казаков, Алексей Миловидов, ноябрь 2019. +Готово к выкладке. + +### 25.3. Подготовка статьи "Секреты оптимизации производительности ClickHouse". + +Алексей Миловидов, Леонид. + +### 25.4. Подготовка статьи "Профайлер запросов: трудный путь". + +Алексей Миловидов, Леонид. + +### 25.5. Подготовка статьи "Тестирование ClickHouse, которое мы заслужили". + +### 25.6. Перевод этих статей на английский. + +Требует 25.3, 25.4, 25.5. Эми + +### 25.7. Перевод статьи Данилы Кутенина на английский. + +Эми + +### 25.8. + Выступление keynote на BDTC. + +Алексей Миловидов + +### 25.9. Подготовка докладчиков: khvostikao, ilezhankin, nikitamikhailov, akuzm и другие. + +[Ольга Хвостикова](https://github.com/stavrolia), [Иван Лежанкин](https://github.com/abyss7), Никита Михайлов, [Александр Кузьменков](https://github.com/akuzm). +Уже готовые докладчики: Алексей Миловидов, [Николай Кочетов](https://github.com/KochetovNicolai), [Александр Сапин](https://github.com/alesapin). +Получаем минимум 7 докладчиков в 2020 году. + +### 25.10. Митапы в России и Беларуси: Москва x2 + митап для разработчиков или хакатон, Санкт-Петербург, Минск, Нижний Новгород, Екатеринбург, Новосибирск и/или Академгородок, Иннополис или Казань. + +Екатерина - организация + +### 25.11. Митапы зарубежные: восток США (Нью Йорк, возможно Raleigh), возможно северо-запад (Сиэтл), Китай (Пекин снова, возможно митап для разработчиков или хакатон), Лондон. + +[Иван Блинков](https://github.com/blinkov/) - организация + +### 25.12. Статья "научная" - про устройство хранения данных и индексов или whitepaper по архитектуре. Есть вариант подать на VLDB. + +Низкий приоритет. Алексей Миловидов. + +### 25.13. Участие во всех мероприятиях Яндекса, которые связаны с разработкой бэкенда, C++ разработкой или с базами данных, возможно участие в DevRel мероприятиях. + +Алексей Миловидов и все подготовленные докладчики + +### 25.14. Конференции в России: все HighLoad, возможно CodeFest, DUMP или UWDC, возможно C++ Russia. + +Алексей Миловидов и все подготовленные докладчики + +### 25.15. Конференции зарубежные: Percona, DataOps, попытка попасть на более крупные. + +Алексей Миловидов и все подготовленные докладчики + +### 25.16. Сайт play.clickhouse. + +Цель состоит в реализации сайта, на котором можно попробовать задавать произвольные запросы к временному экземпляру ClickHouse и изучать его поведение. Из похожих проектов можно отметить: [Compiler Explorer](https://godbolt.org/), http://ideone.com/, [SQLFiddle](http://sqlfiddle.com/), [DB-Fiddle](https://www.db-fiddle.com/). + +С помощью такого сайта можно решать следующие задачи: +- ознакомление с языком запросов ClickHouse; +- демонстрация примеров из документации; +- демонстрация скорости работы на тестовых датасетах; +- сравнение поведения разных версий ClickHouse друг с другом; +- демонстрация неожиданного поведения или багов; + +Требуется проработать вопрос безопасности и изоляции инстансов (поднятие в контейнерах с ограничениями по сети), подключение тестовых датасетов с помощью copy-on-write файловой системы; органичения ресурсов. + +### 25.17. Взаимодействие с ВУЗами: ВШЭ, УрФУ, ICS Beijing. + +Алексей Миловидов и вся группа разработки + +### 25.18. Лекция в ШАД. + +Алексей Миловидов + +### 25.19. Участие в курсе разработки на C++ в ШАД. + +### 25.20. Ещё одно сравнение производительности аналитических СУБД. + +Матвей Бубнов, УрФУ + +Существуют мало известные специализированные СУБД, способные конкурировать с ClickHouse по скорости обработки некоторых классов запросов. Пример: `TDEngine` и `DolphinDB`, `VictoriaMetrics`, а также `Apache Doris` и `LocustDB`. Предлагается изучить и классифицировать архитектурные особенности этих систем - их особенности и преимущества. Установить эти системы, загрузить тестовые данные, изучить производительность. Проанализировать, за счёт чего достигаются преимущества. + +### 25.21. Повторное награждение контрибьюторов в Китае. + +### 25.22. On-site помощь с ClickHouse компаниям в дни рядом с мероприятиями. + +[Иван Блинков](https://github.com/blinkov/) - организация + +### 25.23. Новый мерч для ClickHouse. + +### 25.24. Конкурсы bughunter или оптимизации кода на C++. + +Проведение конкурсов должно начинаться для сотрудников Яндекса, пока нет согласования. + +### 25.25. Семинары для потенциальных клиентов Яндекс.Облака. + +По мере необходимости. Алексей Миловидов, организация - Яндекс.Облако. + +### 25.26. Участие в GSoC. + +Андрей Бородин пытается уговорить нас участвовать, но пока загружены задачей 25.17. + +### 25.27. Обновить сайт ClickHouse. + +Иван Блинков. Есть риски. diff --git a/docs/ru/getting_started/example_datasets/metrica.md b/docs/ru/getting_started/example_datasets/metrica.md index 3aaa4db952a..4bb3dc9e4c6 100644 --- a/docs/ru/getting_started/example_datasets/metrica.md +++ b/docs/ru/getting_started/example_datasets/metrica.md @@ -49,4 +49,4 @@ $ clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" ``` ## Запросы -Примеры запросов к этим таблицам (они называются `test.hits` и `test.visits`) можно найти среди [stateful тестов](https://github.com/ClickHouse/ClickHouse/tree/master/dbms/tests/queries/1_stateful) и в некоторых [performance тестах](https://github.com/ClickHouse/ClickHouse/tree/master/dbms/tests/performance/test_hits) ClickHouse. +Примеры запросов к этим таблицам (они называются `test.hits` и `test.visits`) можно найти среди [stateful тестов](https://github.com/ClickHouse/ClickHouse/tree/master/dbms/tests/queries/1_stateful) и в некоторых [performance тестах](https://github.com/ClickHouse/ClickHouse/tree/master/dbms/tests/performance) ClickHouse. diff --git a/docs/ru/getting_started/index.md b/docs/ru/getting_started/index.md index c03ac58f24b..a8d0fbaa5b1 100644 --- a/docs/ru/getting_started/index.md +++ b/docs/ru/getting_started/index.md @@ -1,138 +1,10 @@ # Начало работы -## Системные требования +Если вы новичок в ClickHouse и хотите получить вживую оценить его производительность, прежде всего нужно пройти через [процесс установки](install.md). -ClickHouse может работать на любом Linux, FreeBSD или Mac OS X с архитектурой процессора x86\_64. +После этого можно выбрать один из следующих вариантов: -Хотя предсобранные релизы обычно компилируются с использованием набора инструкций SSE 4.2, что добавляет использование поддерживающего его процессора в список системных требований. Команда для проверки наличия поддержки инструкций SSE 4.2 на текущем процессоре: - -```bash -$ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" -``` - -## Установка - -### Из DEB пакетов - -Яндекс рекомендует использовать официальные скомпилированные `deb` пакеты для Debian или Ubuntu. - -Чтобы установить официальные пакеты, пропишите репозиторий Яндекса в `/etc/apt/sources.list` или в отдельный файл `/etc/apt/sources.list.d/clickhouse.list`: - -```bash -$ deb http://repo.yandex.ru/clickhouse/deb/stable/ main/ -``` - -Если вы хотите использовать наиболее свежую тестовую, замените `stable` на `testing` (не рекомендуется для production окружений). - -Затем для самой установки пакетов выполните: - -```bash -$ sudo apt-get install dirmngr # optional -$ sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 # optional -$ sudo apt-get update -$ sudo apt-get install clickhouse-client clickhouse-server -``` - -Также эти пакеты можно скачать и установить вручную отсюда: . - -### Из RPM пакетов - -Команда ClickHouse в Яндексе рекомендует использовать официальные предкомпилированные `rpm` пакеты для CentOS, RedHad и всех остальных дистрибутивов Linux, основанных на rpm. - -Сначала нужно подключить официальный репозиторий: -```bash -$ sudo yum install yum-utils -$ sudo rpm --import https://repo.yandex.ru/clickhouse/CLICKHOUSE-KEY.GPG -$ sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/x86_64 -``` - -Для использования наиболее свежих версий нужно заменить `stable` на `testing` (рекомендуется для тестовых окружений). - -Then run these commands to actually install packages: -Для, собственно, установки пакетов необходимо выполнить следующие команды: - -```bash -$ sudo yum install clickhouse-server clickhouse-client -``` - -Также есть возможность установить пакеты вручную, скачав отсюда: . - -### Из Docker образа - -Для запуска ClickHouse в Docker нужно следовать инструкции на [Docker Hub](https://hub.docker.com/r/yandex/clickhouse-server/). Внутри образов используются официальные `deb` пакеты. - -### Из исходного кода - -Для компиляции ClickHouse вручную, используйте инструкцию для [Linux](../development/build.md) или [Mac OS X](../development/build_osx.md). - -Можно скомпилировать пакеты и установить их, либо использовать программы без установки пакетов. Также при ручной сборке можно отключить необходимость поддержки набора инструкций SSE 4.2 или собрать под процессоры архитектуры AArch64. - -```text -Client: dbms/programs/clickhouse-client -Server: dbms/programs/clickhouse-server -``` - -Для работы собранного вручную сервера необходимо создать директории для данных и метаданных, а также сделать их `chown` для желаемого пользователя. Пути к этим директориям могут быть изменены в конфигурационном файле сервера (src/dbms/programs/server/config.xml), по умолчанию используются следующие: - -```text -/opt/clickhouse/data/default/ -/opt/clickhouse/metadata/default/ -``` - -На Gentoo для установки ClickHouse из исходного кода можно использовать просто `emerge clickhouse`. - -## Запуск - -Для запуска сервера в качестве демона, выполните: - -```bash -$ sudo service clickhouse-server start -``` - -Смотрите логи в директории `/var/log/clickhouse-server/`. - -Если сервер не стартует, проверьте корректность конфигурации в файле `/etc/clickhouse-server/config.xml` - -Также можно запустить сервер вручную из консоли: - -```bash -$ clickhouse-server --config-file=/etc/clickhouse-server/config.xml -``` - -При этом, лог будет выводиться в консоль, что удобно для разработки. -Если конфигурационный файл лежит в текущей директории, то указывать параметр `--config-file` не требуется, по умолчанию будет использован файл `./config.xml`. - -После запуска сервера, соединиться с ним можно с помощью клиента командной строки: - -```bash -$ clickhouse-client -``` - -По умолчанию он соединяется с localhost:9000, от имени пользователя `default` без пароля. Также клиент может быть использован для соединения с удалённым сервером с помощью аргумента `--host`. - -Терминал должен использовать кодировку UTF-8. - -Более подробная информация о клиенте располагается в разделе [«Клиент командной строки»](../interfaces/cli.md). - -Пример проверки работоспособности системы: - -```bash -$ ./clickhouse-client -ClickHouse client version 0.0.18749. -Connecting to localhost:9000. -Connected to ClickHouse server version 0.0.18749. -``` -```sql -SELECT 1 -``` -```text -┌─1─┐ -│ 1 │ -└───┘ -``` - -**Поздравляем, система работает!** - -Для дальнейших экспериментов можно попробовать загрузить один из тестовых наборов данных или пройти [пошаговое руководство для начинающих](https://clickhouse.yandex/tutorial.html). +* [Пройти подробное руководство для начинающих](tutorial.md) +* [Поэкспериментировать с тестовыми наборами данных](example_datasets/ontime.md) [Оригинальная статья](https://clickhouse.yandex/docs/ru/getting_started/) diff --git a/docs/ru/getting_started/install.md b/docs/ru/getting_started/install.md new file mode 100644 index 00000000000..29ccd2b14f4 --- /dev/null +++ b/docs/ru/getting_started/install.md @@ -0,0 +1,144 @@ +# Установка + +## Системные требования + +ClickHouse может работать на любой операционной системе Linux, FreeBSD или Mac OS X с архитектурой процессора x86\_64, AArch64 или PowerPC64LE. + +Предварительно собранные пакеты компилируются для x86\_64 и используют набор инструкций SSE 4.2, поэтому, если не указано иное, его поддержка в используемом процессоре, становится дополнительным требованием к системе. Вот команда, чтобы проверить, поддерживает ли текущий процессор SSE 4.2: + +``` bash +$ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" +``` + +Чтобы запустить ClickHouse на процессорах, которые не поддерживают SSE 4.2, либо имеют архитектуру AArch64 или PowerPC64LE, необходимо самостоятельно [собрать ClickHouse из исходного кода](#from-sources) с соответствующими настройками конфигурации. + +## Доступные варианты установки + +### Из DEB пакетов {#from-deb-packages} + +Яндекс рекомендует использовать официальные скомпилированные `deb` пакеты для Debian или Ubuntu. + +Чтобы установить официальные пакеты, пропишите репозиторий Яндекса в `/etc/apt/sources.list` или в отдельный файл `/etc/apt/sources.list.d/clickhouse.list`: + +``` +deb http://repo.yandex.ru/clickhouse/deb/stable/ main/ +``` + +Если вы хотите использовать наиболее свежую тестовую, замените `stable` на `testing` (не рекомендуется для production окружений). + +Затем для самой установки пакетов выполните: + +```bash +sudo apt-get install dirmngr # optional +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 # optional +sudo apt-get update +sudo apt-get install clickhouse-client clickhouse-server +``` + +Также эти пакеты можно скачать и установить вручную отсюда: . + +### Из RPM пакетов {#from-rpm-packages} + +Команда ClickHouse в Яндексе рекомендует использовать официальные предкомпилированные `rpm` пакеты для CentOS, RedHad и всех остальных дистрибутивов Linux, основанных на rpm. + +Сначала нужно подключить официальный репозиторий: +```bash +sudo yum install yum-utils +sudo rpm --import https://repo.yandex.ru/clickhouse/CLICKHOUSE-KEY.GPG +sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/x86_64 +``` + +Для использования наиболее свежих версий нужно заменить `stable` на `testing` (рекомендуется для тестовых окружений). + +Then run these commands to actually install packages: +Для, собственно, установки пакетов необходимо выполнить следующие команды: + +```bash +sudo yum install clickhouse-server clickhouse-client +``` + +Также есть возможность установить пакеты вручную, скачав отсюда: . + +### Из Docker образа {#from-docker-image} + +Для запуска ClickHouse в Docker нужно следовать инструкции на [Docker Hub](https://hub.docker.com/r/yandex/clickhouse-server/). Внутри образов используются официальные `deb` пакеты. + +### Из исходного кода {#from-sources} + +Для компиляции ClickHouse вручную, используйте инструкцию для [Linux](../development/build.md) или [Mac OS X](../development/build_osx.md). + +Можно скомпилировать пакеты и установить их, либо использовать программы без установки пакетов. Также при ручой сборке можно отключить необходимость поддержки набора инструкций SSE 4.2 или собрать под процессоры архитектуры AArch64. + +``` +Client: dbms/programs/clickhouse-client +Server: dbms/programs/clickhouse-server +``` + +Для работы собранного вручную сервера необходимо создать директории для данных и метаданных, а также сделать их `chown` для желаемого пользователя. Пути к этим директориям могут быть изменены в конфигурационном файле сервера (src/dbms/programs/server/config.xml), по умолчанию используются следующие: + +``` +/opt/clickhouse/data/default/ +/opt/clickhouse/metadata/default/ +``` + +На Gentoo для установки ClickHouse из исходного кода можно использовать просто `emerge clickhouse`. + +## Запуск + +Для запуска сервера в качестве демона, выполните: + +``` bash +$ sudo service clickhouse-server start +``` + +Смотрите логи в директории `/var/log/clickhouse-server/`. + +Если сервер не стартует, проверьте корректность конфигурации в файле `/etc/clickhouse-server/config.xml` + +Также можно запустить сервер вручную из консоли: + +``` bash +$ clickhouse-server --config-file=/etc/clickhouse-server/config.xml +``` + +При этом, лог будет выводиться в консоль, что удобно для разработки. +Если конфигурационный файл лежит в текущей директории, то указывать параметр `--config-file` не требуется, по умолчанию будет использован файл `./config.xml`. + +После запуска сервера, соединиться с ним можно с помощью клиента командной строки: + +``` bash +$ clickhouse-client +``` + +По умолчанию он соединяется с localhost:9000, от имени пользователя `default` без пароля. Также клиент может быть использован для соединения с удалённым сервером с помощью аргумента `--host`. + +Терминал должен использовать кодировку UTF-8. + +Более подробная информация о клиенте располагается в разделе [«Клиент командной строки»](../interfaces/cli.md). + +Пример проверки работоспособности системы: + +``` bash +$ ./clickhouse-client +ClickHouse client version 0.0.18749. +Connecting to localhost:9000. +Connected to ClickHouse server version 0.0.18749. + +:) SELECT 1 + +SELECT 1 + +┌─1─┐ +│ 1 │ +└───┘ + +1 rows in set. Elapsed: 0.003 sec. + +:) +``` + +**Поздравляем, система работает!** + +Для дальнейших экспериментов можно попробовать загрузить один из тестовых наборов данных или пройти [пошаговое руководство для начинающих](https://clickhouse.yandex/tutorial.html). + +[Оригинальная статья](https://clickhouse.yandex/docs/ru/getting_started/install/) diff --git a/docs/ru/getting_started/tutorial.md b/docs/ru/getting_started/tutorial.md new file mode 120000 index 00000000000..8bc40816ab2 --- /dev/null +++ b/docs/ru/getting_started/tutorial.md @@ -0,0 +1 @@ +../../en/getting_started/tutorial.md \ No newline at end of file diff --git a/docs/ru/interfaces/cli.md b/docs/ru/interfaces/cli.md index f2040c4af1b..a67ae87f6ab 100644 --- a/docs/ru/interfaces/cli.md +++ b/docs/ru/interfaces/cli.md @@ -1,17 +1,23 @@ # Клиент командной строки -Для работы из командной строки вы можете использовать `clickhouse-client`: +ClickHouse предоставляет собственный клиент командной строки: `clickhouse-client`. Клиент поддерживает запуск с аргументами командной строки и с конфигурационными файлами. Подробнее читайте в разделе [Конфигурирование](#interfaces_cli_configuration). + +Клиент [устанавливается](../getting_started/index.md) пакетом `clickhouse-client` и запускается командой `clickhouse-client`. ```bash $ clickhouse-client -ClickHouse client version 0.0.26176. -Connecting to localhost:9000. -Connected to ClickHouse server version 0.0.26176. +ClickHouse client version 19.17.1.1579 (official build). +Connecting to localhost:9000 as user default. +Connected to ClickHouse server version 19.17.1 revision 54428. :) ``` -Клиент поддерживает параметры командной строки и конфигурационные файлы. Подробнее читайте в разделе "[Конфигурирование](#interfaces_cli_configuration)". +Клиенты и серверы различных версий совместимы, однако если клиент старее сервера, то некоторые новые фукнции могут быть недоступны. Мы рекомендуем использовать одинаковые версии клиента и сервера. При подключении клиента к более новому серверу `clickhouse-client` выводит сообщение: + +``` +ClickHouse client version is older than ClickHouse server. It may lack support for new features. +``` ## Использование {#cli_usage} diff --git a/docs/ru/interfaces/cpp.md b/docs/ru/interfaces/cpp.md new file mode 100644 index 00000000000..00cbbcb411a --- /dev/null +++ b/docs/ru/interfaces/cpp.md @@ -0,0 +1,5 @@ +# C++ клиентская библиотека + +См. README в репозитории [clickhouse-cpp](https://github.com/ClickHouse/clickhouse-cpp). + +[Оригинальная статья](https://clickhouse.yandex/docs/ru/interfaces/cpp/) diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md index c7c32a46a4c..4da101796f1 100644 --- a/docs/ru/interfaces/http.md +++ b/docs/ru/interfaces/http.md @@ -28,8 +28,12 @@ $ wget -O- -q 'http://localhost:8123/?query=SELECT 1' $ echo -ne 'GET /?query=SELECT%201 HTTP/1.0\r\n\r\n' | nc localhost 8123 HTTP/1.0 200 OK +Date: Wed, 27 Nov 2019 10:30:18 GMT Connection: Close -Date: Fri, 16 Nov 2012 19:21:50 GMT +Content-Type: text/tab-separated-values; charset=UTF-8 +X-ClickHouse-Server-Display-Name: clickhouse.ru-central1.internal +X-ClickHouse-Query-Id: 5abe861c-239c-467f-b955-8a201abb8b7f +X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} 1 ``` diff --git a/docs/ru/interfaces/index.md b/docs/ru/interfaces/index.md index dd09427d5ba..4679641872e 100644 --- a/docs/ru/interfaces/index.md +++ b/docs/ru/interfaces/index.md @@ -9,7 +9,8 @@ ClickHouse предоставляет два сетевых интерфейса * [Консольный клиент](cli.md); * [JDBC-драйвер](jdbc.md); -* [ODBC-драйвер](odbc.md). +* [ODBC-драйвер](odbc.md); +* [C++ клиентская библиотека](cpp.md). Существует также широкий спектр сторонних библиотек для работы с ClickHouse: diff --git a/docs/ru/interfaces/third-party/client_libraries.md b/docs/ru/interfaces/third-party/client_libraries.md index 562b863922e..13b7b9d243e 100644 --- a/docs/ru/interfaces/third-party/client_libraries.md +++ b/docs/ru/interfaces/third-party/client_libraries.md @@ -26,7 +26,7 @@ - [HTTP-ClickHouse](https://metacpan.org/release/HTTP-ClickHouse) - [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse) - Ruby - - [clickhouse (Ruby)](https://github.com/archan937/clickhouse) + - [ClickHouse (Ruby)](https://github.com/shlima/click_house) - R - [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r) - [RClickhouse](https://github.com/IMSMWU/RClickhouse) @@ -39,8 +39,6 @@ - C# - [ClickHouse.Ado](https://github.com/killwort/ClickHouse-Net) - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) -- C++ - - [clickhouse-cpp](https://github.com/artpaul/clickhouse-cpp/) - Elixir - [clickhousex](https://github.com/appodeal/clickhousex/) - Nim diff --git a/docs/ru/interfaces/third-party/gui.md b/docs/ru/interfaces/third-party/gui.md index 3280f1f2472..50a82c7e169 100644 --- a/docs/ru/interfaces/third-party/gui.md +++ b/docs/ru/interfaces/third-party/gui.md @@ -51,6 +51,18 @@ - Предварительный просмотр таблицы с фильтрацией и сортировкой; - Выполнение запросов только для чтения. +### Redash + +[Redash](https://github.com/getredash/redash) — платформа для отображения данных. + +Поддерживает множество источников данных, включая ClickHouse. Redash может объединять результаты запросов из разных источников в финальный набор данных. + +Основные возможности: + +- Мощный редактор запросов. +- Проводник по базе данных. +- Инструменты визуализации, позволяющие представить данные в различных формах. + ### DBeaver [DBeaver](https://dbeaver.io/) - универсальный desktop клиент баз данных с поддержкой ClickHouse. @@ -67,6 +79,7 @@ [clickhouse-cli](https://github.com/hatarist/clickhouse-cli) - это альтернативный клиент командной строки для ClickHouse, написанный на Python 3. Основные возможности: + - Автодополнение; - Подсветка синтаксиса для запросов и вывода данных; - Поддержка постраничного просмотра для результирующих данных; @@ -78,6 +91,18 @@ ## Коммерческие +### Holistics Software + +[Holistics](https://www.holistics.io/) вошёл в топ-2 наиболее удобных инструментов для бизнес-аналитики по рейтингу Gartner's Frontrunners в 2019 году. Holistics — full-stack платформа для обработки данных и инструмент бизнес-аналитики, позволяющий вам построить свои процессы с помощью SQL. + +Основные возможности: + +- Автоматизированные отчёты на почту, Slack, и Google Sheet. +- Мощный редактор SQL c визуализацией, контролем версий, автодополнением, повторным использованием частей запроса и динамическими фильтрами. +- Встроенные инструменты анализа отчётов и всплывающие (iframe) дашборды. +- Подготовка данных и возможности ETL. +- Моделирование данных с помощью SQL для их реляционного отображения. + ### DataGrip [DataGrip](https://www.jetbrains.com/datagrip/) — это IDE для баз данных о JetBrains с выделенной поддержкой ClickHouse. Он также встроен в другие инструменты на основе IntelliJ: PyCharm, IntelliJ IDEA, GoLand, PhpStorm и другие. diff --git a/docs/ru/introduction/ya_metrika_task.md b/docs/ru/introduction/history.md similarity index 99% rename from docs/ru/introduction/ya_metrika_task.md rename to docs/ru/introduction/history.md index c7e22346ae5..c0035b51f82 100644 --- a/docs/ru/introduction/ya_metrika_task.md +++ b/docs/ru/introduction/history.md @@ -1,4 +1,4 @@ -# Постановка задачи в Яндекс.Метрике +# История ClickHouse ClickHouse изначально разрабатывался для обеспечения работы [Яндекс.Метрики](https://metrika.yandex.ru/), [второй крупнейшей в мире](http://w3techs.com/technologies/overview/traffic_analysis/all) платформы для веб аналитики, и продолжает быть её ключевым компонентом. При более 13 триллионах записей в базе данных и более 20 миллиардах событий в сутки, ClickHouse позволяет генерировать индивидуально настроенные отчёты на лету напрямую из неагрегированных данных. Данная статья вкратце демонстрирует какие цели исторически стояли перед ClickHouse на ранних этапах его развития. diff --git a/docs/ru/operations/access_rights.md b/docs/ru/operations/access_rights.md index a55c9e0e798..d6a98b7b594 100644 --- a/docs/ru/operations/access_rights.md +++ b/docs/ru/operations/access_rights.md @@ -28,10 +28,10 @@ Каждый элемент списка имеет одну из следующих форм: IP-адрес или маска подсети. Например, 198.51.100.0/24 или 2001:DB8::/32. Имя хоста. Например: example01. Для проверки делается DNS-запрос, и все полученные адреса сравниваются с адресом клиента. - Регулярное выражение для имён хостов. Например, ^example\d\d-\d\d-\d\.yandex\.ru$ + Регулярное выражение для имён хостов. Например, ^example\d\d-\d\d-\d\.host\.ru$ Для проверки, для адреса клиента делается DNS PTR-запрос и к результату применяется регулярное выражение. Потом для результата PTR-запроса делается снова DNS-запрос, и все полученные адреса сравниваются с адресом клиента. - Настоятельно рекомендуется, чтобы регулярное выражение заканчивалось на \.yandex\.ru$. + Настоятельно рекомендуется, чтобы регулярное выражение заканчивалось на \.host\.ru$. Если вы устанавливаете ClickHouse самостоятельно, укажите здесь: diff --git a/docs/ru/operations/monitoring.md b/docs/ru/operations/monitoring.md index da24c7e960b..248d478506b 100644 --- a/docs/ru/operations/monitoring.md +++ b/docs/ru/operations/monitoring.md @@ -34,4 +34,4 @@ ClickHouse собирает: Также, можно отслеживать доступность сервера через HTTP API. Отправьте `HTTP GET` к ресурсу `/`. Если сервер доступен, он отвечает `200 OK`. -Для мониторинга серверов в кластерной конфигурации необходимо установить параметр [max_replica_delay_for_distributed_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries) и использовать HTTP ресурс `/replicas-delay`. Если реплика доступна и не отстаёт от других реплик, то запрос к `/replicas-delay` возвращает `200 OK`. Если реплика отстаёт, то она возвращает информацию о размере отставания. +Для мониторинга серверов в кластерной конфигурации необходимо установить параметр [max_replica_delay_for_distributed_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries) и использовать HTTP ресурс `/replicas_status`. Если реплика доступна и не отстаёт от других реплик, то запрос к `/replicas_status` возвращает `200 OK`. Если реплика отстаёт, то она возвращает информацию о размере отставания. diff --git a/docs/ru/operations/server_settings/settings.md b/docs/ru/operations/server_settings/settings.md index f95b0809650..ca1c255bee3 100644 --- a/docs/ru/operations/server_settings/settings.md +++ b/docs/ru/operations/server_settings/settings.md @@ -60,7 +60,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat База данных по умолчанию. -Перечень баз данных можно получить запросом [SHOW DATABASES](../../query_language/misc.md#show-databases). +Перечень баз данных можно получить запросом [SHOW DATABASES](../../query_language/show.md#show-databases). **Пример** @@ -368,12 +368,12 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat ## mark_cache_size {#server-mark-cache-size} -Приблизительный размер (в байтах) кеша "засечек", используемых движками таблиц семейства [MergeTree](../../operations/table_engines/mergetree.md). +Приблизительный размер (в байтах) кэша засечек, используемых движками таблиц семейства [MergeTree](../../operations/table_engines/mergetree.md). -Кеш общий для сервера, память выделяется по мере необходимости. Кеш не может быть меньше, чем 5368709120. +Кэш общий для сервера, память выделяется по мере необходимости. Кэш не может быть меньше, чем 5368709120. -!!! note ВАЖНО - Этот параметр может быть превышен при большом значении настройки пользователя [mark_cache_min_lifetime](../settings/settings.md#settings-mark_cache_min_lifetime). +!!! warning "Внимание" + Этот параметр может быть превышен при большом значении настройки [mark_cache_min_lifetime](../settings/settings.md#settings-mark_cache_min_lifetime). **Пример** @@ -580,11 +580,36 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat ``` -## remote_servers +## query_thread_log {#server_settings-query-thread-log} -Конфигурация кластеров, которые использует движок таблиц Distributed. +Настройка логирования потоков выполнения запросов, принятых с настройкой [log_query_threads=1](../settings/settings.md#settings-log-query-threads). -Пример настройки смотрите в разделе "[Движки таблиц/Distributed](../../operations/table_engines/distributed.md)". +Запросы логируются не в отдельный файл, а в системную таблицу [system.query_thread_log](../system_tables.md#system_tables-query-thread-log). Вы можете изменить название этой таблицы в параметре `table` (см. ниже). + +При настройке логирования используются следующие параметры: + +- `database` — имя базы данных; +- `table` — имя таблицы, куда будет записываться лог; +- `partition_by` — [произвольный ключ партиционирования](../../operations/table_engines/custom_partitioning_key.md) для таблицы с логами; +- `flush_interval_milliseconds` — период сброса данных из буфера в памяти в таблицу. + +Если таблица не существует, то ClickHouse создаст её. Если структура журнала запросов изменилась при обновлении сервера ClickHouse, то таблица со старой структурой переименовывается, а новая таблица создается автоматически. + +**Пример** + +```xml + + system + query_thread_log
+ toMonday(event_date) + 7500 + +``` + + +## remote_servers {#server_settings_remote_servers} + +Конфигурация кластеров, которые использует движок таблиц [Distributed](../../operations/table_engines/distributed.md) и табличная функция `cluster`. **Пример** @@ -595,6 +620,9 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat Значение атрибута `incl` смотрите в разделе "[Конфигурационные файлы](../configuration_files.md#configuration_files)". +**Смотрите также** + + - [skip_unavailable_shards](../settings/settings.md#settings-skip_unavailable_shards) ## timezone @@ -701,12 +729,12 @@ ClickHouse использует ZooKeeper для хранения метадан Например: - ```xml +```xml example_host 2181 - ``` +``` Атрибут `index` задает порядок опроса нод при попытках подключиться к кластеру ZooKeeper. diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index d37e0911698..e0045bd42ef 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -384,52 +384,86 @@ Ok. При чтении из таблиц [MergeTree*](../table_engines/mergetree.md) ClickHouse использует несколько потоков. Этот параметр включает/выключает равномерное распределение заданий по рабочим потокам. Алгоритм равномерного распределения стремится сделать время выполнения всех потоков примерно равным для одного запроса `SELECT`. -**Возможные значения** +Возможные значения: - 0 — не использовать равномерное распределение заданий на чтение. - 1 — использовать равномерное распределение заданий на чтение. -**Значение по умолчанию**: 1. +Значение по умолчанию — 1. ## merge_tree_min_rows_for_concurrent_read {#setting-merge_tree_min_rows_for_concurrent_read} Если количество строк, считываемых из файла таблицы [MergeTree*](../table_engines/mergetree.md) превышает `merge_tree_min_rows_for_concurrent_read`, то ClickHouse пытается выполнить одновременное чтение из этого файла в несколько потоков. -**Возможные значения** +Возможные значения: -Любое положительное целое число. +- Любое положительное целое число. -**Значение по умолчанию**: 163840. +Значение по умолчанию — 163840. + +## merge_tree_min_bytes_for_concurrent_read {#setting-merge_tree_min_bytes_for_concurrent_read} + +Если число байтов, которые должны быть прочитаны из одного файла таблицы с движком [MergeTree*](../table_engines/mergetree.md) превышает `merge_tree_min_bytes_for_concurrent_read`, то ClickHouse пытается выполнить конкурентное чтение в несколько потоков из этого файла. + +Возможные значения: + +- Положительное целое число. + +Значение по умолчанию — 240 ✕ 1024 ✕ 1024. ## merge_tree_min_rows_for_seek {#setting-merge_tree_min_rows_for_seek} -Если расстояние между двумя блоками данных для чтения в одном файле меньше, чем `merge_tree_min_rows_for_seek` строк, то ClickHouse не перескакивает через блоки, а считывает данные последовательно. +Если расстояние между двумя блоками данных для чтения в одном файле меньше, чем `merge_tree_min_rows_for_seek` строк, то ClickHouse не перескакивает (seek) через блоки, а считывает данные последовательно. -**Возможные значения** +Возможные значения: -Любое положительное целое число. +- Положительное целое число. -**Значение по умолчанию**: 0. +Значение по умолчанию — 0. + +## merge_tree_min_bytes_for_seek {#setting-merge_tree_min_bytes_for_seek} + +Если расстояние между двумя блоками данных для чтения в одном файле меньше, чем `merge_tree_min_bytes_for_seek` байтов, то ClickHouse не перескакивает (seek) через блоки, а считывает данные последовательно. + +Возможные значения: + +- Положительное целое число. + +Значение по умолчанию — 0. ## merge_tree_coarse_index_granularity {#setting-merge_tree_coarse_index_granularity} При поиске данных ClickHouse проверяет засечки данных в файле индекса. Если ClickHouse обнаруживает, что требуемые ключи находятся в некотором диапазоне, он делит этот диапазон на `merge_tree_coarse_index_granularity` поддиапазонов и выполняет в них рекурсивный поиск нужных ключей. -**Возможные значения** +Возможные значения: -Любое положительное целое число. +- Положительное целое число. -**Значение по умолчанию**: 8. +Значение по умолчанию — 8. ## merge_tree_max_rows_to_use_cache {#setting-merge_tree_max_rows_to_use_cache} -Если требуется прочитать более, чем `merge_tree_max_rows_to_use_cache` строк в одном запросе, ClickHouse не используют кэш несжатых блоков. Настройка сервера [uncompressed_cache_size](../server_settings/settings.md#server-settings-uncompressed_cache_size) определяет размер кэша несжатых блоков. +Если требуется прочитать более, чем `merge_tree_max_rows_to_use_cache` строк в одном запросе, ClickHouse не используют кэш несжатых блоков. -**Возможные значения** +Кэш несжатых блоков хранит данные, извлечённые при выполнении запросов. ClickHouse использует этот кэш для ускорения ответов на повторяющиеся небольшие запросы. Настройка защищает кэш от замусоривания запросами, для выполнения которых необходимо извлечь большое количество данных. Настройка сервера [uncompressed_cache_size](../server_settings/settings.md#server-settings-uncompressed_cache_size) определяет размер кэша несжатых блоков. -Любое положительное целое число. +Возможные значения: -**Значение по умолчанию**: 1048576. +- Положительное целое число. + +Значение по умолчанию — 128 ✕ 8192. + +## merge_tree_max_bytes_to_use_cache {#setting-merge_tree_max_bytes_to_use_cache} + +Если требуется прочитать более, чем `merge_tree_max_bytes_to_use_cache` байтов в одном запросе, ClickHouse не используют кэш несжатых блоков. + +Кэш несжатых блоков хранит данные, извлечённые при выполнении запросов. ClickHouse использует этот кэш для ускорения ответов на повторяющиеся небольшие запросы. Настройка защищает кэш от замусоривания запросами, для выполнения которых необходимо извлечь большое количество данных. Настройка сервера [uncompressed_cache_size](../server_settings/settings.md#server-settings-uncompressed_cache_size) определяет размер кэша несжатых блоков. + +Возможные значения: + +- Положительное целое число. + +Значение по умолчанию — 1920 ✕ 1024 ✕ 1024. ## min_bytes_to_use_direct_io {#settings-min_bytes_to_use_direct_io} @@ -454,6 +488,16 @@ ClickHouse использует этот параметр при чтении д log_queries=1 +## log_query_threads {#settings-log-query-threads} + +Установка логирования информации о потоках выполнения запроса. + +Лог информации о потоках выполнения запросов, переданных в ClickHouse с этой установкой, записывается согласно правилам конфигурационного параметра сервера [query_thread_log](../server_settings/settings.md#server_settings-query-thread-log). + +**Пример** : + + log_query_threads=1 + ## max_insert_block_size {#settings-max_insert_block_size} Формировать блоки указанного размера, при вставке в таблицу. @@ -536,6 +580,12 @@ Default value: 10000 seconds. Значение по умолчанию: 10, 300, 300. +## cancel_http_readonly_queries_on_client_close + +Отменяет HTTP readonly запросы (например, SELECT), когда клиент обрывает соединение до завершения получения данных. + +Значение по умолчанию: 0 + ## poll_interval Блокироваться в цикле ожидания запроса в сервере на указанное количество секунд. @@ -853,6 +903,36 @@ load_balancing = first_or_random - [Множественный JOIN](../../query_language/select.md#select-join) +## skip_unavailable_shards {#settings-skip_unavailable_shards} + +Включает или отключает тихий пропуск недоступных шардов. + +Шард считается недоступным, если все его реплики недоступны. Реплика недоступна в следующих случаях: + +- ClickHouse не может установить соединение с репликой по любой причине. + + ClickHouse предпринимает несколько попыток подключиться к реплике. Если все попытки оказались неудачными, реплика считается недоступной. + +- Реплика не может быть разрешена с помощью DNS. + + Если имя хоста реплики не может быть разрешено с помощью DNS, это может указывать на следующие ситуации: + + - Нет записи DNS для хоста. Это может происходить в системах с динамическим DNS, например, [Kubernetes](https://kubernetes.io), где отключенные ноды не разрешаться с помощью DNS и это не ошибка. + + - Ошибка конфигурации. Конфигурационный файл ClickHouse может содержать неправильное имя хоста. + +Возможные значения: + +- 1 — пропуск включен. + + Если шард недоступен, то ClickHouse возвращает результат, основанный на неполных данных и не оповещает о проблемах с доступностью хостов. + +- 0 — пропуск выключен. + + Если шард недоступен, то ClickHouse генерирует исключение. + +Значение по умолчанию — 0. + ## optimize_throw_if_noop {#setting-optimize_throw_if_noop} Включает или отключает генерирование исключения в в случаях, когда запрос [OPTIMIZE](../../query_language/misc.md#misc_operations-optimize) не выполняет мёрж. @@ -866,6 +946,7 @@ load_balancing = first_or_random Значение по умолчанию — 0. + ## os_thread_priority {#setting-os_thread_priority} Устанавливает приоритет ([nice](https://en.wikipedia.org/wiki/Nice_(Unix))) для потоков, исполняющих запросы. Планировщик ОС учитывает эти приоритеты при выборе следующего потока для исполнения на доступном ядре CPU. diff --git a/docs/ru/operations/settings/settings_users.md b/docs/ru/operations/settings/settings_users.md index adb50a02a9d..dfa3e40c660 100644 --- a/docs/ru/operations/settings/settings_users.md +++ b/docs/ru/operations/settings/settings_users.md @@ -63,13 +63,13 @@ - `` — Имя хоста. - Пример: `server01.yandex.ru`. + Пример: `example01.host.ru`. Для проверки доступа выполняется DNS-запрос, и все возвращенные IP-адреса сравниваются с адресом клиента. - `` — Регулярное выражение для имен хостов. - Пример, `^server\d\d-\d\d-\d\.yandex\.ru$` + Пример, `^example\d\d-\d\d-\d\.host\.ru$` Для проверки доступа выполняется [DNS запрос PTR](https://en.wikipedia.org/wiki/Reverse_DNS_lookup) для адреса клиента, а затем применяется заданное регулярное выражение. Затем, для результатов запроса PTR выполняется другой DNS-запрос и все полученные адреса сравниваются с адресом клиента. Рекомендуем завершать регулярное выражение символом $. diff --git a/docs/ru/operations/system_tables.md b/docs/ru/operations/system_tables.md index eb757480774..7aaf7f9ee4f 100644 --- a/docs/ru/operations/system_tables.md +++ b/docs/ru/operations/system_tables.md @@ -201,6 +201,7 @@ SELECT * FROM system.events LIMIT 5 Содержит информацию о том, какие параметры [graphite_rollup](server_settings/settings.md#server_settings-graphite_rollup) используются в таблицах с движками [\*GraphiteMergeTree](table_engines/graphitemergetree.md). Столбцы: + - `config_name` (String) - Имя параметра, используемого для `graphite_rollup`. - `regexp` (String) - Шаблон имени метрики. - `function` (String) - Имя агрегирующей функции. @@ -329,7 +330,8 @@ SELECT * FROM system.metrics LIMIT 10 - `database` (`String`) – имя базы данных. - `table` (`String`) – имя таблицы. - `engine` (`String`) – имя движка таблицы, без параметров. -- `path` (`String`) – абсолютный путь к папке с файлами кусков данных.. +- `path` (`String`) – абсолютный путь к папке с файлами кусков данных. +- `disk` (`String`) – имя диска, на котором находится кусок данных. - `hash_of_all_files` (`String`) – значение [sipHash128](../query_language/functions/hash_functions.md#hash_functions-siphash128) для сжатых файлов. - `hash_of_uncompressed_files` (`String`) – значение [sipHash128](../query_language/functions/hash_functions.md#hash_functions-siphash128) несжатых файлов (файлы с засечками, первичным ключом и пр.) - `uncompressed_hash_of_compressed_files` (`String`) – значение [sipHash128](../query_language/functions/hash_functions.md#hash_functions-siphash128) данных в сжатых файлах как если бы они были разжатыми. @@ -345,49 +347,46 @@ SELECT * FROM system.metrics LIMIT 10 Столбцы: -- `event_type` (Enum) — тип события. Столбец может содержать одно из следующих значений: `NEW_PART` — вставка нового куска; `MERGE_PARTS` — слияние кусков; `DOWNLOAD_PART` — загрузка с реплики; `REMOVE_PART` — удаление или отсоединение из таблицы с помощью [DETACH PARTITION](../query_language/alter.md#alter_detach-partition); `MUTATE_PART` — изменение куска; `MOVE_PART` — перемещение куска между дисками. -- `event_date` (Date) — дата события; -- `event_time` (DateTime) — время события; -- `duration_ms` (UInt64) — длительность; -- `database` (String) — имя базы данных, в которой находится кусок; -- `table` (String) — имя таблицы, в которой находится кусок; -- `part_name` (String) — имя куска; -- `partition_id` (String) — идентификатор партиции, в которую был добавлен кусок. В столбце будет значение 'all', если таблица партициируется по выражению `tuple()`; -- `rows` (UInt64) — число строк в куске; -- `size_in_bytes` (UInt64) — размер куска данных в байтах; -- `merged_from` (Array(String)) — массив имён кусков, из которых образован текущий кусок в результате слияния (также столбец заполняется в случае скачивания уже смерженного куска); -- `bytes_uncompressed` (UInt64) — количество прочитанных разжатых байт; -- `read_rows` (UInt64) — сколько было прочитано строк при слиянии кусков; -- `read_bytes` (UInt64) — сколько было прочитано байт при слиянии кусков; -- `error` (UInt16) — код ошибки, возникшей при текущем событии; +- `event_type` (Enum) — тип события. Столбец может содержать одно из следующих значений: + - `NEW_PART` — вставка нового куска. + - `MERGE_PARTS` — слияние кусков. + - `DOWNLOAD_PART` — загрузка с реплики. + - `REMOVE_PART` — удаление или отсоединение из таблицы с помощью [DETACH PARTITION](../query_language/alter.md#alter_detach-partition). + - `MUTATE_PART` — изменение куска. + - `MOVE_PART` — перемещение куска между дисками. +- `event_date` (Date) — дата события. +- `event_time` (DateTime) — время события. +- `duration_ms` (UInt64) — длительность. +- `database` (String) — имя базы данных, в которой находится кусок. +- `table` (String) — имя таблицы, в которой находится кусок. +- `part_name` (String) — имя куска. +- `partition_id` (String) — идентификатор партиции, в которую был добавлен кусок. В столбце будет значение 'all', если таблица партициируется по выражению `tuple()`. +- `rows` (UInt64) — число строк в куске. +- `size_in_bytes` (UInt64) — размер куска данных в байтах. +- `merged_from` (Array(String)) — массив имён кусков, из которых образован текущий кусок в результате слияния (также столбец заполняется в случае скачивания уже смерженного куска). +- `bytes_uncompressed` (UInt64) — количество прочитанных разжатых байт. +- `read_rows` (UInt64) — сколько было прочитано строк при слиянии кусков. +- `read_bytes` (UInt64) — сколько было прочитано байт при слиянии кусков. +- `error` (UInt16) — код ошибки, возникшей при текущем событии. - `exception` (String) — текст ошибки. Системная таблица `system.part_log` будет создана после первой вставки данных в таблицу `MergeTree`. -## system.processes +## system.processes {#system_tables-processes} + +Используется для реализации запроса `SHOW PROCESSLIST`. -Эта системная таблица используется для реализации запроса `SHOW PROCESSLIST`. Столбцы: -```text -user String - имя пользователя, который задал запрос. При распределённой обработке запроса, относится к пользователю, с помощью которого сервер-инициатор запроса отправил запрос на данный сервер, а не к имени пользователя, который задал распределённый запрос на сервер-инициатор запроса. - -address String - IP-адрес, с которого задан запрос. При распределённой обработке запроса, аналогично. - -elapsed Float64 - время в секундах, прошедшее от начала выполнения запроса. - -rows_read UInt64 - количество прочитанных из таблиц строк. При распределённой обработке запроса, на сервере-инициаторе запроса, представляет собой сумму по всем удалённым серверам. - -bytes_read UInt64 - количество прочитанных из таблиц байт, в несжатом виде. При распределённой обработке запроса, на сервере-инициаторе запроса, представляет собой сумму по всем удалённым серверам. - -total_rows_approx UInt64 - приблизительная оценка общего количества строк, которые должны быть прочитаны. При распределённой обработке запроса, на сервере-инициаторе запроса, представляет собой сумму по всем удалённым серверам. Может обновляться в процессе выполнения запроса, когда становятся известны новые источники для обработки. - -memory_usage UInt64 - потребление памяти запросом. Может не учитывать некоторые виды выделенной памяти. - -query String - текст запроса. В случае INSERT - без данных для INSERT-а. - -query_id String - идентификатор запроса, если был задан. -``` +- `user` (String) – пользователь, инициировавший запрос. При распределённом выполнении запросы отправляются на удалённые серверы от имени пользователя `default`. Поле содержит имя пользователя для конкретного запроса, а не для запроса, который иницировал этот запрос. +- `address` (String) – IP-адрес, с которого пришёл запрос. При распределённой обработке запроса аналогично. Чтобы определить откуда запрос пришел изначально, необходимо смотреть таблицу `system.processes` на сервере-источнике запроса. +- `elapsed` (Float64) – время в секундах с начала обработки запроса. +- `rows_read` (UInt64) – количество прочитанных строк. При распределённой обработке запроса на сервере-инициаторе запроса представляет собой сумму по всем удалённым серверам. +- `bytes_read` (UInt64) – количество прочитанных из таблиц байт, в несжатом виде. При распределённой обработке запроса на сервере-инициаторе запроса представляет собой сумму по всем удалённым серверам. +- `total_rows_approx` (UInt64) – приблизительная оценка общего количества строк, которые должны быть прочитаны. При распределённой обработке запроса, на сервере-инициаторе запроса, представляет собой сумму по всем удалённым серверам. Может обновляться в процессе выполнения запроса, когда становятся известны новые источники для обработки. +- `memory_usage` (UInt64) – потребление памяти запросом. Может не учитывать некоторые виды выделенной памяти. Смотрите описание настройки [max_memory_usage](../operations/settings/query_complexity.md#settings_max_memory_usage). +- `query` (String) – текст запроса. Для запросов `INSERT` не содержит встаявляемые данные. +- `query_id` (String) – идентификатор запроса, если был задан. ## system.query_log {#system_tables-query-log} @@ -407,13 +406,13 @@ ClickHouse создаёт таблицу только в том случае, к Столбцы: -- `type` (UInt8) — тип события, произошедшего при выполнении запроса. Возможные значения: - - 1 — успешное начало выполнения запроса. - - 2 — успешное завершение выполнения запроса. - - 3 — исключение перед началом обработки запроса. - - 4 — исключение во время обработки запроса. -- `event_date` (Date) — дата события. -- `event_time` (DateTime) — время события. +- `type` (`Enum8`) — тип события, произошедшего при выполнении запроса. Значения: + - `'QueryStart' = 1` — успешное начало выполнения запроса. + - `'QueryFinish' = 2` — успешное завершение выполнения запроса. + - `'ExceptionBeforeStart' = 3` — исключение перед началом обработки запроса. + - `'ExceptionWhileProcessing' = 4` — исключение во время обработки запроса. +- `event_date` (Date) — дата начала запроса. +- `event_time` (DateTime) — время начала запроса. - `query_start_time` (DateTime) — время начала обработки запроса. - `query_duration_ms` (UInt64) — длительность обработки запроса. - `read_rows` (UInt64) — количество прочитанных строк. @@ -423,43 +422,39 @@ ClickHouse создаёт таблицу только в том случае, к - `result_rows` (UInt64) — количество строк в результате. - `result_bytes` (UInt64) — объём результата в байтах. - `memory_usage` (UInt64) — потребление RAM запросом. -- `query` (String) — строка запроса. -- `exception` (String) — сообщение исключения. +- `query` (String) — текст запроса. +- `exception` (String) — сообщение исключения, если запрос завершился по исключению. - `stack_trace` (String) — трассировка (список функций, последовательно вызванных перед ошибкой). Пустая строка, если запрос успешно завершен. - `is_initial_query` (UInt8) — вид запроса. Возможные значения: - 1 — запрос был инициирован клиентом. - 0 — запрос был инициирован другим запросом при распределенном запросе. - `user` (String) — пользователь, запустивший текущий запрос. - `query_id` (String) — ID запроса. -- `address` (FixedString(16)) — IP адрес, с которого пришел запрос. -- `port` (UInt16) — порт, на котором сервер принял запрос. +- `address` (IPv6) — IP адрес, с которого пришел запрос. +- `port` (UInt16) — порт, с которого клиент сделал запрос - `initial_user` (String) — пользователь, запустивший первоначальный запрос (для распределенных запросов). - `initial_query_id` (String) — ID родительского запроса. -- `initial_address` (FixedString(16)) — IP адрес, с которого пришел родительский запрос. -- `initial_port` (UInt16) — порт, на котором сервер принял родительский запрос от клиента. +- `initial_address` (IPv6) — IP адрес, с которого пришел родительский запрос. +- `initial_port` (UInt16) — порт, с которого клиент сделал родительский запрос. - `interface` (UInt8) — интерфейс, с которого ушёл запрос. Возможные значения: - 1 — TCP. - 2 — HTTP. -- `os_user` (String) — операционная система пользователя. -- `client_hostname` (String) — имя сервера, к которому присоединился [clickhouse-client](../interfaces/cli.md). -- `client_name` (String) — [clickhouse-client](../interfaces/cli.md). -- `client_revision` (UInt32) — ревизия [clickhouse-client](../interfaces/cli.md). -- `client_version_major` (UInt32) — старшая версия [clickhouse-client](../interfaces/cli.md). -- `client_version_minor` (UInt32) — младшая версия [clickhouse-client](../interfaces/cli.md). -- `client_version_patch` (UInt32) — патч [clickhouse-client](../interfaces/cli.md). +- `os_user` (String) — имя пользователя в OS, который запустил [clickhouse-client](../interfaces/cli.md). +- `client_hostname` (String) — имя сервера, с которого присоединился [clickhouse-client](../interfaces/cli.md) или другой TCP клиент. +- `client_name` (String) — [clickhouse-client](../interfaces/cli.md) или другой TCP клиент. +- `client_revision` (UInt32) — ревизия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. +- `client_version_major` (UInt32) — старшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. +- `client_version_minor` (UInt32) — младшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. +- `client_version_patch` (UInt32) — патч [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. - `http_method` (UInt8) — HTTP метод, инициировавший запрос. Возможные значения: - 0 — запрос запущен с интерфейса TCP. - 1 — `GET`. - 2 — `POST`. - `http_user_agent` (String) — HTTP заголовок `UserAgent`. -- `quota_key` (String) — идентификатор квоты из настроек [квот](quotas.md). +- `quota_key` (String) — "ключ квоты" из настроек [квот](quotas.md) (см. `keyed`). - `revision` (UInt32) — ревизия ClickHouse. - `thread_numbers` (Array(UInt32)) — количество потоков, участвующих в обработке запросов. -- `ProfileEvents.Names` (Array(String)) — Счетчики для изменения метрик: - - Время, потраченное на чтение и запись по сети. - - Время, потраченное на чтение и запись на диск. - - Количество сетевых ошибок. - - Время, потраченное на ожидание, когда пропускная способность сети ограничена. +- `ProfileEvents.Names` (Array(String)) — Счетчики для изменения различных метрик. Описание метрик можно получить из таблицы [system.events](#system_tables-events - `ProfileEvents.Values` (Array(UInt64)) — метрики, перечисленные в столбце `ProfileEvents.Names`. - `Settings.Names` (Array(String)) — имена настроек, которые меняются, когда клиент выполняет запрос. Чтобы разрешить логирование изменений настроек, установите параметр `log_query_settings` равным 1. - `Settings.Values` (Array(String)) — Значения настроек, которые перечислены в столбце `Settings.Names`. @@ -479,6 +474,72 @@ ClickHouse создаёт таблицу только в том случае, к Можно указать произвольный ключ партиционирования для таблицы `system.query_log` в конфигурации [query_log](server_settings/settings.md#server_settings-query-log) (параметр `partition_by`). +## system.query_thread_log {#system_tables-query-thread-log} + +Содержит информацию о каждом потоке выполняемых запросов. + +ClickHouse создаёт таблицу только в том случае, когда установлен конфигурационный параметр сервера [query_thread_log](server_settings/settings.md#server_settings-query-thread-log). Параметр задаёт правила ведения лога, такие как интервал логирования или имя таблицы, в которую будут логгироваться запросы. + +Чтобы включить логирование, задайте значение параметра [log_query_threads](settings/settings.md#settings-log-query-threads) равным 1. Подробности смотрите в разделе [Настройки](settings/settings.md). + +Столбцы: + +- `event_date` (Date) — дата завершения выполнения запроса потоком. +- `event_time` (DateTime) — дата и время завершения выполнения запроса потоком. +- `query_start_time` (DateTime) — время начала обработки запроса. +- `query_duration_ms` (UInt64) — длительность обработки запроса в миллисекундах. +- `read_rows` (UInt64) — количество прочитанных строк. +- `read_bytes` (UInt64) — количество прочитанных байтов. +- `written_rows` (UInt64) — количество записанных строк для запросов `INSERT`. Для других запросов, значение столбца 0. +- `written_bytes` (UInt64) — объем записанных данных в байтах для запросов `INSERT`. Для других запросов, значение столбца 0. +- `memory_usage` (Int64) — разница между выделенной и освобождённой памятью в контексте потока. +- `peak_memory_usage` (Int64) — максимальная разница между выделенной и освобождённой памятью в контексте потока. +- `thread_name` (String) — Имя потока. +- `thread_number` (UInt32) — Внутренний ID потока. +- `os_thread_id` (Int32) — tid (ID потока операционной системы). +- `master_thread_number` (UInt32) — Внутренний ID главного потока. +- `master_os_thread_id` (Int32) — tid (ID потока операционной системы) главного потока. +- `query` (String) — текст запроса. +- `is_initial_query` (UInt8) — вид запроса. Возможные значения: + - 1 — запрос был инициирован клиентом. + - 0 — запрос был инициирован другим запросом при распределенном запросе. +- `user` (String) — пользователь, запустивший текущий запрос. +- `query_id` (String) — ID запроса. +- `address` (IPv6) — IP адрес, с которого пришел запрос. +- `port` (UInt16) — порт, с которого пришел запрос. +- `initial_user` (String) — пользователь, запустивший первоначальный запрос (для распределенных запросов). +- `initial_query_id` (String) — ID родительского запроса. +- `initial_address` (IPv6) — IP адрес, с которого пришел родительский запрос. +- `initial_port` (UInt16) — порт, пришел родительский запрос. +- `interface` (UInt8) — интерфейс, с которого ушёл запрос. Возможные значения: + - 1 — TCP. + - 2 — HTTP. +- `os_user` (String) — имя пользователя в OS, который запустил [clickhouse-client](../interfaces/cli.md). +- `client_hostname` (String) — hostname клиентской машины, с которой присоединился [clickhouse-client](../interfaces/cli.md) или другой TCP клиент. +- `client_name` (String) — [clickhouse-client](../interfaces/cli.md) или другой TCP клиент. +- `client_revision` (UInt32) — ревизия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. +- `client_version_major` (UInt32) — старшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. +- `client_version_minor` (UInt32) — младшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. +- `client_version_patch` (UInt32) — патч [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. +- `http_method` (UInt8) — HTTP метод, инициировавший запрос. Возможные значения: + - 0 — запрос запущен с интерфейса TCP. + - 1 — `GET`. + - 2 — `POST`. +- `http_user_agent` (String) — HTTP заголовок `UserAgent`. +- `quota_key` (String) — "ключ квоты" из настроек [квот](quotas.md) (см. `keyed`). +- `revision` (UInt32) — ревизия ClickHouse. +- `ProfileEvents.Names` (Array(String)) — Счетчики для изменения различных метрик для данного потока. Описание метрик можно получить из таблицы [system.events](#system_tables-events +- `ProfileEvents.Values` (Array(UInt64)) — метрики для данного потока, перечисленные в столбце `ProfileEvents.Names`. + +По умолчанию, строки добавляются в таблицу логирования с интервалом в 7,5 секунд. Можно задать интервал в конфигурационном параметре сервера [query_thread_log](server_settings/settings.md#server_settings-query-thread-log) (смотрите параметр `flush_interval_milliseconds`). Чтобы принудительно записать логи из буффера памяти в таблицу, используйте запрос `SYSTEM FLUSH LOGS`. + +Если таблицу удалить вручную, она пересоздастся автоматически "на лету". При этом все логи на момент удаления таблицы будут удалены. + +!!! note "Примечание" + Срок хранения логов не ограничен. Логи не удаляются из таблицы автоматически. Вам необходимо самостоятельно организовать удаление устаревших логов. + +Можно указать произвольный ключ партиционирования для таблицы `system.query_log` в конфигурации [query_thread_log](server_settings/settings.md#server_settings-query-thread-log) (параметр `partition_by`). + ## system.replicas {#system_tables-replicas} Содержит информацию и статус для реплицируемых таблиц, расположенных на локальном сервере. @@ -756,26 +817,32 @@ path: /clickhouse/tables/01-08/visits/replicas **latest_fail_reason** — причина последней ошибки мутации. -[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/system_tables/) - ## system.disks {#system_tables-disks} -Таблица содержит информацию о дисках, заданных в [конфигурации сервера](table_engines/mergetree.md#table_engine-mergetree-multiple-volumes_configure). Имеет следующие столбцы: +Cодержит информацию о дисках, заданных в [конфигурации сервера](table_engines/mergetree.md#table_engine-mergetree-multiple-volumes_configure). -- `name String` — имя диска в конфигурации сервера. -- `path String` — путь к точке монтирования на файловой системе. -- `free_space UInt64` — свободное место на диске в данный момент времени в байтах. -- `total_space UInt64` — общее количество места на диске в данный момент времени в байтах. -- `keep_free_space UInt64` — количество байт, которое должно оставаться свободным (задается в конфигурации). +Столбцы: + +- `name` ([String](../data_types/string.md)) — имя диска в конфигурации сервера. +- `path` ([String](../data_types/string.md)) — путь к точке монтирования в файловой системе. +- `free_space` ([UInt64](../data_types/int_uint.md)) — свободное место на диске в байтах. +- `total_space` ([UInt64](../data_types/int_uint.md)) — объём диска в байтах. +- `keep_free_space` ([UInt64](../data_types/int_uint.md)) — место, которое должно остаться свободным на диске в байтах. Задаётся значением параметра `keep_free_space_bytes` конфигурации дисков. ## system.storage_policies {#system_tables-storage_policies} -Таблица содержит информацию о политиках хранения и томах, заданных в [конфигурации сервера](table_engines/mergetree.md#table_engine-mergetree-multiple-volumes_configure). Данные в таблице денормализованны, имя одной политики хранения может содержаться несколько раз, по количеству томов в ней. Имеет следующие столбцы: +Содержит информацию о политиках хранения и томах, заданных в [конфигурации сервера](table_engines/mergetree.md#table_engine-mergetree-multiple-volumes_configure). -- `policy_name String` — имя политики хранения в конфигурации сервера. -- `volume_name String` — имя тома, который содержится в данной политике хранения. -- `volume_priority UInt64` — порядковый номер тома, согласно конфигурации. -- `disks Array(String)` — имена дисков, содержащихся в данной политике хранения. -- `max_data_part_size UInt64` — максимальный размер куска, который может храниться на дисках этого тома (0 — без ограничений). -- `move_factor Float64` — доля свободного места, при превышении которой данные начинают перемещаться на следующий том. +Столбцы: + +- `policy_name` ([String](../data_types/string.md)) — имя политики хранения. +- `volume_name` ([String](../data_types/string.md)) — имя тома, который содержится в политике хранения. +- `volume_priority` ([UInt64](../data_types/int_uint.md)) — порядковый номер тома согласно конфигурации. +- `disks` ([Array(String)](../data_types/array.md)) — имена дисков, содержащихся в политике хранения. +- `max_data_part_size` ([UInt64](../data_types/int_uint.md)) — максимальный размер куска данных, который может храниться на дисках тома (0 — без ограничений). +- `move_factor` ([Float64](../data_types/float.md))` — доля свободного места, при превышении которой данные начинают перемещаться на следующий том. + +Если политика хранения содержит несколько томов, то каждому тому соответствует отдельная запись в таблице. + +[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/system_tables/) diff --git a/docs/ru/operations/table_engines/buffer.md b/docs/ru/operations/table_engines/buffer.md index bf3c1b450fc..964897162c2 100644 --- a/docs/ru/operations/table_engines/buffer.md +++ b/docs/ru/operations/table_engines/buffer.md @@ -7,18 +7,21 @@ Buffer(database, table, num_layers, min_time, max_time, min_rows, max_rows, min_ ``` Параметры движка: -database, table - таблица, в которую сбрасывать данные. Вместо имени базы данных может использоваться константное выражение, возвращающее строку. -num_layers - уровень параллелизма. Физически таблица будет представлена в виде num_layers независимых буферов. Рекомендуемое значение - 16. -min_time, max_time, min_rows, max_rows, min_bytes, max_bytes - условия для сброса данных из буфера. -Данные сбрасываются из буфера и записываются в таблицу назначения, если выполнены все min-условия или хотя бы одно max-условие. -min_time, max_time - условие на время в секундах от момента первой записи в буфер; -min_rows, max_rows - условие на количество строк в буфере; -min_bytes, max_bytes - условие на количество байт в буфере. +`database` — имя базы данных. Вместо имени базы данных может использоваться константное выражение, возвращающее строку. +`table` — таблица, в которую сбрасывать данные. +`num_layers` — уровень параллелизма. Физически таблица будет представлена в виде `num_layers` независимых буферов. Рекомендуемое значение — 16. +`min_time`, `max_time`, `min_rows`, `max_rows`, `min_bytes`, `max_bytes` — условия для сброса данных из буфера. -При записи, данные вставляются в случайный из num_layers буферов. Или, если размер куска вставляемых данных достаточно большой (больше max_rows или max_bytes), то он записывается в таблицу назначения минуя буфер. +Данные сбрасываются из буфера и записываются в таблицу назначения, если выполнены все `min`-условия или хотя бы одно `max`-условие. -Условия для сброса данных учитываются отдельно для каждого из num_layers буферов. Например, если num_layers = 16 и max_bytes = 100000000, то максимальный расход оперативки будет 1.6 GB. +- `min_time`, `max_time` — условие на время в секундах от момента первой записи в буфер. +- `min_rows`, `max_rows` — условие на количество строк в буфере. +- `min_bytes`, `max_bytes` — условие на количество байт в буфере. + +При записи, данные вставляются в случайный из `num_layers` буферов. Или, если размер куска вставляемых данных достаточно большой (больше `max_rows` или `max_bytes`), то он записывается в таблицу назначения минуя буфер. + +Условия для сброса данных учитываются отдельно для каждого из `num_layers` буферов. Например, если `num_layers = 16` и `max_bytes = 100000000`, то максимальный расход оперативки будет 1.6 GB. Пример: diff --git a/docs/ru/operations/table_engines/file.md b/docs/ru/operations/table_engines/file.md index bb8f831235f..4d2d3d4d6fb 100644 --- a/docs/ru/operations/table_engines/file.md +++ b/docs/ru/operations/table_engines/file.md @@ -69,6 +69,8 @@ $ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64 ## Детали реализации - Поддерживается одновременное выполнение множества запросов `SELECT`, запросы `INSERT` могут выполняться только последовательно. +- Поддерживается создание ещё не существующего файла при запросе `INSERT`. +- Для существующих файлов `INSERT` записывает в конец файла. - Не поддерживается: - использование операций `ALTER` и `SELECT...SAMPLE`; - индексы; diff --git a/docs/ru/operations/table_engines/mergetree.md b/docs/ru/operations/table_engines/mergetree.md index 931a969f076..f3eba70f0e2 100644 --- a/docs/ru/operations/table_engines/mergetree.md +++ b/docs/ru/operations/table_engines/mergetree.md @@ -40,7 +40,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] [ORDER BY expr] [PRIMARY KEY expr] [SAMPLE BY expr] -[TTL expr] +[TTL expr [DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'], ...] [SETTINGS name=value, ...] ``` @@ -70,19 +70,26 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] Если используется выражение для сэмплирования, то первичный ключ должен содержать его. Пример: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`. -- `TTL` — выражение, определяющее длительность хранения строк. +- `TTL` — список правил, определяющих длительности хранения строк, а также задающих правила перемещения частей на определённые тома или диски. - Должно зависеть от столбца `Date` или `DateTime` и возвращать столбец `Date` или `DateTime`. Пример:`TTL date + INTERVAL 1 DAY` + Выражение должно возвращать столбец `Date` или `DateTime`. Пример: `TTL date + INTERVAL 1 DAY`. + + Тип правила `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'` указывает действие, которое будет выполнено с частью, удаление строк (прореживание), перемещение (при выполнении условия для всех строк части) на определённый диск (`TO DISK 'xxx'`) или том (`TO VOLUME 'xxx'`). Поведение по умолчанию соответствует удалению строк (`DELETE`). В списке правил может быть указано только одно выражение с поведением `DELETE`. Дополнительные сведения смотрите в разделе [TTL для столбцов и таблиц](#table_engine-mergetree-ttl) - `SETTINGS` — дополнительные параметры, регулирующие поведение `MergeTree`: - - `index_granularity` — гранулярность индекса. Число строк данных между «засечками» индекса. По умолчанию — 8192. Список всех доступных параметров можно посмотреть в [MergeTreeSettings.h](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/src/Storages/MergeTree/MergeTreeSettings.h). - - `min_merge_bytes_to_use_direct_io` — минимальный объем данных, необходимый для прямого (небуферизованного) чтения/записи (direct I/O) на диск. При слиянии частей данных ClickHouse вычисляет общий объем хранения всех данных, подлежащих слиянию. Если общий объем хранения всех данных для чтения превышает `min_bytes_to_use_direct_io` байт, тогда ClickHouse использует флаг `O_DIRECT` при чтении данных с диска. Если `min_merge_bytes_to_use_direct_io = 0`, тогда прямой ввод-вывод отключен. Значение по умолчанию: `10 * 1024 * 1024 * 1024` байт. + - `index_granularity` — максимальное количество строк данных между засечками индекса. По умолчанию — 8192. Смотрите [Хранение данных](#mergetree-data-storage). + - `index_granularity_bytes` — максимальный размер гранул данных в байтах. По умолчанию — 10Mb. Чтобы ограничить размер гранул только количеством строк, установите значение 0 (не рекомендовано). Смотрите [Хранение данных](#mergetree-data-storage). + - `enable_mixed_granularity_parts` — включает или выключает переход к ограничению размера гранул с помощью настройки `index_granularity_bytes`. До версии 19.11, размер гранул ограничивался только настройкой `index_granularity`. Настройка `index_granularity_bytes` улучшает производительность ClickHouse при выборке данных из таблиц с большими (десятки и сотни мегабайтов) строками. Если у вас есть таблицы с большими строками, можно включить эту настройку, чтобы повысить эффективность запросов `SELECT`. + - `use_minimalistic_part_header_in_zookeeper` — Способ хранения заголовков кусков данных в ZooKeeper. Если `use_minimalistic_part_header_in_zookeeper = 1`, то ZooKeeper хранит меньше данных. Подробнее читайте в [описании настройки](../server_settings/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) в разделе "Конфигурационные параметры сервера". + - `min_merge_bytes_to_use_direct_io` — минимальный объем данных при слиянии, необходимый для прямого (небуферизованного) чтения/записи (direct I/O) на диск. При слиянии частей данных ClickHouse вычисляет общий объем хранения всех данных, подлежащих слиянию. Если общий объем хранения всех данных для чтения превышает `min_bytes_to_use_direct_io` байт, тогда ClickHouse использует флаг `O_DIRECT` при чтении данных с диска. Если `min_merge_bytes_to_use_direct_io = 0`, тогда прямой ввод-вывод отключен. Значение по умолчанию: `10 * 1024 * 1024 * 1024` байтов. - - `merge_with_ttl_timeout` - Минимальное время в секундах для повторного выполнения слияний с TTL. По умолчанию - 86400 (1 день). - + - `merge_with_ttl_timeout` — минимальное время в секундах перед повторным слиянием с TTL. По умолчанию — 86400 (1 день). + - `write_final_mark` — включает или отключает запись последней засечки индекса в конце куска данных. По умолчанию — 1. Не отключайте её. + - `storage_policy` — политика хранения данных. Смотрите [Хранение данных таблицы на нескольких блочных устройствах](#table_engine-mergetree-multiple-volumes). + **Пример задания секций** ```sql @@ -126,7 +133,7 @@ MergeTree(EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID) -## Хранение данных +## Хранение данных {#mergetree-data-storage} Таблица состоит из *кусков* данных (data parts), отсортированных по первичному ключу. @@ -134,9 +141,10 @@ MergeTree(EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID) Данные, относящиеся к разным партициям, разбиваются на разные куски. В фоновом режиме ClickHouse выполняет слияния (merge) кусков данных для более эффективного хранения. Куски, относящиеся к разным партициям не объединяются. Механизм слияния не гарантирует, что все строки с одинаковым первичным ключом окажутся в одном куске. -Для каждого куска данных ClickHouse создаёт индексный файл, который содержит значение первичного ключа для каждой индексной строки («засечка»). Номера строк индекса определяются как `n * index_granularity`. Максимальное значение `n` равно целой части деления общего числа строк на `index_granularity`. Для каждого столбца "засечки" также записываются для тех же строк индекса, что и первичный ключ. Эти "засечки" позволяют находить данные непосредственно в столбцах. +Каждый кусок данных логически делится на гранулы. Гранула — это минимальный неделимый набор данных, который ClickHouse считывает при выборке данных. ClickHouse не разбивает строки и значения и гранула всегда содержит целое число строк. Первая строка гранулы помечается значением первичного ключа для этой строки (засечка). Для каждого куска данных ClickHouse создаёт файл с засечками (индексный файл). Для каждого столбца, независимо от того, входит он в первичный ключ или нет, ClickHouse также сохраняет эти же засечки. Засечки используются для поиска данных напрямую в файлах столбцов. + +Размер гранул оганичен настройками движка `index_granularity` и `index_granularity_bytes`. Количество строк в грануле лежит в диапазоне `[1, index_granularity]`, в зависимости от размера строк. Размер гранулы может превышать `index_granularity_bytes` в том случае, когда размер единственной строки в грануле превышает значение настройки. В этом случае, размер гранулы равен размеру строки. -Вы можете использовать одну большую таблицу, постоянно добавляя в неё данные пачками, именно для этого предназначен движок `MergeTree`. ## Первичные ключи и индексы в запросах {#primary-keys-and-indexes-in-queries} @@ -159,9 +167,9 @@ Marks numbers: 0 1 2 3 4 5 6 7 8 Примеры выше показывают, что использование индекса всегда эффективнее, чем full scan. -Разреженный индекс допускает чтение лишних строк. При чтении одного диапазона первичного ключа, может быть прочитано до `index_granularity * 2` лишних строк в каждом блоке данных. В большинстве случаев ClickHouse не теряет производительности при `index_granularity = 8192`. +Разреженный индекс допускает чтение лишних строк. При чтении одного диапазона первичного ключа, может быть прочитано до `index_granularity * 2` лишних строк в каждом блоке данных. -Разреженность индекса позволяет работать даже с очень большим количеством строк в таблицах, поскольку такой индекс всегда помещается в оперативную память компьютера. +Разреженный индекс почти всегда помещаеся в оперативную память и поволяет работать с очень большим количеством строк в таблицах. ClickHouse не требует уникального первичного ключа. Можно вставить много строк с одинаковым первичным ключом. @@ -355,11 +363,13 @@ hasToken | ✗ | ✗ | ✗ | ✔ | ✗ ## TTL для столбцов и таблиц {#table_engine-mergetree-ttl} -Определяет время жизни значений. +Определяет время жизни значений, а также правила перемещения данных на другой диск или том. -Секция `TTL` может быть установлена как для всей таблицы, так и для каждого отдельного столбца. Если установлены оба `TTL`, то ClickHouse использует тот, что истекает раньше. +Секция `TTL` может быть установлена как для всей таблицы, так и для каждого отдельного столбца. Правила `TTL` для таблицы позволяют указать целевые диски или тома для фонового перемещения на них частей данных. -Таблица должна иметь столбец типа [Date](../../data_types/date.md) или [DateTime](../../data_types/datetime.md). Для установки времени жизни данных, следует использовать операцию со столбцом с временем, например: +Выражения должны возвращать тип [Date](../../data_types/date.md) или [DateTime](../../data_types/datetime.md). + +Для задания времени жизни столбца, например: ```sql TTL time_column @@ -414,7 +424,17 @@ ALTER TABLE example_table **TTL таблицы** -Когда некоторые данные в таблице устаревают, ClickHouse удаляет все соответствующие строки. +Для таблицы можно задать одно выражение для устаревания данных, а также несколько выражений, по срабатывании которых данные переместятся на [некоторый диск или том](#table_engine-mergetree-multiple-volumes). Когда некоторые данные в таблице устаревают, ClickHouse удаляет все соответствующие строки. + +```sql +TTL expr [DELETE|TO DISK 'aaa'|TO VOLUME 'bbb'], ... +``` + +За каждым TTL выражением может следовать тип действия, которое выполняется после достижения времени, соответствующего результату TTL выражения: + +- `DELETE` - удалить данные (действие по умолчанию); +- `TO DISK 'aaa'` - переместить данные на диск `aaa`; +- `TO VOLUME 'bbb'` - переместить данные на том `bbb`. Примеры: @@ -427,7 +447,9 @@ CREATE TABLE example_table ENGINE = MergeTree PARTITION BY toYYYYMM(d) ORDER BY d -TTL d + INTERVAL 1 MONTH; +TTL d + INTERVAL 1 MONTH [DELETE], + d + INTERVAL 1 WEEK TO VOLUME 'aaa', + d + INTERVAL 2 WEEK TO DISK 'bbb'; ``` Изменение TTL @@ -445,55 +467,89 @@ ALTER TABLE example_table Если вы выполните запрос `SELECT` между слияниями вы можете получить устаревшие данные. Чтобы избежать этого используйте запрос [OPTIMIZE](../../query_language/misc.md#misc_operations-optimize) перед `SELECT`. -[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/mergetree/) - - ## Хранение данных таблицы на нескольких блочных устройствах {#table_engine-mergetree-multiple-volumes} -### Общее -Данные таблиц семейства MergeTree могут храниться на нескольких блочных устройствах. Это может оказаться полезным, например, при неявном разделении данных одной таблицы на "горячие" и "холодные", когда наиболее свежая часть занимает малый объем и запрашивается регулярно, а большой хвост исторических данных запрашивается редко. При наличии в системе нескольких дисков, "горячая" часть данных может быть размещена на быстрых дисках (NVMe SSDs или даже в памяти), а холодная на более медленных (HDD). +### Введение -Минимальной перемещаемой единицей для MergeTree является кусок (part). Данные одного куска могут находится только на одном диске. Куски могут перемещаться между дисками в фоне, согласно пользовательским настройкам, а также с помощью [ALTER](../../query_language/alter.md#alter_move-partition) запросов. +Движки таблиц семейства `MergeTree` могут хранить данные на нескольких блочных устройствах. Это может оказаться полезным, например, при неявном разделении данных одной таблицы на "горячие" и "холодные". Наиболее свежая часть занимает малый объем и запрашивается регулярно, а большой хвост исторических данных запрашивается редко. При наличии в системе нескольких дисков, "горячая" часть данных может быть размещена на быстрых дисках (например, на NVMe SSD или в памяти), а холодная на более медленных (например, HDD). + +Минимальной перемещаемой единицей для `MergeTree` является кусок данных (data part). Данные одного куска могут находится только на одном диске. Куски могут перемещаться между дисками в фоне, согласно пользовательским настройкам, а также с помощью запросов [ALTER](../../query_language/alter.md#alter_move-partition). ### Термины -* Диск — примонтированное в файловой системе блочное устройство. -* Диск по умолчанию — диск, на котором находится путь, указанный в корне `config.xml` в теге ``. -* Том (Volume) — упорядоченный набор равноценных дисков (схоже с [JBOD](https://ru.wikipedia.org/wiki/JBOD)) -* Политика хранения (StoragePolicy) — множество томов с правилами перемещения данных между ними. -У всех описанных сущностей, при создании, указываются имена, которые будут отражены в системных таблицах [system.storage_policies](../system_tables.md#system_tables-storage_policies) и [system.disks](../system_tables.md#system_tables-disks). Имя политики хранения используется как настройка у таблиц семейства MergeTree. +- Диск — примонтированное в файловой системе блочное устройство. +- Диск по умолчанию — диск, на котором находится путь, указанный в конфигурационной настройке сервера [path](../server_settings/settings.md#server_settings-path). +- Том (Volume) — упорядоченный набор равноценных дисков (схоже с [JBOD](https://ru.wikipedia.org/wiki/JBOD)) +- Политика хранения (StoragePolicy) — множество томов с правилами перемещения данных между ними. + +У всех описанных сущностей при создании указываются имена, можно найти в системных таблицах [system.storage_policies](../system_tables.md#system_tables-storage_policies) и [system.disks](../system_tables.md#system_tables-disks). Имя политики хранения можно указать в настройке `storage_policy` движков таблиц семейства `MergeTree`. ### Конфигурация {#table_engine-mergetree-multiple-volumes_configure} -Диски, тома и политики хранения задаются внутри тега `` в основном файле `config.xml` или в отдельном файле в директории `config.d`. Правила составления данной секции конфигурации имеет следующую структуру: +Диски, тома и политики хранения задаются внутри тега `` в основном файле `config.xml` или в отдельном файле в директории `config.d`. + +Структура конфигурации: ```xml - + /mnt/fast_ssd/clickhouse - - + + /mnt/hdd1/clickhouse 10485760_ - - + + /mnt/hdd2/clickhouse 10485760_ - + ... ``` -Где, +Теги: -* имя диска задается внутри имени тега. -* `path` — путь по которому будут храниться данные сервера (каталоги `data` и `shadow`), должен быть терминирован `/`. -* `keep_free_space_bytes` — размер зарезервированного свободного места на диске. +- `` — имя диска. Имена должны быть разными для всех дисков. +- `path` — путь по которому будут храниться данные сервера (каталоги `data` и `shadow`), должен быть терминирован `/`. +- `keep_free_space_bytes` — размер зарезервированного свободного места на диске. Порядок задания дисков не имеет значения. -Конфигурация политик хранения: +Общий вид конфигурации политик хранения: + +```xml + + + + + disk_name_from_disks_configuration + 1073741824 + + + + + + + 0.2 + + + + + + + +``` + +Тэги: + +- `policy_name_N` — название политики. Названия политик должны быть уникальны. +- `volume_name_N` — название тома. Названия томов должны быть уникальны. +- `disk` — диск, находящийся внутри тома. +- `max_data_part_size_bytes` — максимальный размер куска данных, который может находится на любом из дисков этого тома. +- `move_factor` — доля свободного места, при превышении которого данные начинают перемещаться на следующий том, если он есть (по умолчанию 0.1). + +Примеры конфигураций: ```xml @@ -521,14 +577,9 @@ ALTER TABLE example_table ``` -Где, +В приведенном примере, политика `hdd_in_order` реализует прицип [round-robin](https://ru.wikipedia.org/wiki/Round-robin_(%D0%B0%D0%BB%D0%B3%D0%BE%D1%80%D0%B8%D1%82%D0%BC)). Так как в политике есть всего один том (`single`), то все записи производятся на его диски по круговому циклу. Такая политика может быть полезна при наличии в системе нескольких похожих дисков, но при этом не сконфигурирован RAID. Учтите, что каждый отдельный диск ненадёжен и чтобы не потерять важные данные это необходимо скомпенсировать за счет хранения данных в трёх копиях. -* имя политики и тома задаются внутри имен тегов. -* `disk` — диск, находящийся внутри тома. -* `max_data_part_size_bytes` — максимальный размер куска, который может находится на любом из дисков этого тома. -* `move_factor` — доля свободного места, при превышении которого данные начинают перемещаться на следующий том, если он есть (по умолчанию 0.1). - -В приведенном примере, политика `hdd_in_order` реализует прицип [round-robin](https://ru.wikipedia.org/wiki/Round-robin_(%D0%B0%D0%BB%D0%B3%D0%BE%D1%80%D0%B8%D1%82%D0%BC)). Так как в политике есть всего 1 том (`single`) все записи производятся на его диски по круговому циклу. Такая политика может быть полезна при наличии в системе нескольких похожих дисков. Политика `moving_from_ssd_to_hdd` полезна при наличии в разных типов дисков. В томе `hot` находится один SSD-диск (`fast_ssd`), а также задается ограничение на максимальный размер куска, который может храниться на этом томе (1GB). Все куски такой таблицы больше 1GB будут записываться сразу на том `cold`, в котором содержится один HDD-диск `disk1`. Также, при заполнении диска `fast_ssd` более чем на 80% данные будут переносится на диск `disk1` фоновым процессом. +Если система содержит диски различных типов, то может пригодиться политика `moving_from_ssd_to_hdd`. В томе `hot` находится один SSD-диск (`fast_ssd`), а также задается ограничение на максимальный размер куска, который может храниться на этом томе (1GB). Все куски такой таблицы больше 1GB будут записываться сразу на том `cold`, в котором содержится один HDD-диск `disk1`. Также, при заполнении диска `fast_ssd` более чем на 80% данные будут переносится на диск `disk1` фоновым процессом. Порядок томов в политиках хранения важен, при достижении условий на переполнение тома данные переносятся на следующий. Порядок дисков в томах так же важен, данные пишутся по очереди на каждый из них. @@ -550,12 +601,12 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' ### Особенности работы -В таблицах MergeTree данные попадают на диск несколькими способами: +В таблицах `MergeTree` данные попадают на диск несколькими способами: -* В результате вставки (запрос `INSERT`). -* В фоновых операциях слияний и [мутаций](../../query_language/alter.md#alter-mutations). -* При скачивании данных с другой реплики. -* В результате заморозки партиций [ALTER TABLE ... FREEZE PARTITION](../../query_language/alter.md#alter_freeze-partition). +- В результате вставки (запрос `INSERT`). +- В фоновых операциях слияний и [мутаций](../../query_language/alter.md#alter-mutations). +- При скачивании данных с другой реплики. +- В результате заморозки партиций [ALTER TABLE ... FREEZE PARTITION](../../query_language/alter.md#alter_freeze-partition). Во всех случаях, кроме мутаций и заморозки партиций, при записи куска выбирается том и диск в соответствии с указанной конфигурацией хранилища: @@ -571,3 +622,5 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' Перемещения данных не взаимодействуют с репликацией данных, поэтому на разных репликах одной и той же таблицы могут быть указаны разные политики хранения. После выполнения фоновых слияний или мутаций старые куски не удаляются сразу, а через некоторое время (табличная настройка `old_parts_lifetime`). Также они не перемещаются на другие тома или диски, поэтому до момента удаления они продолжают учитываться при подсчёте занятого дискового пространства. + +[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/mergetree/) diff --git a/docs/ru/operations/table_engines/mysql.md b/docs/ru/operations/table_engines/mysql.md index efd11d7e09b..ef7dd5b9d5d 100644 --- a/docs/ru/operations/table_engines/mysql.md +++ b/docs/ru/operations/table_engines/mysql.md @@ -63,7 +63,7 @@ mysql> select * from test; 1 row in set (0,00 sec) ``` -Таблица в ClickHouse, которая получает данные из таблицы MySQL: +Таблица в ClickHouse, которая получает данные из созданной ранее таблицы MySQL: ```sql CREATE TABLE mysql_table @@ -75,7 +75,7 @@ ENGINE = MySQL('localhost:3306', 'test', 'test', 'bayonet', '123') ``` ```sql -SELECT * FROM mysql_table6 +SELECT * FROM mysql_table ``` ```text diff --git a/docs/ru/operations/table_engines/replication.md b/docs/ru/operations/table_engines/replication.md index 61a5cf3b56f..cd55296cc07 100644 --- a/docs/ru/operations/table_engines/replication.md +++ b/docs/ru/operations/table_engines/replication.md @@ -22,7 +22,14 @@ - `DROP TABLE` удаляет реплику, расположенную на том сервере, где выполняется запрос. - Запрос `RENAME` переименовывает таблицу на одной реплик. Другими словами, реплицируемые таблицы на разных репликах могут называться по-разному. -Чтобы использовать репликацию, укажите в конфигурационном файле адреса ZooKeeper кластера. Пример: +ClickHouse хранит метаинформацию о репликах в [Apache ZooKeeper](https://zookeeper.apache.org). Используйте ZooKeeper 3.4.5 или новее. + +Для использовании репликации, установите параметры в секции [zookeeper](../server_settings/settings.md#server-settings_zookeeper) конфигурации сервера. + + !!! attention "Внимание" + Не пренебрегайте настройками безопасности. ClickHouse поддерживает [ACL схему](https://zookeeper.apache.org/doc/current/zookeeperProgrammers.html#sc_ZooKeeperAccessControl) `digest` подсистемы безопасности ZooKeeper. + +Пример указания адресов кластера ZooKeeper: ```xml @@ -41,8 +48,6 @@ ``` -Используйте ZooKeeper версии 3.4.5 или более новый. - Можно указать любой имеющийся у вас ZooKeeper-кластер - система будет использовать в нём одну директорию для своих данных (директория указывается при создании реплицируемой таблицы). Если в конфигурационном файле не настроен ZooKeeper, то вы не сможете создать реплицируемые таблицы, а уже имеющиеся реплицируемые таблицы будут доступны в режиме только на чтение. diff --git a/docs/ru/query_language/agg_functions/parametric_functions.md b/docs/ru/query_language/agg_functions/parametric_functions.md index 62c5181a42e..0b018d8876c 100644 --- a/docs/ru/query_language/agg_functions/parametric_functions.md +++ b/docs/ru/query_language/agg_functions/parametric_functions.md @@ -71,51 +71,148 @@ FROM В этом случае необходимо помнить, что границы корзин гистограммы не известны. -## sequenceMatch(pattern)(time, cond1, cond2, ...) +## sequenceMatch(pattern)(timestamp, cond1, cond2, ...) {#function-sequencematch} -Сопоставление с образцом для цепочки событий. - -`pattern` - строка, содержащая шаблон для сопоставления. Шаблон похож на регулярное выражение. - -`time` - время события, тип DateTime - -`cond1`, `cond2` ... - от одного до 32 аргументов типа UInt8 - признаков, было ли выполнено некоторое условие для события. - -Функция собирает в оперативке последовательность событий. Затем производит проверку на соответствие этой последовательности шаблону. -Возвращает UInt8 - 0, если шаблон не подходит и 1, если шаблон подходит. - -Пример: `sequenceMatch('(?1).*(?2)')(EventTime, URL LIKE '%company%', URL LIKE '%cart%')` - -- была ли цепочка событий, в которой посещение страницы с адресом, содержащим company было раньше по времени посещения страницы с адресом, содержащим cart. - -Это вырожденный пример. Его можно записать с помощью других агрегатных функций: +Проверяет, содержит ли последовательность событий цепочку, которая соответствует указанному шаблону. ```sql -minIf(EventTime, URL LIKE '%company%') < maxIf(EventTime, URL LIKE '%cart%'). +sequenceMatch(pattern)(timestamp, cond1, cond2, ...) ``` -Но в более сложных случаях, такого решения нет. +!!! warning "Предупреждение" + События, произошедшие в одну и ту же секунду, располагаются в последовательности в неопределенном порядке, что может повлиять на результат работы функции. -Синтаксис шаблонов: -`(?1)` - ссылка на условие (вместо 1 - любой номер); +**Параметры** -`.*` - произвольное количество любых событий; +- `pattern` — строка с шаблоном. Смотрите [Синтаксис шаблонов](#sequence-function-pattern-syntax). -`(?t>=1800)` - условие на время; +- `timestamp` — столбец, содержащий метки времени. Типичный тип данных столбца — `Date` или `DateTime`. Также можно использовать любой из поддержанных типов данных [UInt](../../data_types/int_uint.md). -за указанное время допускается любое количество любых событий; +- `cond1`, `cond2` — условия, описывающие цепочку событий. Тип данных — `UInt8`. Можно использовать до 32 условий. Функция учитывает только те события, которые указаны в условиях. Функция пропускает данные из последовательности, если они не описаны ни в одном из условий. -вместо `>=` могут использоваться операторы `<`, `>`, `<=`; -вместо 1800 может быть любое число; +**Возвращаемые значения** -События, произошедшие в одну секунду, могут оказаться в цепочке в произвольном порядке. От этого может зависеть результат работы функции. +- 1, если цепочка событий, соответствующая шаблону найдена. +- 0, если цепочка событий, соответствующая шаблону не найдена. -## sequenceCount(pattern)(time, cond1, cond2, ...) +Тип: `UInt8`. -Аналогично функции sequenceMatch, но возвращает не факт наличия цепочки событий, а UInt64 - количество найденных цепочек. -Цепочки ищутся без перекрытия. То есть, следующая цепочка может начаться только после окончания предыдущей. + +**Синтаксис шаблонов** + +- `(?N)` — соответствует условию на позиции `N`. Условия пронумерованы по порядку в диапазоне `[1, 32]`. Например, `(?1)` соответствует условию, заданному параметром `cond1`. + +- `.*` — соответствует любому количеству событий. Для этого элемента шаблона не надо задавать условия. + +- `(?t operator value)` — устанавливает время в секундах, которое должно разделять два события. Например, шаблон `(?1)(?t>1800)(?2)` соответствует событиям, которые произошли более чем через 1800 секунд друг от друга. Между этими событиями может находиться произвольное количество любых событий. Операторы могут быть `>=`, `>`, `<`, `<=`. + +**Примеры** + +Пусть таблица `t` содержит следующие данные: + +```text +┌─time─┬─number─┐ +│ 1 │ 1 │ +│ 2 │ 3 │ +│ 3 │ 2 │ +└──────┴────────┘ +``` + +Выполним запрос: + +```sql +SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2) FROM t +``` +```text +┌─sequenceMatch('(?1)(?2)')(time, equals(number, 1), equals(number, 2))─┐ +│ 1 │ +└───────────────────────────────────────────────────────────────────────┘ +``` + +Функция нашла цепочку событий, в которой число 2 следует за числом 1. Число 3 между ними было пропущено, поскольку оно не было использовано ни в одном из условий. + +```sql +SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 3) FROM t +``` +```text +┌─sequenceMatch('(?1)(?2)')(time, equals(number, 1), equals(number, 2), equals(number, 3))─┐ +│ 0 │ +└──────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +В этом случае функция не может найти цепочку событий, соответствующую шаблону, поскольку событие для числа 3 произошло между 1 и 2. Если бы в этом же случае мы бы проверяли условие на событие для числа 4, то цепочка бы соответствовала шаблону. + +```sql +SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 4) FROM t +``` +```text +┌─sequenceMatch('(?1)(?2)')(time, equals(number, 1), equals(number, 2), equals(number, 4))─┐ +│ 1 │ +└──────────────────────────────────────────────────────────────────────────────────────────┘ +``` + + +**Смотрите также** + +- [sequenceCount](#function-sequencecount) + + +## sequenceCount(pattern)(time, cond1, cond2, ...) {#function-sequencecount} + +Вычисляет количество цепочек событий, соответствующих шаблону. Функция обнаруживает только непересекающиеся цепочки событий. Она начитает искать следующую цепочку только после того, как полностью совпала текущая цепочка событий. + +!!! warning "Предупреждение" + События, произошедшие в одну и ту же секунду, располагаются в последовательности в неопределенном порядке, что может повлиять на результат работы функции. + +```sql +sequenceCount(pattern)(timestamp, cond1, cond2, ...) +``` + +**Параметры** + +- `pattern` — строка с шаблоном. Смотрите [Синтаксис шаблонов](#sequence-function-pattern-syntax). + +- `timestamp` — столбец, содержащий метки времени. Типичный тип данных столбца — `Date` или `DateTime`. Также можно использовать любой из поддержанных типов данных [UInt](../../data_types/int_uint.md). + +- `cond1`, `cond2` — условия, описывающие цепочку событий. Тип данных — `UInt8`. Можно использовать до 32 условий. Функция учитывает только те события, которые указаны в условиях. Функция пропускает данные из последовательности, если они не описаны ни в одном из условий. + +**Возвращаемое значение** + +- Число непересекающихся цепочек событий, соответствущих шаблону. + +Тип: `UInt64`. + +**Пример** + +Пусть таблица `t` содержит следующие данные: + +```text +┌─time─┬─number─┐ +│ 1 │ 1 │ +│ 2 │ 3 │ +│ 3 │ 2 │ +│ 4 │ 1 │ +│ 5 │ 3 │ +│ 6 │ 2 │ +└──────┴────────┘ +``` + +Вычислим сколько раз число 2 стоит после числа 1, причем между 1 и 2 могут быть любые числа: + +```sql +SELECT sequenceCount('(?1).*(?2)')(time, number = 1, number = 2) FROM t +``` +```text +┌─sequenceCount('(?1).*(?2)')(time, equals(number, 1), equals(number, 2))─┐ +│ 2 │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +**Смотрите также** + +- [sequenceMatch](#function-sequencematch) ## windowFunnel(window)(timestamp, cond1, cond2, cond3, ...) diff --git a/docs/ru/query_language/agg_functions/reference.md b/docs/ru/query_language/agg_functions/reference.md index 7ef6ef5f500..758d762e26b 100644 --- a/docs/ru/query_language/agg_functions/reference.md +++ b/docs/ru/query_language/agg_functions/reference.md @@ -1087,38 +1087,39 @@ stochasticLinearRegression(1.0, 1.0, 10, 'SGD') Для прогнозирования мы используем функцию [evalMLMethod](../functions/machine_learning_functions.md#machine_learning_methods-evalmlmethod), которая принимает в качестве аргументов состояние и свойства для прогнозирования. -1. Построение модели - Пример запроса: +**1.** Построение модели - ```sql - CREATE TABLE IF NOT EXISTS train_data - ( - param1 Float64, - param2 Float64, - target Float64 - ) ENGINE = Memory; +Пример запроса: - CREATE TABLE your_model ENGINE = Memory AS SELECT - stochasticLinearRegressionState(0.1, 0.0, 5, 'SGD')(target, param1, param2) - AS state FROM train_data; - ``` +```sql +CREATE TABLE IF NOT EXISTS train_data +( + param1 Float64, + param2 Float64, + target Float64 +) ENGINE = Memory; - Здесь нам также нужно вставить данные в таблицу `train_data`. Количество параметров не фиксировано, оно зависит только от количества аргументов, перешедших в `linearRegressionState`. Все они должны быть числовыми значениями. +CREATE TABLE your_model ENGINE = Memory AS SELECT +stochasticLinearRegressionState(0.1, 0.0, 5, 'SGD')(target, param1, param2) +AS state FROM train_data; +``` + +Здесь нам также нужно вставить данные в таблицу `train_data`. Количество параметров не фиксировано, оно зависит только от количества аргументов, перешедших в `linearRegressionState`. Все они должны быть числовыми значениями. Обратите внимание, что столбец с целевым значением (которое мы хотели бы научиться предсказывать) вставляется в качестве первого аргумента. -2. Прогнозирование +**2.** Прогнозирование - После сохранения состояния в таблице мы можем использовать его несколько раз для прогнозирования или смёржить с другими состояниями и создать новые, улучшенные модели. +После сохранения состояния в таблице мы можем использовать его несколько раз для прогнозирования или смёржить с другими состояниями и создать новые, улучшенные модели. - ```sql - WITH (SELECT state FROM your_model) AS model SELECT - evalMLMethod(model, param1, param2) FROM test_data - ``` +```sql +WITH (SELECT state FROM your_model) AS model SELECT +evalMLMethod(model, param1, param2) FROM test_data +``` - Запрос возвращает столбец прогнозируемых значений. Обратите внимание, что первый аргумент `evalMLMethod` это объект `AggregateFunctionState`, далее идут столбцы свойств. +Запрос возвращает столбец прогнозируемых значений. Обратите внимание, что первый аргумент `evalMLMethod` это объект `AggregateFunctionState`, далее идут столбцы свойств. - `test_data` — это таблица, подобная `train_data`, но при этом может не содержать целевое значение. +`test_data` — это таблица, подобная `train_data`, но при этом может не содержать целевое значение. ### Примечания {#agg_functions-stochasticlinearregression-notes} diff --git a/docs/ru/query_language/alter.md b/docs/ru/query_language/alter.md index d8a5d88d899..9cba92f23fd 100644 --- a/docs/ru/query_language/alter.md +++ b/docs/ru/query_language/alter.md @@ -26,10 +26,10 @@ ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN #### ADD COLUMN {#alter_add-column} ```sql -ADD COLUMN [IF NOT EXISTS] name [type] [default_expr] [AFTER name_after] +ADD COLUMN [IF NOT EXISTS] name [type] [default_expr] [codec] [AFTER name_after] ``` -Добавляет в таблицу новый столбец с именем `name`, типом `type` и выражением для умолчания `default_expr` (смотрите раздел [Значения по умолчанию](create.md#create-default-values)). +Добавляет в таблицу новый столбец с именем `name`, типом `type`, [кодеком](create.md#codecs) `codec` и выражением для умолчания `default_expr` (смотрите раздел [Значения по умолчанию](create.md#create-default-values)). Если указано `IF NOT EXISTS`, запрос не будет возвращать ошибку, если столбец уже существует. Если указано `AFTER name_after` (имя другого столбца), то столбец добавляется (в список столбцов таблицы) после указанного. Иначе, столбец добавляется в конец таблицы. Обратите внимание, ClickHouse не позволяет добавлять столбцы в начало таблицы. Для цепочки действий, `name_after` может быть именем столбца, который добавляется в одном из предыдущих действий. @@ -188,7 +188,8 @@ ALTER TABLE [db].name DROP CONSTRAINT constraint_name; - [DETACH PARTITION](#alter_detach-partition) – перенести партицию в директорию `detached`; - [DROP PARTITION](#alter_drop-partition) – удалить партицию; - [ATTACH PARTITION|PART](#alter_attach-partition) – добавить партицию/кусок в таблицу из директории `detached`; -- [REPLACE PARTITION](#alter_replace-partition) – скопировать партицию из другой таблицы; +- [ATTACH PARTITION FROM](#alter_attach-partition-from) – скопировать партицию из другой таблицы; +- [REPLACE PARTITION](#alter_replace-partition) – скопировать партицию из другой таблицы с заменой; - [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) – удалить все значения в столбце для заданной партиции; - [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) - очистить построенные вторичные индексы для заданной партиции; - [FREEZE PARTITION](#alter_freeze-partition) – создать резервную копию партиции; @@ -251,17 +252,33 @@ ALTER TABLE visits ATTACH PART 201901_2_2_0; Как корректно задать имя партиции или куска, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). -Этот запрос реплицируется. Каждая реплика проверяет, есть ли данные в директории `detached`. Если данные есть, то запрос проверяет их целостность и соответствие данным на сервере-инициаторе запроса. В случае успеха данные добавляются в таблицу. В противном случае, реплика загружает данные с реплики-инициатора запроса или с другой реплики, на которой эти данные уже добавлены. +Этот запрос реплицируется. Реплика-иницатор проверяет, есть ли данные в директории `detached`. Если данные есть, то запрос проверяет их целостность. В случае успеха данные добавляются в таблицу. Все остальные реплики загружают данные с реплики-инициатора запроса. Это означает, что вы можете разместить данные в директории `detached` на одной реплике и с помощью запроса `ALTER ... ATTACH` добавить их в таблицу на всех репликах. +#### ATTACH PARTITION FROM {#alter_attach-partition-from} + +```sql +ALTER TABLE table2 ATTACH PARTITION partition_expr FROM table1 +``` + +Копирует партицию из таблицы `table1` в таблицу `table2` и добавляет к существующим данным `table2`. Данные из `table1` не удаляются. + +Следует иметь в виду: + +- Таблицы должны иметь одинаковую структуру. +- Для таблиц должен быть задан одинаковый ключ партиционирования. + +Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). + + #### REPLACE PARTITION {#alter_replace-partition} ```sql ALTER TABLE table2 REPLACE PARTITION partition_expr FROM table1 ``` -Копирует партицию из таблицы `table1` в таблицу `table2`. Данные из `table1` не удаляются. +Копирует партицию из таблицы `table1` в таблицу `table2` с заменой существующих данных в `table2`. Данные из `table1` не удаляются. Следует иметь в виду: @@ -300,7 +317,7 @@ ALTER TABLE table_name FREEZE [PARTITION partition_expr] Создаёт резервную копию для заданной партиции. Если выражение `PARTITION` опущено, резервные копии будут созданы для всех партиций. -!!! note +!!! note "Примечание" Создание резервной копии не требует остановки сервера. Для таблиц старого стиля имя партиций можно задавать в виде префикса (например, '2019'). В этом случае резервные копии будут созданы для всех соответствующих партиций. Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). @@ -361,17 +378,16 @@ ALTER TABLE users ATTACH PARTITION 201902; #### MOVE PARTITION|PART {#alter_move-partition} +Перемещает партицию или кусок данных на другой том или диск для таблиц с движком `MergeTree`. Смотрите [Хранение данных таблицы на нескольких блочных устройствах](../operations/table_engines/mergetree.md#table_engine-mergetree-multiple-volumes). + ```sql ALTER TABLE table_name MOVE PARTITION|PART partition_expr TO DISK|VOLUME 'disk_name' ``` +Запрос `ALTER TABLE t MOVE`: -Перемещает партицию или кусок на другой том или диск. Запрос работает только для движков семейства MergeTree. Подробнее о хранении данных на разных дисках читайте в разделе [Хранение данных таблицы на нескольких блочных устройствах](../operations/table_engines/mergetree.md#table_engine-mergetree-multiple-volumes). - -Следует иметь ввиду: - -- Запрос `ALTER TABLE t MOVE` не реплицируется, т.к. на разных репликах могут быть различные конфигурации политик хранения. -- Запрос `ALTER TABLE t MOVE` будет возвращать ошибку, если указан несуществующий том или диск, а также в случае невыполнения условий перемещения данных, которые указаны в конфигурации политики хранения. -- Запрос `ALTER TABLE t MOVE` может возвращать ошибку в случае, когда перемещаемые данные уже оказались перемещены в результате фонового процесса, конкурентного запроса `ALTER TABLE t MOVE` или как часть результата фоновой операции слияния. В данном случае никаких дополнительных действий от пользователя не требуется. +- Не реплицируется, т.к. на разных репликах могут быть различные конфигурации политик хранения. +- Возвращает ошибку, если указан несконфигурированный том или диск. Ошибка также возвращается в случае невыполнения условий перемещения данных, которые указаны в конфигурации политики хранения. +- Может возвращать ошибку в случае, когда перемещаемые данные уже оказались перемещены в результате фонового процесса, конкурентного запроса `ALTER TABLE t MOVE` или как часть результата фоновой операции слияния. В данном случае никаких дополнительных действий от пользователя не требуется. Примеры: @@ -410,9 +426,7 @@ OPTIMIZE TABLE table_not_partitioned PARTITION tuple() FINAL; ### Мутации {#alter-mutations} -Мутации - разновидность запроса ALTER, позволяющая изменять или удалять данные в таблице. В отличие от стандартных запросов `DELETE` и `UPDATE`, рассчитанных на точечное изменение данных, область применения мутаций - достаточно тяжёлые изменения, затрагивающие много строк в таблице. - -Функциональность находится в состоянии beta и доступна начиная с версии 1.1.54388. Реализована поддержка \*MergeTree таблиц (с репликацией и без). +Мутации - разновидность запроса ALTER, позволяющая изменять или удалять данные в таблице. В отличие от стандартных запросов `DELETE` и `UPDATE`, рассчитанных на точечное изменение данных, область применения мутаций - достаточно тяжёлые изменения, затрагивающие много строк в таблице. Поддержана для движков таблиц семейства `MergeTree`, в том числе для движков с репликацией. Конвертировать существующие таблицы для работы с мутациями не нужно. Но после применения первой мутации формат данных таблицы становится несовместимым с предыдущими версиями и откатиться на предыдущую версию уже не получится. @@ -422,13 +436,13 @@ OPTIMIZE TABLE table_not_partitioned PARTITION tuple() FINAL; ALTER TABLE [db.]table DELETE WHERE filter_expr ``` -Выражение `filter_expr` должно иметь тип UInt8. Запрос удаляет строки таблицы, для которых это выражение принимает ненулевое значение. +Выражение `filter_expr` должно иметь тип `UInt8`. Запрос удаляет строки таблицы, для которых это выражение принимает ненулевое значение. ```sql ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] WHERE filter_expr ``` -Команда доступна начиная с версии 18.12.14. Выражение `filter_expr` должно иметь тип UInt8. Запрос изменяет значение указанных столбцов на вычисленное значение соответствующих выражений в каждой строке, для которой `filter_expr` принимает ненулевое значение. Вычисленные значения преобразуются к типу столбца с помощью оператора `CAST`. Изменение столбцов, которые используются при вычислении первичного ключа или ключа партиционирования, не поддерживается. +Выражение `filter_expr` должно иметь тип `UInt8`. Запрос изменяет значение указанных столбцов на вычисленное значение соответствующих выражений в каждой строке, для которой `filter_expr` принимает ненулевое значение. Вычисленные значения преобразуются к типу столбца с помощью оператора `CAST`. Изменение столбцов, которые используются при вычислении первичного ключа или ключа партиционирования, не поддерживается. ```sql ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name diff --git a/docs/ru/query_language/create.md b/docs/ru/query_language/create.md index 71fe71abbf2..cf25e3a2d39 100644 --- a/docs/ru/query_language/create.md +++ b/docs/ru/query_language/create.md @@ -125,7 +125,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] Определяет время хранения значений. Может быть указано только для таблиц семейства MergeTree. Подробнее смотрите в [TTL для столбцов и таблиц](../operations/table_engines/mergetree.md#table_engine-mergetree-ttl). -### Кодеки сжатия столбцов +### Кодеки сжатия столбцов {#codecs} По умолчанию, ClickHouse применяет к столбцу метод сжатия, определённый в [конфигурации сервера](../operations/server_settings/settings.md#compression). Кроме этого, можно задать метод сжатия для каждого отдельного столбца в запросе `CREATE TABLE`. @@ -194,16 +194,17 @@ ENGINE = MergeTree() ClickHouse поддерживает временные таблицы со следующими характеристиками: -- временные таблицы исчезают после завершения сессии; в том числе, при обрыве соединения; +- Временные таблицы исчезают после завершения сессии, в том числе при обрыве соединения. - Временная таблица использует только модуль памяти. -- Невозможно указать базу данных для временной таблицы. Временные таблицы создается вне баз данных. -- если временная таблица имеет то же имя, что и некоторая другая, то, при упоминании в запросе без указания БД, будет использована временная таблица; -- при распределённой обработке запроса, используемые в запросе временные таблицы, передаются на удалённые серверы. +- Невозможно указать базу данных для временной таблицы. Она создается вне баз данных. +- Невозможно создать временную таблицу распределнным DDL запросом на всех серверах кластера (с опцией `ON CLUSTER`): такая таблица существует только в рамках существующей сессии. +- Если временная таблица имеет то же имя, что и некоторая другая, то, при упоминании в запросе без указания БД, будет использована временная таблица. +- При распределённой обработке запроса, используемые в запросе временные таблицы, передаются на удалённые серверы. Чтобы создать временную таблицу, используйте следующий синтаксис: ```sql -CREATE TEMPORARY TABLE [IF NOT EXISTS] table_name [ON CLUSTER cluster] +CREATE TEMPORARY TABLE [IF NOT EXISTS] table_name ( name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], @@ -213,6 +214,8 @@ CREATE TEMPORARY TABLE [IF NOT EXISTS] table_name [ON CLUSTER cluster] В большинстве случаев, временные таблицы создаются не вручную, а при использовании внешних данных для запроса, или при распределённом `(GLOBAL) IN`. Подробнее см. соответствующие разделы +Вместо временных можно использовать обычные таблицы с [ENGINE = Memory](../operations/table_engines/memory.md). + ## Распределенные DDL запросы (секция ON CLUSTER) Запросы `CREATE`, `DROP`, `ALTER`, `RENAME` поддерживают возможность распределенного выполнения на кластере. @@ -223,7 +226,7 @@ CREATE TABLE IF NOT EXISTS all_hits ON CLUSTER cluster (p Date, i Int32) ENGINE ``` Для корректного выполнения таких запросов необходимо на каждом хосте иметь одинаковое определение кластера (для упрощения синхронизации конфигов можете использовать подстановки из ZooKeeper). Также необходимо подключение к ZooKeeper серверам. -Локальная версия запроса в конечном итоге будет выполнена на каждом хосте кластера, даже если некоторые хосты в данный момент не доступны. Гарантируется упорядоченность выполнения запросов в рамках одного хоста. Для реплицированных таблиц не поддерживаются запросы `ALTER`. +Локальная версия запроса в конечном итоге будет выполнена на каждом хосте кластера, даже если некоторые хосты в данный момент не доступны. Гарантируется упорядоченность выполнения запросов в рамках одного хоста. ## CREATE VIEW diff --git a/docs/ru/query_language/dicts/external_dicts_dict_layout.md b/docs/ru/query_language/dicts/external_dicts_dict_layout.md index e038a5c0610..7d6919347ab 100644 --- a/docs/ru/query_language/dicts/external_dicts_dict_layout.md +++ b/docs/ru/query_language/dicts/external_dicts_dict_layout.md @@ -47,7 +47,7 @@ ### flat -Словарь полностью хранится в оперативной памяти в виде плоских массивов. Объем памяти, занимаемой словарем? пропорционален размеру самого большого (по размеру) ключа. +Словарь полностью хранится в оперативной памяти в виде плоских массивов. Объем памяти, занимаемой словарём пропорционален размеру самого большого по размеру ключа. Ключ словаря имеет тип `UInt64` и его величина ограничена 500 000. Если при создании словаря обнаружен ключ больше, то ClickHouse бросает исключение и не создает словарь. diff --git a/docs/ru/query_language/functions/array_functions.md b/docs/ru/query_language/functions/array_functions.md index 93c75ac3525..6e0f0fa174d 100644 --- a/docs/ru/query_language/functions/array_functions.md +++ b/docs/ru/query_language/functions/array_functions.md @@ -384,7 +384,7 @@ arrayPushFront(array, single_value) **Пример** ```sql -SELECT arrayPushBack(['b'], 'a') AS res +SELECT arrayPushFront(['b'], 'a') AS res ``` ```text @@ -778,6 +778,22 @@ SELECT arrayReduce('uniqUpTo(3)', [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) └─────────────────────────────────────────────────────────────┘ ``` +## arrayFlatten(arr) {#array_functions-arrayflatten} + +Функция `arrayFlatten` (или `flatten`) соеденит вложенные массивы и вернет массив из их элементов. + +Пример: + +```sql +SELECT arrayFlatten([[1, 2, 3], [4, 5]]) +``` + +```text +┌─arrayFlatten([[1, 2, 3], [4, 5]])─┐ +│ [1,2,3,4,5] │ +└───────────────────────────────────┘ +``` + ## arrayReverse(arr) {#array_functions-arrayreverse} Возвращает массив того же размера, что и исходный массив, содержащий элементы в обратном порядке. @@ -798,4 +814,40 @@ SELECT arrayReverse([1, 2, 3]) Синоним для ["arrayReverse"](#array_functions-arrayreverse) +## arrayCompact {#arraycompact} + +Удаляет дубликаты из массива. Порядок результирующих значений определяется порядком в исходном массиве. + +**Синтаксис** + +```sql +arrayCompact(arr) +``` + +**Параметры** + +`arr` — [Массив](../../data_types/array.md) для обхода. + +**Возвращаемое значение** + +Массив без дубликатов. + +Тип: `Array`. + +**Пример** + +Запрос: + +```sql +SELECT arrayCompact([1, 1, nan, nan, 2, 3, 3, 3]) +``` + +Ответ: + +```text +┌─arrayCompact([1, 1, nan, nan, 2, 3, 3, 3])─┐ +│ [1,nan,nan,2,3] │ +└────────────────────────────────────────────┘ +``` + [Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/array_functions/) diff --git a/docs/ru/query_language/functions/bit_functions.md b/docs/ru/query_language/functions/bit_functions.md index 7ba32ad6ba8..e4af747fa47 100644 --- a/docs/ru/query_language/functions/bit_functions.md +++ b/docs/ru/query_language/functions/bit_functions.md @@ -16,4 +16,183 @@ ## bitShiftRight(a, b) +## bitTest {#bittest} + +Принимает любое целое число и конвертирует его в [двоичное число](https://en.wikipedia.org/wiki/Binary_number), возвращает значение бита в указанной позиции. Отсчет начинается с 0 справа налево. + +**Синтаксис** + +```sql +SELECT bitTest(number, index) +``` + +**Параметры** + +- `number` – целое число. +- `index` – position of bit. + +**Возвращаемое значение** + +Возвращает значение бита в указанной позиции. + +Тип: `UInt8`. + +**Пример** + +Например, число 43 в двоичной системе счисления равно: 101011. + +Запрос: + +```sql +SELECT bitTest(43, 1) +``` + +Ответ: + +```text +┌─bitTest(43, 1)─┐ +│ 1 │ +└────────────────┘ +``` + +Другой пример: + +Запрос: + +```sql +SELECT bitTest(43, 2) +``` + +Ответ: + +```text +┌─bitTest(43, 2)─┐ +│ 0 │ +└────────────────┘ +``` + +## bitTestAll {#bittestall} + +Возвращает результат [логической конъюнкции](https://en.wikipedia.org/wiki/Logical_conjunction) (оператор AND) всех битов в указанных позициях. Отсчет начинается с 0 справа налево. + +Бинарная конъюнкция: + +0 AND 0 = 0 +0 AND 1 = 0 +1 AND 0 = 0 +1 AND 1 = 1 + +**Синтаксис** + +```sql +SELECT bitTestAll(number, index1, index2, index3, index4, ...) +``` + +**Параметры** + +- `number` – целое число. +- `index1`, `index2`, `index3`, `index4` – позиция бита. Например, конъюнкция для набора позиций `index1`, `index2`, `index3`, `index4` является истинной, если все его позиции истинны `index1` ⋀ `index2` ⋀ `index3` ⋀ `index4`. + +**Возвращаемое значение** + +Возвращает результат логической конъюнкции. + +Тип: `UInt8`. + +**Пример** + +Например, число 43 в двоичной системе счисления равно: 101011. + +Запрос: + +```sql +SELECT bitTestAll(43, 0, 1, 3, 5) +``` + +Ответ: + +```text +┌─bitTestAll(43, 0, 1, 3, 5)─┐ +│ 1 │ +└────────────────────────────┘ +``` + +Другой пример: + +Запрос: + +```sql +SELECT bitTestAll(43, 0, 1, 3, 5, 2) +``` + +Ответ: + +```text +┌─bitTestAll(43, 0, 1, 3, 5, 2)─┐ +│ 0 │ +└───────────────────────────────┘ +``` + +## bitTestAny {#bittestany} + +Возвращает результат [логической дизъюнкции](https://en.wikipedia.org/wiki/Logical_disjunction) (оператор OR) всех битов в указанных позициях. Отсчет начинается с 0 справа налево. + +Бинарная дизъюнкция: + +0 OR 0 = 0 +0 OR 1 = 1 +1 OR 0 = 1 +1 OR 1 = 1 + +**Синтаксис** + +```sql +SELECT bitTestAny(number, index1, index2, index3, index4, ...) +``` + +**Параметры** + +- `number` – целое число. +- `index1`, `index2`, `index3`, `index4` – позиции бита. + +**Возвращаемое значение** + +Возвращает результат логической дизъюнкции. + +Тип: `UInt8`. + +**Пример** + +Например, число 43 в двоичной системе счисления равно: 101011. + +Запрос: + +```sql +SELECT bitTestAny(43, 0, 2) +``` + +Ответ: + +```text +┌─bitTestAny(43, 0, 2)─┐ +│ 1 │ +└──────────────────────┘ +``` + +Другой пример: + +Запрос: + +```sql +SELECT bitTestAny(43, 4, 2) +``` + +Ответ: + +```text +┌─bitTestAny(43, 4, 2)─┐ +│ 0 │ +└──────────────────────┘ +``` + [Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/bit_functions/) diff --git a/docs/ru/query_language/functions/date_time_functions.md b/docs/ru/query_language/functions/date_time_functions.md index 432bf56652b..11156285434 100644 --- a/docs/ru/query_language/functions/date_time_functions.md +++ b/docs/ru/query_language/functions/date_time_functions.md @@ -145,7 +145,7 @@ SELECT | ----------- | -------- | --------------- | |%C|номер года, поделённый на 100 (00-99)|20 |%d|день месяца, с ведущим нулём (01-31)|02 -|%D|короткая запись %m/%d/%y|01/02/2018| +|%D|короткая запись %m/%d/%y|01/02/18| |%e|день месяца, с ведущим пробелом ( 1-31)| 2| |%F|короткая запись %Y-%m-%d|2018-01-02 |%H|час в 24-часовом формате (00-23)|22| diff --git a/docs/ru/query_language/functions/encoding_functions.md b/docs/ru/query_language/functions/encoding_functions.md index c3825cd22af..9c7737e5d14 100644 --- a/docs/ru/query_language/functions/encoding_functions.md +++ b/docs/ru/query_language/functions/encoding_functions.md @@ -2,7 +2,7 @@ ## hex -Принимает аргументы типов: `String`, `unsigned integer`, `Date`, or `DateTime`. Возвращает строку, содержащую шестнадцатеричное представление аргумента. Используются заглавные буквы `A-F`. Не используются префиксы `0x` и суффиксы `h`. Для строк просто все байты кодируются в виде двух шестнадцатеричных цифр. Числа выводятся в big endian ("человеческом") формате. Для чисел вырезаются старшие нули, но только по целым байтам. Например, `hex(1) = '01'`. `Date` кодируется как число дней с начала unix-эпохи. `DateTime` кодируются как число секунд с начала unix-эпохи. +Принимает аргументы типов: `String`, `unsigned integer`, `float`, `decimal`, `Date`, or `DateTime`. Возвращает строку, содержащую шестнадцатеричное представление аргумента. Используются заглавные буквы `A-F`. Не используются префиксы `0x` и суффиксы `h`. Для строк просто все байты кодируются в виде двух шестнадцатеричных цифр. Числа выводятся в big endian ("человеческом") формате. Для чисел вырезаются старшие нули, но только по целым байтам. Например, `hex(1) = '01'`. `Date` кодируется как число дней с начала unix-эпохи. `DateTime` кодируются как число секунд с начала unix-эпохи. `float` и `decimal` кодируются как их шестнадцатеричное представление в памяти. ## unhex(str) Принимает строку, содержащую произвольное количество шестнадцатеричных цифр, и возвращает строку, содержащую соответствующие байты. Поддерживаются как строчные, так и заглавные буквы A-F. Число шестнадцатеричных цифр не обязано быть чётным. Если оно нечётное - последняя цифра интерпретируется как младшая половинка байта 00-0F. Если строка-аргумент содержит что-либо кроме шестнадцатеричных цифр, то будет возвращён какой-либо implementation-defined результат (не кидается исключение). diff --git a/docs/ru/query_language/functions/geo.md b/docs/ru/query_language/functions/geo.md index 74ea1e12219..867f90765e9 100644 --- a/docs/ru/query_language/functions/geo.md +++ b/docs/ru/query_language/functions/geo.md @@ -35,6 +35,38 @@ SELECT greatCircleDistance(55.755831, 37.617673, -55.755831, -37.617673) └───────────────────────────────────────────────────────────────────┘ ``` +## greatCircleAngle + +Вычисляет угловое расстояние на сфере по [формуле большого круга](https://en.wikipedia.org/wiki/Great-circle_distance). + +```sql +greatCircleDistance(lon1Deg, lat1Deg, lon2Deg, lat2Deg) +``` + +**Входные параметры** + +- `lon1Deg` — долгота первой точки в градусах. +- `lat1Deg` — широта первой точки в градусах. +- `lon2Deg` — долгота второй точки в градусах. +- `lat2Deg` — широта второй точки в градусах. + +**Возвращаемое значение** + +Длина дуги большого круга между двумя точками в градусах. + +**Пример** + +```sql +SELECT greatCircleAngle(0, 0, 45, 0) AS arc +``` + +```text +┌─arc─┐ +│ 45 │ +└─────┘ +``` + + ## pointInEllipses Проверяет, принадлежит ли точка хотя бы одному из эллипсов. @@ -157,6 +189,121 @@ SELECT geohashDecode('ezs42') AS res └─────────────────────────────────┘ ``` +## h3IsValid {#h3isvalid} + +Проверяет корректность H3-индекса. + +```sql +h3IsValid(h3index) +``` + +**Входные значения** + +- `h3index` — идентификатор шестиугольника. Тип данных — [UInt64](../../data_types/int_uint.md). + +**Возвращаемые значения** + +- 0 — число не является H3-индексом +- 1 — число является H3-индексом + +Тип — [UInt8](../../data_types/int_uint.md). + +**Пример** + +```sql +SELECT h3IsValid(630814730351855103) as h3IsValid +``` +```text +┌─h3IsValid─┐ +│ 1 │ +└───────────┘ +``` + +## h3GetResolution + +Извлекает разрешение H3-индекса. + +```sql +h3GetResolution(h3index) +``` + +**Входные значения** + +- `h3index` — идентификатор шестиугольника. Тип данных — [UInt64](../../data_types/int_uint.md). + +**Возвращаемые значения** + +- Разрешение сетки, от 0 до 15. +- Для несуществующего идентификатора может быть возвращено произвольное значение, используйте [h3IsValid](#h3isvalid) для проверки идентификаторов + +Тип — [UInt8](../../data_types/int_uint.md). + +**Пример** + +```sql +SELECT h3GetResolution(639821929606596015) as resolution +``` +```text +┌─resolution─┐ +│ 14 │ +└────────────┘ +``` + +## h3EdgeAngle + +Информирует о среднем размере стороны шестигранника H3 в градусах + +```sql +h3EdgeAngle(resolution) +``` + +**Входные значения** + +- `resolution` — требуемое разрешение индекса. Тип данных — [UInt8](../../data_types/int_uint.md). Диапазон возможных значений — `[0, 15]`. + +**Возвращаемые значения** + +Средняя длина стороны многоугольника H3 в градусах, тип — [Float64](../../data_types/float.md). + +**Пример** + +```sql +SELECT h3EdgeAngle(10) as edgeAngle +``` +```text +┌─────────h3EdgeAngle(10)─┐ +│ 0.0005927224846720883 │ +└───────────────────────┘ +``` + + +## h3EdgeLengthM + +Информирует о среднем размере стороны шестигранника H3 в метрах + +```sql +h3EdgeLengthM(resolution) +``` + +**Входные значения** + +- `resolution` — требуемое разрешение индекса. Тип данных — [UInt8](../../data_types/int_uint.md). Диапазон возможных значений — `[0, 15]`. + +**Возвращаемые значения** + +Средняя длина стороны многоугольника H3 в метрах, тип — [Float64](../../data_types/float.md). + +**Пример** + +```sql +SELECT h3EdgeLengthM(15) as edgeLengthM +``` +```text +┌─edgeLengthM─┐ +│ 0.509713273 │ +└─────────────┘ +``` + ## geoToH3 Получает H3 индекс точки `(lon, lat)` с заданным разрешением @@ -189,4 +336,38 @@ SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index └────────────────────┘ ``` +## h3kRing + +Возвращает H3-индексы шестиугольников в радиусе `k` от данного в произвольном порядке + +```sql +h3kRing(h3index, k) +``` +**Входные значения** + +- `h3index` — идентификатор шестиугольника. Тип данных — [UInt64](../../data_types/int_uint.md). +- `k` — радиус. Тип данных — [целое число](../../data_types/int_uint.md) + +**Возвращаемые значения** + +[Массив](../../data_types/array.md) из H3-индексов типа [UInt64](../../data_types/int_uint.md). + +**Пример** + +```sql +SELECT arrayJoin(h3kRing(644325529233966508, 1)) AS h3index +``` +```text +┌────────────h3index─┐ +│ 644325529233966508 │ +│ 644325529233966497 │ +│ 644325529233966510 │ +│ 644325529233966504 │ +│ 644325529233966509 │ +│ 644325529233966355 │ +│ 644325529233966354 │ +└────────────────────┘ +``` + + [Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/geo/) diff --git a/docs/ru/query_language/functions/hash_functions.md b/docs/ru/query_language/functions/hash_functions.md index f7d2237a071..47384e78565 100644 --- a/docs/ru/query_language/functions/hash_functions.md +++ b/docs/ru/query_language/functions/hash_functions.md @@ -207,6 +207,44 @@ SELECT javaHash('Hello, world!'); └───────────────────────────┘ ``` +## javaHashUTF16LE {#javahashutf16le} + +Вычисляет [JavaHash](http://hg.openjdk.java.net/jdk8u/jdk8u/jdk/file/478a4add975b/src/share/classes/java/lang/String.java#l1452) от строки, при допущении, что строка представлена в кодировке `UTF-16LE`. + +**Синтаксис** + +```sql +javaHashUTF16LE(stringUtf16le) +``` + +**Параметры** + +- `stringUtf16le` — строка в `UTF-16LE`. + +**Возвращаемое значение** + +Хэш-значение типа `Int32`. + +Тип: `javaHash`. + +**Пример** + +Верный запрос для строки кодированной в `UTF-16LE`. + +Запрос: + +```sql +SELECT javaHashUTF16LE(convertCharset('test', 'utf-8', 'utf-16le')) +``` + +Ответ: + +```text +┌─javaHashUTF16LE(convertCharset('test', 'utf-8', 'utf-16le'))─┐ +│ 3556498 │ +└──────────────────────────────────────────────────────────────┘ +``` + ## hiveHash {#hash_functions-hivehash} Вычисляет `HiveHash` от строки. diff --git a/docs/ru/query_language/functions/json_functions.md b/docs/ru/query_language/functions/json_functions.md index 49f575f4b78..9269493473b 100644 --- a/docs/ru/query_language/functions/json_functions.md +++ b/docs/ru/query_language/functions/json_functions.md @@ -199,9 +199,9 @@ SELECT JSONExtractKeysAndValues('{"x": {"a": 5, "b": 7, "c": 11}}', 'x', 'Int8') ## JSONExtractRaw(json[, indices_or_keys]...) -Возвращает часть JSON. +Возвращает часть JSON в виде строки, содержащей неразобранную подстроку. -Если значение не существует или имеет неверный тип, то возвращается пустая строка. +Если значение не существует, то возвращается пустая строка. Пример: @@ -209,4 +209,16 @@ SELECT JSONExtractKeysAndValues('{"x": {"a": 5, "b": 7, "c": 11}}', 'x', 'Int8') SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = '[-100, 200.0, 300]' ``` +## JSONExtractArrayRaw(json[, indices_or_keys]...) + +Возвращает массив из элементов JSON массива, каждый из которых представлен в виде строки с неразобранными подстроками из JSON. + +Если значение не существует или не является массивом, то возвращается пустой массив. + +Пример: + +```sql +SELECT JSONExtractArrayRaw('{"a": "hello", "b": [-100, 200.0, "hello"]}', 'b') = ['-100', '200.0', '"hello"']' +``` + [Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/json_functions/) diff --git a/docs/ru/query_language/functions/other_functions.md b/docs/ru/query_language/functions/other_functions.md index a7e6051e541..da47839d3ca 100644 --- a/docs/ru/query_language/functions/other_functions.md +++ b/docs/ru/query_language/functions/other_functions.md @@ -4,6 +4,40 @@ Возвращает строку - имя хоста, на котором эта функция была выполнена. При распределённой обработке запроса, это будет имя хоста удалённого сервера, если функция выполняется на удалённом сервере. +## FQDN {#fqdn} + +Возвращает полное имя домена. + +**Синтаксис** + +```sql +fqdn(); +``` + +Эта функция регистронезависимая. + +**Возвращаемое значение** + +- Полное имя домена. + +Тип: `String`. + +**Пример** + +Запрос: + +```sql +SELECT FQDN(); +``` + +Ответ: + +```text +┌─FQDN()──────────────────────────┐ +│ clickhouse.ru-central1.internal │ +└─────────────────────────────────┘ +``` + ## basename Извлекает конечную часть строки после последнего слэша или бэкслэша. Функция часто используется для извлечения имени файла из пути. diff --git a/docs/ru/query_language/functions/string_functions.md b/docs/ru/query_language/functions/string_functions.md index e6753247ea9..2169cb794e0 100644 --- a/docs/ru/query_language/functions/string_functions.md +++ b/docs/ru/query_language/functions/string_functions.md @@ -66,6 +66,42 @@ SELECT toValidUTF8('\x61\xF0\x80\x80\x80b') └───────────────────────┘ ``` +## repeat {#repeat} + +Повторяет строку определенное количество раз и объединяет повторяемые значения в одну строку. + +**Синтаксис** + +```sql +repeat(s, n) +``` + +**Параметры** + +- `s` — Строка для повторения. [String](../../data_types/string.md). +- `n` — Количество повторов. [UInt](../../data_types/int_uint.md). + +**Возвращаемое значение** + +Строка, состоящая из повторений `n` раз исходной строки `s`. Если `n` < 1, то функция вернет пустую строку. + +Тип: `String`. + +**Пример** + +Запрос: + +```sql +SELECT repeat('abc', 10) +``` + +Ответ: + +```text +┌─repeat('abc', 10)──────────────┐ +│ abcabcabcabcabcabcabcabcabcabc │ +└────────────────────────────────┘ +``` ## reverse Разворачивает строку (как последовательность байт). @@ -153,6 +189,120 @@ SELECT startsWith('Hello, world!', 'He'); └───────────────────────────────────┘ ``` +## trimLeft {#trimleft} + +Удаляет все последовательные вхождения обычных пробелов (32 символ ASCII) с левого конца строки. Не удаляет другие виды пробелов (табуляция, пробел без разрыва и т. д.). + +**Синтаксис** + +```sql +trimLeft() +``` + +Алиас: `ltrim`. + +**Параметры** + +- `string` — строка для обрезки. [String](../../data_types/string.md). + +**Возвращаемое значение** + +Исходную строку без общих пробельных символов слева. + +Тип: `String`. + +**Пример** + +Запрос: + +```sql +SELECT trimLeft(' Hello, world! ') +``` + +Ответ: + +```text +┌─trimLeft(' Hello, world! ')─┐ +│ Hello, world! │ +└─────────────────────────────────────┘ +``` + +## trimRight {#trimright} + +Удаляет все последовательные вхождения обычных пробелов (32 символ ASCII) с правого конца строки. Не удаляет другие виды пробелов (табуляция, пробел без разрыва и т. д.). + +**Синтаксис** + +```sql +trimRight() +``` + +Алиас: `rtrim`. + +**Параметры** + +- `string` — строка для обрезки. [String](../../data_types/string.md). + +**Возвращаемое значение** + +Исходную строку без общих пробельных символов справа. + +Тип: `String`. + +**Пример** + +Запрос: + +```sql +SELECT trimRight(' Hello, world! ') +``` + +Ответ: + +```text +┌─trimRight(' Hello, world! ')─┐ +│ Hello, world! │ +└──────────────────────────────────────┘ +``` + +## trimBoth {#trimboth} + +Удаляет все последовательные вхождения обычных пробелов (32 символ ASCII) с обоих концов строки. Не удаляет другие виды пробелов (табуляция, пробел без разрыва и т. д.). + +**Синтаксис** + +```sql +trimBoth() +``` + +Алиас: `trim`. + +**Параметры** + +- `string` — строка для обрезки. [String](../../data_types/string.md). + +**Возвращаемое значение** + +Исходную строку без общих пробельных символов с обоих концов строки. + +Тип: `String`. + +**Пример** + +Запрос: + +```sql +SELECT trimBoth(' Hello, world! ') +``` + +Ответ: + +```text +┌─trimBoth(' Hello, world! ')─┐ +│ Hello, world! │ +└─────────────────────────────────────┘ +``` + ## CRC32(s) Возвращает чексумму CRC32 данной строки, используется CRC-32-IEEE 802.3 многочлен и начальным значением `0xffffffff` (т.к. используется реализация из zlib). diff --git a/docs/ru/query_language/functions/type_conversion_functions.md b/docs/ru/query_language/functions/type_conversion_functions.md index af02eeae835..3fb431fa3b2 100644 --- a/docs/ru/query_language/functions/type_conversion_functions.md +++ b/docs/ru/query_language/functions/type_conversion_functions.md @@ -40,8 +40,36 @@ SELECT toInt64(nan), toInt32(32), toInt16('16'), toInt8(8.8) ## toInt(8|16|32|64)OrZero +Принимает аргумент типа String и пытается его распарсить в Int(8|16|32|64). Если не удалось - возвращает 0. + +**Пример** + +```sql +select toInt64OrZero('123123'), toInt8OrZero('123qwe123') +``` +```text +┌─toInt64OrZero('123123')─┬─toInt8OrZero('123qwe123')─┐ +│ 123123 │ 0 │ +└─────────────────────────┴───────────────────────────┘ +``` + + ## toInt(8|16|32|64)OrNull +Принимает аргумент типа String и пытается его распарсить в Int(8|16|32|64). Если не удалось - возвращает NULL. + +**Пример** + +```sql +select toInt64OrNull('123123'), toInt8OrNull('123qwe123') +``` +```text +┌─toInt64OrNull('123123')─┬─toInt8OrNull('123qwe123')─┐ +│ 123123 │ ᴺᵁᴸᴸ │ +└─────────────────────────┴───────────────────────────┘ +``` + + ## toUInt(8|16|32|64) Преобраует входное значение к типу [UInt](../../data_types/int_uint.md). Семейство функций включает: @@ -321,4 +349,48 @@ SELECT toTypeName(CAST(x, 'Nullable(UInt16)')) FROM t_null └─────────────────────────────────────────┘ ``` +## toInterval(Year|Quarter|Month|Week|Day|Hour|Minute|Second) {#function-tointerval} + +Приводит аргумент из числового типа данных к типу данных [IntervalType](../../data_types/special_data_types/interval.md). + +**Синтксис** + +```sql +toIntervalSecond(number) +toIntervalMinute(number) +toIntervalHour(number) +toIntervalDay(number) +toIntervalWeek(number) +toIntervalMonth(number) +toIntervalQuarter(number) +toIntervalYear(number) +``` + +**Параметры** + +- `number` — длительность интервала. Положительное целое число. + +**Возвращаемые значения** + +- Значение с типом данных `Interval`. + +**Пример** + +```sql +WITH + toDate('2019-01-01') AS date, + INTERVAL 1 WEEK AS interval_week, + toIntervalWeek(1) AS interval_to_week +SELECT + date + interval_week, + date + interval_to_week +``` + +```text +┌─plus(date, interval_week)─┬─plus(date, interval_to_week)─┐ +│ 2019-01-08 │ 2019-01-08 │ +└───────────────────────────┴──────────────────────────────┘ +``` + + [Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/type_conversion_functions/) diff --git a/docs/ru/query_language/insert_into.md b/docs/ru/query_language/insert_into.md index 88c548d394c..4cd14e21871 100644 --- a/docs/ru/query_language/insert_into.md +++ b/docs/ru/query_language/insert_into.md @@ -62,10 +62,10 @@ INSERT INTO [db.]table [(c1, c2, c3)] SELECT ... ### Замечания о производительности -`INSERT` сортирует входящие данные по первичному ключу и разбивает их на партиции по месяцам. Если вы вставляете данные за разные месяцы вперемешку, то это может значительно снизить производительность запроса `INSERT`. Чтобы избежать этого: +`INSERT` сортирует входящие данные по первичному ключу и разбивает их на партиции по ключу партиционирования. Если вы вставляете данные в несколько партиций одновременно, то это может значительно снизить производительность запроса `INSERT`. Чтобы избежать этого: - Добавляйте данные достаточно большими пачками. Например, по 100 000 строк. -- Группируйте данные по месяцам самостоятельно перед загрузкой в ClickHouse. +- Группируйте данные по ключу партиционирования самостоятельно перед загрузкой в ClickHouse. Снижения производительности не будет, если: diff --git a/docs/ru/query_language/misc.md b/docs/ru/query_language/misc.md index ce73a5aafdb..5a89856a143 100644 --- a/docs/ru/query_language/misc.md +++ b/docs/ru/query_language/misc.md @@ -173,7 +173,7 @@ KILL MUTATION WHERE database = 'default' AND table = 'table' AND mutation_id = ' ## OPTIMIZE {#misc_operations-optimize} ```sql -OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION ID 'partition_id'] [FINAL] +OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION ID 'partition_id'] [FINAL] [DEDUPLICATE] ``` Запрос пытается запустить внеплановый мёрж кусков данных для таблиц семейства [MergeTree](../operations/table_engines/mergetree.md). Другие движки таблиц не поддерживаются. @@ -183,6 +183,7 @@ OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION I - Если `OPTIMIZE` не выполняет мёрж по любой причине, ClickHouse не оповещает об этом клиента. Чтобы включить оповещения, используйте настройку [optimize_throw_if_noop](../operations/settings/settings.md#setting-optimize_throw_if_noop). - Если указать `PARTITION`, то оптимизация выполняется только для указанной партиции. [Как задавать имя партиции в запросах](alter.md#alter-how-to-specify-part-expr). - Если указать `FINAL`, то оптимизация выполняется даже в том случае, если все данные уже лежат в одном куске. +- Если указать `DEDUPLICATE`, то произойдет схлопывание полностью одинаковых строк (сравниваются значения во всех колонках), имеет смысл только для движка MergeTree. !!! warning "Внимание" Запрос `OPTIMIZE` не может устранить причину появления ошибки "Too many parts". @@ -213,72 +214,6 @@ SET profile = 'profile-name-from-the-settings-file' Подробности смотрите в разделе [Настройки](../operations/settings/settings.md). -## SHOW CREATE TABLE - -```sql -SHOW CREATE [TEMPORARY] TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] -``` - -Возвращает один столбец statement типа `String`, содержащий одно значение - запрос `CREATE`, с помощью которого создана указанная таблица. - -## SHOW DATABASES {#show-databases} - -```sql -SHOW DATABASES [INTO OUTFILE filename] [FORMAT format] -``` - -Выводит список всех баз данных. -Запрос полностью аналогичен запросу `SELECT name FROM system.databases [INTO OUTFILE filename] [FORMAT format]`. - -Смотрите также раздел "Форматы". - -## SHOW PROCESSLIST - -```sql -SHOW PROCESSLIST [INTO OUTFILE filename] [FORMAT format] -``` - -Выводит список запросов, выполняющихся в данный момент времени, кроме самих запросов `SHOW PROCESSLIST`. - -Выдаёт таблицу, содержащую столбцы: - -**user** - пользователь, под которым был задан запрос. Следует иметь ввиду, что при распределённой обработке запроса на удалённые серверы запросы отправляются под пользователем 'default'. И SHOW PROCESSLIST показывает имя пользователя для конкретного запроса, а не для запроса, который данный запрос инициировал. - -**address** - имя хоста, с которого был отправлен запрос. При распределённой обработке запроса на удалённых серверах — это имя хоста-инициатора запроса. Чтобы проследить, откуда был задан распределённый запрос изначально, следует смотреть SHOW PROCESSLIST на сервере-инициаторе запроса. - -**elapsed** - время выполнения запроса, в секундах. Запросы выводятся в порядке убывания времени выполнения. - -**rows_read**, **bytes_read** - сколько было прочитано строк, байт несжатых данных при обработке запроса. При распределённой обработке запроса суммируются данные со всех удалённых серверов. Именно эти данные используются для ограничений и квот. - -**memory_usage** - текущее потребление оперативки в байтах. Смотрите настройку 'max_memory_usage'. - -**query** - сам запрос. В запросах INSERT данные для вставки не выводятся. - -**query_id** - идентификатор запроса. Непустой, только если был явно задан пользователем. При распределённой обработке запроса идентификатор запроса не передаётся на удалённые серверы. - -Этот запрос аналогичен запросу `SELECT * FROM system.processes` за тем исключением, что последний отображает список запросов, включая самого себя. - -Полезный совет (выполните в консоли): - -```bash -$ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'" -``` - -## SHOW TABLES - -```sql -SHOW [TEMPORARY] TABLES [FROM db] [LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format] -``` - -Выводит список таблиц: - -- из текущей базы данных или из базы db, если указано `FROM db`; -- всех, или имя которых соответствует шаблону pattern, если указано `LIKE 'pattern'`; - -Запрос полностью аналогичен запросу: `SELECT name FROM system.tables WHERE database = 'db' [AND name LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format]`. - -Смотрите также раздел "Оператор LIKE". - ## TRUNCATE ```sql diff --git a/docs/ru/query_language/operators.md b/docs/ru/query_language/operators.md index 74aa2270e90..31378760fcb 100644 --- a/docs/ru/query_language/operators.md +++ b/docs/ru/query_language/operators.md @@ -67,6 +67,8 @@ ## Оператор для работы с датами и временем {#operators-datetime} +### EXTRACT + ```sql EXTRACT(part FROM date); ``` @@ -128,6 +130,39 @@ FROM test.Orders; Больше примеров приведено в [тестах](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00619_extract.sql). +### INTERVAL {#operator-interval} + +Создаёт значение типа [Interval](../data_types/special_data_types/interval.md) которое должно использоваться в арифметических операциях со значениями типов [Date](../data_types/date.md) и [DateTime](../data_types/datetime.md). + +Типы интервалов: +- `SECOND` +- `MINUTE` +- `HOUR` +- `DAY` +- `WEEK` +- `MONTH` +- `QUARTER` +- `YEAR` + +!!! warning "Внимание" + Интервалы различных типов нельзя объединять. Нельзя использовать выражения вида `INTERVAL 4 DAY 1 HOUR`. Вместо этого интервалы можно выразить в единицах меньших или равных наименьшей единице интервала, Например, `INTERVAL 25 HOUR`. Также можно выполнять последовательные операции как показано в примере ниже. + +Пример: + +```sql +SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL 3 HOUR +``` +```text +┌───current_date_time─┬─plus(plus(now(), toIntervalDay(4)), toIntervalHour(3))─┐ +│ 2019-10-23 11:16:28 │ 2019-10-27 14:16:28 │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +``` + +**Смотрите также** + +- Тип данных [Interval](../data_types/special_data_types/interval.md) +- Функции преобразования типов [toInterval](functions/type_conversion_functions.md#function-tointerval) + ## Оператор логического отрицания `NOT a` - функция `not(a)` diff --git a/docs/ru/query_language/select.md b/docs/ru/query_language/select.md index 49e2c0692ef..56549f21e53 100644 --- a/docs/ru/query_language/select.md +++ b/docs/ru/query_language/select.md @@ -984,12 +984,66 @@ WHERE и HAVING отличаются тем, что WHERE выполняется Внешняя сортировка работает существенно менее эффективно, чем сортировка в оперативке. -### Секция SELECT +### Секция SELECT {#select-select} -После вычислений, соответствующих всем перечисленным выше секциям, производится вычисление выражений, указанных в секции SELECT. -Вернее, вычисляются выражения, стоящие над агрегатными функциями, если есть агрегатные функции. -Сами агрегатные функции и то, что под ними, вычисляются при агрегации (GROUP BY). -Эти выражения работают так, как будто применяются к отдельным строкам результата. +[Выражения](syntax.md#syntax-expressions) указанные в секции `SELECT` анализируются после завершения всех вычислений из секций, описанных выше. Вернее, анализируются выражения, стоящие над агрегатными функциями, если есть агрегатные функции. +Сами агрегатные функции и то, что под ними, вычисляются при агрегации (`GROUP BY`). Эти выражения работают так, как будто применяются к отдельным строкам результата. + +Если в результат необходимо включить все столбцы, используйте символ звёздочка (`*`). Например, `SELECT * FROM ...`. + +Чтобы включить в результат несколько столбцов, выбрав их имена с помощью регулярных выражений [re2](https://en.wikipedia.org/wiki/RE2_(software)), используйте выражение `COLUMNS`. + +```sql +COLUMNS('regexp') +``` + +Например, рассмотрим таблицу: + +```sql +CREATE TABLE default.col_names (aa Int8, ab Int8, bc Int8) ENGINE = TinyLog +``` + +Следующий запрос выбирает данные из всех столбцов, содержащих в имени символ `a`. + +```sql +SELECT COLUMNS('a') FROM col_names +``` +```text +┌─aa─┬─ab─┐ +│ 1 │ 1 │ +└────┴────┘ +``` + +Выбранные стоблцы возвращаются не в алфавитном порядке. + +В запросе можно использовать несколько выражений `COLUMNS`, а также вызывать над ними функции. + +Например: + +```sql +SELECT COLUMNS('a'), COLUMNS('c'), toTypeName(COLUMNS('c')) FROM col_names +``` +```text +┌─aa─┬─ab─┬─bc─┬─toTypeName(bc)─┐ +│ 1 │ 1 │ 1 │ Int8 │ +└────┴────┴────┴────────────────┘ +``` + +Каждый столбец, возвращённый выражением `COLUMNS`, передаётся в функцию отдельным аргументом. Также можно передавать и другие аргументы, если функция их поддерживаем. Аккуратно используйте функции. Если функция не поддерживает переданное количество аргументов, то ClickHouse генерирует исключение. + +Например: + +```sql +SELECT COLUMNS('a') + COLUMNS('c') FROM col_names +``` +```text +Received exception from server (version 19.14.1): +Code: 42. DB::Exception: Received from localhost:9000. DB::Exception: Number of arguments for function plus doesn't match: passed 3, should be 2. +``` + +В этом примере, `COLUMNS('a')` возвращает два столбца: `aa` и `ab`. `COLUMNS('c')` возвращает столбец `bc`. Оператор `+` не работает с тремя аргументами, поэтому ClickHouse генерирует исключение с соответствущим сообщением. + +Столбцы, которые возвращаются выражением `COLUMNS` могут быть разных типов. Если `COLUMNS` не возвращает ни одного столбца и это единственное выражение в запросе `SELECT`, то ClickHouse генерирует исключение. ### Секция DISTINCT {#select-distinct} diff --git a/docs/ru/query_language/show.md b/docs/ru/query_language/show.md new file mode 100644 index 00000000000..4eec70a8002 --- /dev/null +++ b/docs/ru/query_language/show.md @@ -0,0 +1,64 @@ +# SHOW Queries + +## SHOW CREATE TABLE + +```sql +SHOW CREATE [TEMPORARY] TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] +``` + +Возвращает один столбец типа `String` с именем statement, содержащий одно значение — запрос `CREATE TABLE`, с помощью которого была создана указанная таблица. + +## SHOW DATABASES {#show-databases} + +```sql +SHOW DATABASES [INTO OUTFILE filename] [FORMAT format] +``` + +Выводит список всех баз данных. +Запрос полностью аналогичен запросу `SELECT name FROM system.databases [INTO OUTFILE filename] [FORMAT format]`. + +## SHOW PROCESSLIST + +```sql +SHOW PROCESSLIST [INTO OUTFILE filename] [FORMAT format] +``` + +Выводит содержимое таблицы [system.processes](../operations/system_tables.md#system_tables-processes), которая содержит список запросов, выполняющихся в данный момент времени, кроме самих запросов `SHOW PROCESSLIST`. + +Запрос `SELECT * FROM system.processes` возвращает данные обо всех текущих запросах. + +Полезный совет (выполните в консоли): + +```bash +$ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'" +``` + +## SHOW TABLES + +Выводит список таблиц. + +```sql +SHOW [TEMPORARY] TABLES [FROM ] [LIKE ''] [LIMIT ] [INTO OUTFILE ] [FORMAT ] +``` + +Если секция `FROM` не используется, то запрос возвращает список таблиц из текущей базы данных. + +Результат, идентичный тому, что выдаёт запрос `SHOW TABLES` можно получить также запросом следующего вида: + +```sql +SELECT name FROM system.tables WHERE database = [AND name LIKE ] [LIMIT ] [INTO OUTFILE ] [FORMAT ] +``` + +**Пример** + +Следующий запрос выбирает первые две строки из списка таблиц в базе данных `system`, чьи имена содержат `co`. + +```sql +SHOW TABLES FROM system LIKE '%co%' LIMIT 2 +``` +```text +┌─name───────────────────────────┐ +│ aggregate_function_combinators │ +│ collations │ +└────────────────────────────────┘ +``` diff --git a/docs/ru/query_language/system.md b/docs/ru/query_language/system.md index 474574b0b19..31e0c3cf90b 100644 --- a/docs/ru/query_language/system.md +++ b/docs/ru/query_language/system.md @@ -3,7 +3,7 @@ - [RELOAD DICTIONARIES](#query_language-system-reload-dictionaries) - [RELOAD DICTIONARY](#query_language-system-reload-dictionary) - [DROP DNS CACHE](#query_language-system-drop-dns-cache) -- [DROP MARKS CACHE](#query_language-system-drop-marks-cache) +- [DROP MARK CACHE](#query_language-system-drop-mark-cache) - [FLUSH LOGS](#query_language-system-flush_logs) - [RELOAD CONFIG](#query_language-system-reload-config) - [SHUTDOWN](#query_language-system-shutdown) @@ -11,6 +11,8 @@ - [STOP DISTRIBUTED SENDS](#query_language-system-stop-distributed-sends) - [FLUSH DISTRIBUTED](#query_language-system-flush-distributed) - [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends) +- [STOP MERGES](#query_language-system-stop-merges) +- [START MERGES](#query_language-system-start-merges) ## RELOAD DICTIONARIES {#query_language-system-reload-dictionaries} @@ -34,7 +36,7 @@ SELECT name, status FROM system.dictionaries; Для более удобного (автоматического) управления кешем см. параметры disable_internal_dns_cache, dns_cache_update_period. -## DROP MARKS CACHE {#query_language-system-drop-marks-cache} +## DROP MARK CACHE {#query_language-system-drop-mark-cache} Сбрасывает кеш "засечек" (`mark cache`). Используется при разработке ClickHouse и тестах производительности. @@ -82,5 +84,24 @@ SYSTEM FLUSH DISTRIBUTED [db.] SYSTEM START DISTRIBUTED SENDS [db.] ``` +### STOP MERGES {#query_language-system-stop-merges} + +Позволяет остановить фоновые мержи для таблиц семейства MergeTree: + +```sql +SYSTEM STOP MERGES [[db.]merge_tree_family_table_name] +``` +!!! note "Note" + `DETACH / ATTACH` таблицы восстанавливает фоновые мержи для этой таблицы (даже в случае отключения фоновых мержей для всех таблиц семейства MergeTree до `DETACH`). + + +### START MERGES {#query_language-system-start-merges} + +Включает фоновые мержи для таблиц семейства MergeTree: + +```sql +SYSTEM START MERGES [[db.]merge_tree_family_table_name] +``` + [Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/system/) diff --git a/docs/ru/security_changelog.md b/docs/ru/security_changelog.md index 77408e7ba30..17ae1eba19d 100644 --- a/docs/ru/security_changelog.md +++ b/docs/ru/security_changelog.md @@ -1,3 +1,10 @@ +## Исправлено в релизе 19.13.6.1 от 20 сентября 2019 + +### CVE-2019-18657 +Уязвимость в табличной функции `url` позволяла злоумышленнику добавлять произвольные HTTP-заголовки в запрос. + +Обнаружено благодаря: [Никите Тихомирову](https://github.com/NSTikhomirov) + ## Исправлено в релизе 18.12.13 от 10 сентября 2018 ### CVE-2018-14672 diff --git a/docs/toc_en.yml b/docs/toc_en.yml index 46cef8feeeb..8a2b32b240a 100644 --- a/docs/toc_en.yml +++ b/docs/toc_en.yml @@ -5,10 +5,12 @@ nav: - 'Distinctive Features of ClickHouse': 'introduction/distinctive_features.md' - 'ClickHouse Features that Can Be Considered Disadvantages': 'introduction/features_considered_disadvantages.md' - 'Performance': 'introduction/performance.md' - - 'The Yandex.Metrica Task': 'introduction/ya_metrika_task.md' + - 'History': 'introduction/history.md' - 'Getting Started': - - 'Deploying and Running': 'getting_started/index.md' + - 'hidden': 'getting_started/index.md' + - 'Installation': 'getting_started/install.md' + - 'Tutorial': 'getting_started/tutorial.md' - 'Example Datasets': - 'OnTime': 'getting_started/example_datasets/ontime.md' - 'New York Taxi Data': 'getting_started/example_datasets/nyc_taxi.md' @@ -26,6 +28,7 @@ nav: - 'Input and Output Formats': 'interfaces/formats.md' - 'JDBC Driver': 'interfaces/jdbc.md' - 'ODBC Driver': 'interfaces/odbc.md' + - 'C++ Client Library': 'interfaces/cpp.md' - 'Third-Party': - 'Client Libraries': 'interfaces/third-party/client_libraries.md' - 'Integrations': 'interfaces/third-party/integrations.md' @@ -118,6 +121,7 @@ nav: - 'Working with geographical coordinates': 'query_language/functions/geo.md' - 'Working with Nullable arguments': 'query_language/functions/functions_for_nulls.md' - 'Machine Learning Functions': 'query_language/functions/machine_learning_functions.md' + - 'Introspection': 'query_language/functions/introspection.md' - 'Other': 'query_language/functions/other_functions.md' - 'Aggregate Functions': - 'Introduction': 'query_language/agg_functions/index.md' @@ -171,11 +175,16 @@ nav: - 'Expression': 'data_types/special_data_types/expression.md' - 'Set': 'data_types/special_data_types/set.md' - 'Nothing': 'data_types/special_data_types/nothing.md' + - 'Interval': 'data_types/special_data_types/interval.md' - 'Domains': - 'Overview': 'data_types/domains/overview.md' - 'IPv4': 'data_types/domains/ipv4.md' - 'IPv6': 'data_types/domains/ipv6.md' +- 'Guides': + - 'Overview': 'guides/index.md' + - 'Applying CatBoost Models': 'guides/apply_catboost_model.md' + - 'Operations': - 'Introduction': 'operations/index.md' - 'Requirements': 'operations/requirements.md' @@ -204,18 +213,16 @@ nav: - 'clickhouse-copier': 'operations/utils/clickhouse-copier.md' - 'clickhouse-local': 'operations/utils/clickhouse-local.md' -- 'Guides': - - 'Overview': 'guides/index.md' - - 'Applying CatBoost Models': 'guides/apply_catboost_model.md' - - 'Development': - 'hidden': 'development/index.md' - 'Overview of ClickHouse Architecture': 'development/architecture.md' - 'How to Build ClickHouse on Linux': 'development/build.md' - 'How to Build ClickHouse on Mac OS X': 'development/build_osx.md' - - 'How to Build ClickHouse on Linux for Mac OS X': 'development/build_cross.md' - - 'How to Write C++ code': 'development/style.md' + - 'How to Build ClickHouse on Linux for Mac OS X': 'development/build_cross_osx.md' + - 'How to Build ClickHouse on Linux for AARCH64 (ARM64)': 'development/build_cross_arm.md' + - 'How to Write C++ Code': 'development/style.md' - 'How to Run ClickHouse Tests': 'development/tests.md' + - 'The Beginner ClickHouse Developer Instruction': 'development/developer_instruction.md' - 'Third-Party Libraries Used': 'development/contrib.md' - 'What''s New': diff --git a/docs/toc_fa.yml b/docs/toc_fa.yml index afbe8709a47..c5a2a7fd80b 100644 --- a/docs/toc_fa.yml +++ b/docs/toc_fa.yml @@ -1,15 +1,17 @@ nav: -- 'Introduction': +- 'ﯽﻓﺮﻌﻣ': - 'ClickHouse چیست؟': 'index.md' - ' ویژگی های برجسته ClickHouse': 'introduction/distinctive_features.md' - ' ویژگی های از ClickHouse که می تواند معایبی باشد': 'introduction/features_considered_disadvantages.md' - - 'Performance': 'introduction/performance.md' - - 'The Yandex.Metrica task': 'introduction/ya_metrika_task.md' + - 'ﯽﯾﺍﺭﺎﮐ': 'introduction/performance.md' + - 'ﺦﯾﺭﺎﺗ': 'introduction/history.md' - 'Getting started': - - ' شروع به کار': 'getting_started/index.md' - - 'Example datasets': + - 'hidden': 'getting_started/index.md' + - 'ﯼﺯﺍﺪﻧﺍ ﻩﺍﺭ ﻭ ﺐﺼﻧ': 'getting_started/install.md' + - 'ﺵﺯﻮﻣﺁ': 'getting_started/tutorial.md' + - 'ﻪﻧﻮﻤﻧ ﯼﺎﻫ ﻩﺩﺍﺩ ﻪﻋﻮﻤﺠﻣ': - 'OnTime': 'getting_started/example_datasets/ontime.md' - ' داده های تاکسی New York': 'getting_started/example_datasets/nyc_taxi.md' - ' بنچمارک AMPLab Big Data': 'getting_started/example_datasets/amplab_benchmark.md' @@ -18,7 +20,7 @@ nav: - ' بنچمارک Star Schema': 'getting_started/example_datasets/star_schema.md' - 'Yandex.Metrica Data': 'getting_started/example_datasets/metrica.md' -- 'Interfaces': +- 'ﻂﺑﺍﺭ': - 'Interface ها': 'interfaces/index.md' - ' کلاینت Command-line': 'interfaces/cli.md' - 'Native interface (TCP)': 'interfaces/tcp.md' @@ -26,13 +28,14 @@ nav: - ' فرمت های Input و Output': 'interfaces/formats.md' - ' درایور JDBC': 'interfaces/jdbc.md' - ' درایور ODBC': 'interfaces/odbc.md' + - 'C ++ کتابخانه مشتری': 'interfaces/cpp.md' - 'Third-party': - 'کتابخانه های مشتری': 'interfaces/third-party/client_libraries.md' - 'یکپارچگی': 'interfaces/third-party/integrations.md' - 'رابط های بصری': 'interfaces/third-party/gui.md' - 'پروکسی': 'interfaces/third-party/proxy.md' -- 'Data types': +- 'ﻩﺩﺍﺩ ﻉﺍﻮﻧﺍ': - 'Introduction': 'data_types/index.md' - 'UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64': 'data_types/int_uint.md' - 'Float32, Float64': 'data_types/float.md' @@ -56,6 +59,7 @@ nav: - 'Expression': 'data_types/special_data_types/expression.md' - 'Set': 'data_types/special_data_types/set.md' - 'Nothing': 'data_types/special_data_types/nothing.md' + - 'Interval': 'data_types/special_data_types/interval.md' - 'Domains': - 'Overview': 'data_types/domains/overview.md' - 'IPv4': 'data_types/domains/ipv4.md' @@ -145,6 +149,7 @@ nav: - 'Working with geographical coordinates': 'query_language/functions/geo.md' - 'Working with Nullable arguments': 'query_language/functions/functions_for_nulls.md' - 'Machine Learning Functions': 'query_language/functions/machine_learning_functions.md' + - 'Introspection': 'query_language/functions/introspection.md' - 'Other': 'query_language/functions/other_functions.md' - 'Aggregate Functions': - 'Introduction': 'query_language/agg_functions/index.md' @@ -214,6 +219,7 @@ nav: - 'How to Build ClickHouse on Mac OS X': 'development/build_osx.md' - 'How to Write C++ code': 'development/style.md' - 'How to Run ClickHouse Tests': 'development/tests.md' + - 'The Beginner ClickHouse Developer Instruction': 'development/developer_instruction.md' - 'Third-Party Libraries Used': 'development/contrib.md' - 'What''s New': diff --git a/docs/toc_ja.yml b/docs/toc_ja.yml new file mode 100644 index 00000000000..8a2b32b240a --- /dev/null +++ b/docs/toc_ja.yml @@ -0,0 +1,234 @@ +nav: + +- 'Introduction': + - 'Overview': 'index.md' + - 'Distinctive Features of ClickHouse': 'introduction/distinctive_features.md' + - 'ClickHouse Features that Can Be Considered Disadvantages': 'introduction/features_considered_disadvantages.md' + - 'Performance': 'introduction/performance.md' + - 'History': 'introduction/history.md' + +- 'Getting Started': + - 'hidden': 'getting_started/index.md' + - 'Installation': 'getting_started/install.md' + - 'Tutorial': 'getting_started/tutorial.md' + - 'Example Datasets': + - 'OnTime': 'getting_started/example_datasets/ontime.md' + - 'New York Taxi Data': 'getting_started/example_datasets/nyc_taxi.md' + - 'AMPLab Big Data Benchmark': 'getting_started/example_datasets/amplab_benchmark.md' + - 'WikiStat': 'getting_started/example_datasets/wikistat.md' + - 'Terabyte Click Logs from Criteo': 'getting_started/example_datasets/criteo.md' + - 'Star Schema Benchmark': 'getting_started/example_datasets/star_schema.md' + - 'Yandex.Metrica Data': 'getting_started/example_datasets/metrica.md' + +- 'Interfaces': + - 'Introduction': 'interfaces/index.md' + - 'Command-Line Client': 'interfaces/cli.md' + - 'Native Interface (TCP)': 'interfaces/tcp.md' + - 'HTTP Interface': 'interfaces/http.md' + - 'Input and Output Formats': 'interfaces/formats.md' + - 'JDBC Driver': 'interfaces/jdbc.md' + - 'ODBC Driver': 'interfaces/odbc.md' + - 'C++ Client Library': 'interfaces/cpp.md' + - 'Third-Party': + - 'Client Libraries': 'interfaces/third-party/client_libraries.md' + - 'Integrations': 'interfaces/third-party/integrations.md' + - 'Visual Interfaces': 'interfaces/third-party/gui.md' + - 'Proxies': 'interfaces/third-party/proxy.md' + +- 'Database Engines': + - 'Introduction': 'database_engines/index.md' + - 'MySQL': 'database_engines/mysql.md' + - 'Lazy': 'database_engines/lazy.md' + +- 'Table Engines': + - 'Introduction': 'operations/table_engines/index.md' + - 'MergeTree Family': + - 'MergeTree': 'operations/table_engines/mergetree.md' + - 'Data Replication': 'operations/table_engines/replication.md' + - 'Custom Partitioning Key': 'operations/table_engines/custom_partitioning_key.md' + - 'ReplacingMergeTree': 'operations/table_engines/replacingmergetree.md' + - 'SummingMergeTree': 'operations/table_engines/summingmergetree.md' + - 'AggregatingMergeTree': 'operations/table_engines/aggregatingmergetree.md' + - 'CollapsingMergeTree': 'operations/table_engines/collapsingmergetree.md' + - 'VersionedCollapsingMergeTree': 'operations/table_engines/versionedcollapsingmergetree.md' + - 'GraphiteMergeTree': 'operations/table_engines/graphitemergetree.md' + - 'Log Family': + - 'Introduction': 'operations/table_engines/log_family.md' + - 'StripeLog': 'operations/table_engines/stripelog.md' + - 'Log': 'operations/table_engines/log.md' + - 'TinyLog': 'operations/table_engines/tinylog.md' + - 'Integrations': + - 'Kafka': 'operations/table_engines/kafka.md' + - 'MySQL': 'operations/table_engines/mysql.md' + - 'JDBC': 'operations/table_engines/jdbc.md' + - 'ODBC': 'operations/table_engines/odbc.md' + - 'HDFS': 'operations/table_engines/hdfs.md' + - 'Special': + - 'Distributed': 'operations/table_engines/distributed.md' + - 'External data': 'operations/table_engines/external_data.md' + - 'Dictionary': 'operations/table_engines/dictionary.md' + - 'Merge': 'operations/table_engines/merge.md' + - 'File': 'operations/table_engines/file.md' + - 'Null': 'operations/table_engines/null.md' + - 'Set': 'operations/table_engines/set.md' + - 'Join': 'operations/table_engines/join.md' + - 'URL': 'operations/table_engines/url.md' + - 'View': 'operations/table_engines/view.md' + - 'MaterializedView': 'operations/table_engines/materializedview.md' + - 'Memory': 'operations/table_engines/memory.md' + - 'Buffer': 'operations/table_engines/buffer.md' + +- 'SQL Reference': + - 'hidden': 'query_language/index.md' + - 'Syntax': 'query_language/syntax.md' + - 'Statements': + - 'SELECT': 'query_language/select.md' + - 'INSERT INTO': 'query_language/insert_into.md' + - 'CREATE': 'query_language/create.md' + - 'ALTER': 'query_language/alter.md' + - 'SYSTEM': 'query_language/system.md' + - 'SHOW': 'query_language/show.md' + - 'Other': 'query_language/misc.md' + - 'Functions': + - 'Introduction': 'query_language/functions/index.md' + - 'Arithmetic': 'query_language/functions/arithmetic_functions.md' + - 'Comparison': 'query_language/functions/comparison_functions.md' + - 'Logical': 'query_language/functions/logical_functions.md' + - 'Type Conversion': 'query_language/functions/type_conversion_functions.md' + - 'Working with Dates and Times': 'query_language/functions/date_time_functions.md' + - 'Working with strings': 'query_language/functions/string_functions.md' + - 'For Searching Strings': 'query_language/functions/string_search_functions.md' + - 'For Replacing in Strings': 'query_language/functions/string_replace_functions.md' + - 'Conditional ': 'query_language/functions/conditional_functions.md' + - 'Mathematical': 'query_language/functions/math_functions.md' + - 'Rounding': 'query_language/functions/rounding_functions.md' + - 'Working with Arrays': 'query_language/functions/array_functions.md' + - 'Splitting and Merging Strings and Arrays': 'query_language/functions/splitting_merging_functions.md' + - 'Bit': 'query_language/functions/bit_functions.md' + - 'Bitmap': 'query_language/functions/bitmap_functions.md' + - 'Hash': 'query_language/functions/hash_functions.md' + - 'Generating Pseudo-Random Numbers': 'query_language/functions/random_functions.md' + - 'Encoding': 'query_language/functions/encoding_functions.md' + - 'Working with UUID': 'query_language/functions/uuid_functions.md' + - 'Working with URLs': 'query_language/functions/url_functions.md' + - 'Working with IP Addresses': 'query_language/functions/ip_address_functions.md' + - 'Working with JSON.': 'query_language/functions/json_functions.md' + - 'Higher-Order': 'query_language/functions/higher_order_functions.md' + - 'Working with External Dictionaries': 'query_language/functions/ext_dict_functions.md' + - 'Working with Yandex.Metrica Dictionaries': 'query_language/functions/ym_dict_functions.md' + - 'Implementing the IN Operator': 'query_language/functions/in_functions.md' + - 'arrayJoin': 'query_language/functions/array_join.md' + - 'Working with geographical coordinates': 'query_language/functions/geo.md' + - 'Working with Nullable arguments': 'query_language/functions/functions_for_nulls.md' + - 'Machine Learning Functions': 'query_language/functions/machine_learning_functions.md' + - 'Introspection': 'query_language/functions/introspection.md' + - 'Other': 'query_language/functions/other_functions.md' + - 'Aggregate Functions': + - 'Introduction': 'query_language/agg_functions/index.md' + - 'Reference': 'query_language/agg_functions/reference.md' + - 'Aggregate function combinators': 'query_language/agg_functions/combinators.md' + - 'Parametric aggregate functions': 'query_language/agg_functions/parametric_functions.md' + - 'Table Functions': + - 'Introduction': 'query_language/table_functions/index.md' + - 'file': 'query_language/table_functions/file.md' + - 'merge': 'query_language/table_functions/merge.md' + - 'numbers': 'query_language/table_functions/numbers.md' + - 'remote': 'query_language/table_functions/remote.md' + - 'url': 'query_language/table_functions/url.md' + - 'mysql': 'query_language/table_functions/mysql.md' + - 'jdbc': 'query_language/table_functions/jdbc.md' + - 'odbc': 'query_language/table_functions/odbc.md' + - 'hdfs': 'query_language/table_functions/hdfs.md' + - 'input': 'query_language/table_functions/input.md' + - 'Dictionaries': + - 'Introduction': 'query_language/dicts/index.md' + - 'External Dictionaries': + - 'General Description': 'query_language/dicts/external_dicts.md' + - 'Configuring an External Dictionary': 'query_language/dicts/external_dicts_dict.md' + - 'Storing Dictionaries in Memory': 'query_language/dicts/external_dicts_dict_layout.md' + - 'Dictionary Updates': 'query_language/dicts/external_dicts_dict_lifetime.md' + - 'Sources of External Dictionaries': 'query_language/dicts/external_dicts_dict_sources.md' + - 'Dictionary Key and Fields': 'query_language/dicts/external_dicts_dict_structure.md' + - 'Internal Dictionaries': 'query_language/dicts/internal_dicts.md' + - 'Operators': 'query_language/operators.md' + - 'Data Types': + - 'Introduction': 'data_types/index.md' + - 'UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64': 'data_types/int_uint.md' + - 'Float32, Float64': 'data_types/float.md' + - 'Decimal': 'data_types/decimal.md' + - 'Boolean': 'data_types/boolean.md' + - 'String': 'data_types/string.md' + - 'FixedString(N)': 'data_types/fixedstring.md' + - 'UUID': 'data_types/uuid.md' + - 'Date': 'data_types/date.md' + - 'DateTime': 'data_types/datetime.md' + - 'Enum': 'data_types/enum.md' + - 'Array(T)': 'data_types/array.md' + - 'AggregateFunction(name, types_of_arguments...)': 'data_types/nested_data_structures/aggregatefunction.md' + - 'Tuple(T1, T2, ...)': 'data_types/tuple.md' + - 'Nullable': 'data_types/nullable.md' + - 'Nested Data Structures': + - 'hidden': 'data_types/nested_data_structures/index.md' + - 'Nested(Name1 Type1, Name2 Type2, ...)': 'data_types/nested_data_structures/nested.md' + - 'Special Data Types': + - 'hidden': 'data_types/special_data_types/index.md' + - 'Expression': 'data_types/special_data_types/expression.md' + - 'Set': 'data_types/special_data_types/set.md' + - 'Nothing': 'data_types/special_data_types/nothing.md' + - 'Interval': 'data_types/special_data_types/interval.md' + - 'Domains': + - 'Overview': 'data_types/domains/overview.md' + - 'IPv4': 'data_types/domains/ipv4.md' + - 'IPv6': 'data_types/domains/ipv6.md' + +- 'Guides': + - 'Overview': 'guides/index.md' + - 'Applying CatBoost Models': 'guides/apply_catboost_model.md' + +- 'Operations': + - 'Introduction': 'operations/index.md' + - 'Requirements': 'operations/requirements.md' + - 'Monitoring': 'operations/monitoring.md' + - 'Troubleshooting': 'operations/troubleshooting.md' + - 'Usage Recommendations': 'operations/tips.md' + - 'ClickHouse Update': 'operations/update.md' + - 'Access Rights': 'operations/access_rights.md' + - 'Data Backup': 'operations/backup.md' + - 'Configuration Files': 'operations/configuration_files.md' + - 'Quotas': 'operations/quotas.md' + - 'System Tables': 'operations/system_tables.md' + - 'Server Configuration Parameters': + - 'Introduction': 'operations/server_settings/index.md' + - 'Server Settings': 'operations/server_settings/settings.md' + - 'Settings': + - 'Introduction': 'operations/settings/index.md' + - 'Permissions for Queries': 'operations/settings/permissions_for_queries.md' + - 'Restrictions on Query Complexity': 'operations/settings/query_complexity.md' + - 'Settings': 'operations/settings/settings.md' + - 'Settings Profiles': 'operations/settings/settings_profiles.md' + - 'Constraints on Settings': 'operations/settings/constraints_on_settings.md' + - 'User Settings': 'operations/settings/settings_users.md' + - 'Utilities': + - 'Overview': 'operations/utils/index.md' + - 'clickhouse-copier': 'operations/utils/clickhouse-copier.md' + - 'clickhouse-local': 'operations/utils/clickhouse-local.md' + +- 'Development': + - 'hidden': 'development/index.md' + - 'Overview of ClickHouse Architecture': 'development/architecture.md' + - 'How to Build ClickHouse on Linux': 'development/build.md' + - 'How to Build ClickHouse on Mac OS X': 'development/build_osx.md' + - 'How to Build ClickHouse on Linux for Mac OS X': 'development/build_cross_osx.md' + - 'How to Build ClickHouse on Linux for AARCH64 (ARM64)': 'development/build_cross_arm.md' + - 'How to Write C++ Code': 'development/style.md' + - 'How to Run ClickHouse Tests': 'development/tests.md' + - 'The Beginner ClickHouse Developer Instruction': 'development/developer_instruction.md' + - 'Third-Party Libraries Used': 'development/contrib.md' + +- 'What''s New': + - 'Roadmap': 'roadmap.md' + - 'Changelog': 'changelog.md' + - 'Security Changelog': 'security_changelog.md' + +- 'F.A.Q.': + - 'General Questions': 'faq/general.md' diff --git a/docs/toc_ru.yml b/docs/toc_ru.yml index ce3f87e92fc..469590b6bc8 100644 --- a/docs/toc_ru.yml +++ b/docs/toc_ru.yml @@ -5,11 +5,13 @@ nav: - 'Отличительные возможности ClickHouse': 'introduction/distinctive_features.md' - 'Особенности ClickHouse, которые могут считаться недостатками': 'introduction/features_considered_disadvantages.md' - 'Производительность': 'introduction/performance.md' - - 'Постановка задачи в Яндекс.Метрике': 'introduction/ya_metrika_task.md' + - 'История': 'introduction/history.md' - 'Информационная поддержка': 'introduction/info.md' - 'Начало работы': - - 'Установка и запуск': 'getting_started/index.md' + - 'hidden': 'getting_started/index.md' + - 'Установка': 'getting_started/install.md' + - 'Руководство для начинающих': 'getting_started/tutorial.md' - 'Тестовые наборы данных': - 'OnTime': 'getting_started/example_datasets/ontime.md' - 'Данные о такси в Нью-Йорке': 'getting_started/example_datasets/nyc_taxi.md' @@ -27,41 +29,13 @@ nav: - 'Форматы входных и выходных данных': 'interfaces/formats.md' - 'JDBC-драйвер': 'interfaces/jdbc.md' - 'ODBC-драйвер': 'interfaces/odbc.md' + - 'C++ клиентская библиотека': 'interfaces/cpp.md' - 'От сторонних разработчиков': - 'Клиентские библиотеки': 'interfaces/third-party/client_libraries.md' - 'Интеграции': 'interfaces/third-party/integrations.md' - 'Визуальные интерфейсы': 'interfaces/third-party/gui.md' - 'Прокси': 'interfaces/third-party/proxy.md' -- 'Типы данных': - - 'Введение': 'data_types/index.md' - - 'UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64': 'data_types/int_uint.md' - - 'Float32, Float64': 'data_types/float.md' - - 'Decimal': 'data_types/decimal.md' - - 'Булевы значения': 'data_types/boolean.md' - - 'String': 'data_types/string.md' - - 'FixedString(N)': 'data_types/fixedstring.md' - - 'UUID': 'data_types/uuid.md' - - 'Date': 'data_types/date.md' - - 'DateTime': 'data_types/datetime.md' - - 'Enum': 'data_types/enum.md' - - 'Array(T)': 'data_types/array.md' - - 'AggregateFunction(name, types_of_arguments...)': 'data_types/nested_data_structures/aggregatefunction.md' - - 'Tuple(T1, T2, ...)': 'data_types/tuple.md' - - 'Nullable': 'data_types/nullable.md' - - 'Вложенные структуры данных': - - 'hidden': 'data_types/nested_data_structures/index.md' - - 'Nested(Name1 Type1, Name2 Type2, ...)': 'data_types/nested_data_structures/nested.md' - - 'Служебные типы данных': - - 'hidden': 'data_types/special_data_types/index.md' - - 'Expression': 'data_types/special_data_types/expression.md' - - 'Set': 'data_types/special_data_types/set.md' - - 'Nothing': 'data_types/special_data_types/nothing.md' - - 'Domains': - - 'Overview': 'data_types/domains/overview.md' - - 'IPv4': 'data_types/domains/ipv4.md' - - 'IPv6': 'data_types/domains/ipv6.md' - - 'Движки баз данных': - 'Введение': 'database_engines/index.md' - 'MySQL': 'database_engines/mysql.md' @@ -106,12 +80,15 @@ nav: - 'Справка по SQL': - 'hidden': 'query_language/index.md' - - 'SELECT': 'query_language/select.md' - - 'INSERT INTO': 'query_language/insert_into.md' - - 'CREATE': 'query_language/create.md' - - 'ALTER': 'query_language/alter.md' - - 'SYSTEM': 'query_language/system.md' - - 'Прочие виды запросов': 'query_language/misc.md' + - 'Общий синтаксис': 'query_language/syntax.md' + - 'Запросы': + - 'SELECT': 'query_language/select.md' + - 'INSERT INTO': 'query_language/insert_into.md' + - 'CREATE': 'query_language/create.md' + - 'ALTER': 'query_language/alter.md' + - 'SYSTEM': 'query_language/system.md' + - 'SHOW': 'query_language/show.md' + - 'Прочие': 'query_language/misc.md' - 'Функции': - 'Введение': 'query_language/functions/index.md' - 'Арифметические функции': 'query_language/functions/arithmetic_functions.md' @@ -173,7 +150,39 @@ nav: - 'Ключ и поля словаря': 'query_language/dicts/external_dicts_dict_structure.md' - 'Встроенные словари': 'query_language/dicts/internal_dicts.md' - 'Операторы': 'query_language/operators.md' - - 'Общий синтаксис': 'query_language/syntax.md' + - 'Типы данных': + - 'Введение': 'data_types/index.md' + - 'UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64': 'data_types/int_uint.md' + - 'Float32, Float64': 'data_types/float.md' + - 'Decimal': 'data_types/decimal.md' + - 'Булевы значения': 'data_types/boolean.md' + - 'String': 'data_types/string.md' + - 'FixedString(N)': 'data_types/fixedstring.md' + - 'UUID': 'data_types/uuid.md' + - 'Date': 'data_types/date.md' + - 'DateTime': 'data_types/datetime.md' + - 'Enum': 'data_types/enum.md' + - 'Array(T)': 'data_types/array.md' + - 'AggregateFunction(name, types_of_arguments...)': 'data_types/nested_data_structures/aggregatefunction.md' + - 'Tuple(T1, T2, ...)': 'data_types/tuple.md' + - 'Nullable': 'data_types/nullable.md' + - 'Вложенные структуры данных': + - 'hidden': 'data_types/nested_data_structures/index.md' + - 'Nested(Name1 Type1, Name2 Type2, ...)': 'data_types/nested_data_structures/nested.md' + - 'Служебные типы данных': + - 'hidden': 'data_types/special_data_types/index.md' + - 'Expression': 'data_types/special_data_types/expression.md' + - 'Set': 'data_types/special_data_types/set.md' + - 'Nothing': 'data_types/special_data_types/nothing.md' + - 'Interval': 'data_types/special_data_types/interval.md' + - 'Domains': + - 'Overview': 'data_types/domains/overview.md' + - 'IPv4': 'data_types/domains/ipv4.md' + - 'IPv6': 'data_types/domains/ipv6.md' + +- 'Руководства': + - 'Обзор': 'guides/index.md' + - 'Применение CatBoost моделей': 'guides/apply_catboost_model.md' - 'Эксплуатация': - 'Введение': 'operations/index.md' @@ -203,24 +212,22 @@ nav: - 'clickhouse-copier': 'operations/utils/clickhouse-copier.md' - 'clickhouse-local': 'operations/utils/clickhouse-local.md' -- 'Руководства': - - 'Обзор': 'guides/index.md' - - 'Применение CatBoost моделей': 'guides/apply_catboost_model.md' - -- 'F.A.Q.': - - 'Общие вопросы': 'faq/general.md' - - 'Разработка': - 'hidden': 'development/index.md' - 'Обзор архитектуры ClickHouse': 'development/architecture.md' - 'Как собрать ClickHouse на Linux': 'development/build.md' - 'Как собрать ClickHouse на Mac OS X': 'development/build_osx.md' - - 'Как собрать ClickHouse на Linux для Mac OS X': 'development/build_cross.md' + - 'Как собрать ClickHouse на Linux для Mac OS X': 'development/build_cross_osx.md' - 'Как писать код на C++': 'development/style.md' - 'Как запустить тесты': 'development/tests.md' + - 'Инструкция для начинающего разработчика ClickHouse': 'development/developer_instruction.md' - 'Сторонние библиотеки': 'development/contrib.md' - 'Что нового': - 'Changelog': 'changelog.md' - 'Security changelog': 'security_changelog.md' - 'Roadmap': 'roadmap.md' + - 'Подробный roadmap 2020': 'extended_roadmap.md' + +- 'F.A.Q.': + - 'Общие вопросы': 'faq/general.md' diff --git a/docs/toc_zh.yml b/docs/toc_zh.yml index 8a40a1fb133..7395dcfe145 100644 --- a/docs/toc_zh.yml +++ b/docs/toc_zh.yml @@ -5,10 +5,12 @@ nav: - 'ClickHouse的独特功能': 'introduction/distinctive_features.md' - 'ClickHouse功能可被视为缺点': 'introduction/features_considered_disadvantages.md' - '性能': 'introduction/performance.md' - - 'Yandex.Metrica使用案例': 'introduction/ya_metrika_task.md' + - '历史': 'introduction/history.md' - '入门指南': - - '部署运行': 'getting_started/index.md' + - 'hidden': 'getting_started/index.md' + - '安装': 'getting_started/install.md' + - '教程': 'getting_started/tutorial.md' - '示例数据集': - '航班飞行数据': 'getting_started/example_datasets/ontime.md' - '纽约市出租车数据': 'getting_started/example_datasets/nyc_taxi.md' @@ -16,6 +18,7 @@ nav: - '维基访问数据': 'getting_started/example_datasets/wikistat.md' - 'Criteo TB级别点击日志': 'getting_started/example_datasets/criteo.md' - 'Star Schema基准测试': 'getting_started/example_datasets/star_schema.md' + - 'Yandex.Metrica': 'getting_started/example_datasets/metrica.md' - '客户端': - '介绍': 'interfaces/index.md' @@ -25,6 +28,7 @@ nav: - '输入输出格式': 'interfaces/formats.md' - 'JDBC 驱动': 'interfaces/jdbc.md' - 'ODBC 驱动': 'interfaces/odbc.md' + - 'C ++客户端库': 'interfaces/cpp.md' - '第三方': - '客户端库': 'interfaces/third-party/client_libraries.md' - '集成': 'interfaces/third-party/integrations.md' @@ -55,6 +59,7 @@ nav: - 'Expression': 'data_types/special_data_types/expression.md' - 'Set': 'data_types/special_data_types/set.md' - 'Nothing': 'data_types/special_data_types/nothing.md' + - 'Interval': 'data_types/special_data_types/interval.md' - 'Domain类型': - '介绍': 'data_types/domains/overview.md' - 'IPv4': 'data_types/domains/ipv4.md' @@ -143,6 +148,7 @@ nav: - 'GEO函数': 'query_language/functions/geo.md' - 'Nullable处理函数': 'query_language/functions/functions_for_nulls.md' - '机器学习函数': 'query_language/functions/machine_learning_functions.md' + - 'Introspection': 'query_language/functions/introspection.md' - '其他函数': 'query_language/functions/other_functions.md' - '聚合函数': - '介绍': 'query_language/agg_functions/index.md' @@ -210,9 +216,10 @@ nav: - 'ClickHouse架构概述': 'development/architecture.md' - '如何在Linux中编译ClickHouse': 'development/build.md' - '如何在Mac OS X中编译ClickHouse': 'development/build_osx.md' - - 'How to Build ClickHouse on Linux for Mac OS X': 'development/build_cross.md' + - '如何在Linux中编译Mac OS X ClickHouse': 'development/build_cross_osx.md' - '如何编写C++代码': 'development/style.md' - '如何运行ClickHouse测试': 'development/tests.md' + - '开发者指南': 'development/developer_instruction.md' - '使用的第三方库': 'development/contrib.md' - '新功能特性': diff --git a/docs/tools/build.py b/docs/tools/build.py index 72a16839bef..0e855ce9f1e 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -45,6 +45,9 @@ def build_for_lang(lang, args): os.environ['SINGLE_PAGE'] = '0' config_path = os.path.join(args.docs_dir, 'toc_%s.yml' % lang) + if args.is_stable_release and not os.path.exists(config_path): + logging.warn('Skipping %s docs, because %s does not exist' % (lang, config_path)) + return try: theme_cfg = { @@ -74,6 +77,7 @@ def build_for_lang(lang, args): 'en': 'ClickHouse %s Documentation', 'ru': 'Документация ClickHouse %s', 'zh': 'ClickHouse文档 %s', + 'ja': 'ClickHouseドキュメント %s', 'fa': 'مستندات %sClickHouse' } @@ -92,8 +96,8 @@ def build_for_lang(lang, args): theme=theme_cfg, copyright='©2016–2019 Yandex LLC', use_directory_urls=True, - repo_name='yandex/ClickHouse', - repo_url='https://github.com/yandex/ClickHouse/', + repo_name='ClickHouse/ClickHouse', + repo_url='https://github.com/ClickHouse/ClickHouse/', edit_uri='edit/master/docs/%s' % lang, extra_css=['assets/stylesheets/custom.css'], markdown_extensions=[ @@ -241,13 +245,14 @@ if __name__ == '__main__': os.chdir(os.path.join(os.path.dirname(__file__), '..')) arg_parser = argparse.ArgumentParser() - arg_parser.add_argument('--lang', default='en,ru,zh,fa') + arg_parser.add_argument('--lang', default='en,ru,zh,ja,fa') arg_parser.add_argument('--docs-dir', default='.') arg_parser.add_argument('--theme-dir', default='mkdocs-material-theme') arg_parser.add_argument('--website-dir', default=os.path.join('..', 'website')) arg_parser.add_argument('--output-dir', default='build') arg_parser.add_argument('--enable-stable-releases', action='store_true') arg_parser.add_argument('--version-prefix', type=str, default='') + arg_parser.add_argument('--is-stable-release', action='store_true') arg_parser.add_argument('--skip-single-page', action='store_true') arg_parser.add_argument('--skip-pdf', action='store_true') arg_parser.add_argument('--skip-website', action='store_true') @@ -259,8 +264,6 @@ if __name__ == '__main__': from github import choose_latest_releases args.stable_releases = choose_latest_releases() if args.enable_stable_releases else [] - - logging.basicConfig( level=logging.DEBUG if args.verbose else logging.INFO, diff --git a/docs/tools/concatenate.py b/docs/tools/concatenate.py index 4eb8fcf9562..4ff9f9214df 100755 --- a/docs/tools/concatenate.py +++ b/docs/tools/concatenate.py @@ -11,11 +11,8 @@ def concatenate(lang, docs_path, single_page_file): az_re = re.compile(r'[a-z]') with open(proj_config) as cfg_file: - files_to_concatenate = [] - for l in cfg_file: - if '.md' in l and 'single_page' not in l: - path = (l[l.index(':') + 1:]).strip(" '\n") - files_to_concatenate.append(path) + files_to_concatenate = [(l[l.index(':') + 1:]).strip(" '\n") for l in cfg_file + if '.md' in l and 'single_page' not in l] logging.info( str(len(files_to_concatenate)) + diff --git a/docs/tools/easy_diff.py b/docs/tools/easy_diff.py new file mode 100755 index 00000000000..2c7b5429994 --- /dev/null +++ b/docs/tools/easy_diff.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os, sys +import argparse +import subprocess +import contextlib +from git import cmd +from tempfile import NamedTemporaryFile + +SCRIPT_DESCRIPTION = ''' + usage: ./easy_diff.py language/document path + + Show the difference between a language document and an English document. + + This script is based on the assumption that documents in other languages are fully synchronized with the en document at a commit. + + For example: + Execute: + ./easy_diff.py --no-pager zh/data_types + Output: + Need translate document:~/ClickHouse/docs/en/data_types/uuid.md + Need link document:~/ClickHouse/docs/en/data_types/decimal.md to ~/ClickHouse/docs/zh/data_types/decimal.md + diff --git a/docs/en/data_types/domains/ipv6.md b/docs/en/data_types/domains/ipv6.md + index 1bfbe3400b..e2abaff017 100644 + --- a/docs/en/data_types/domains/ipv6.md + +++ b/docs/en/data_types/domains/ipv6.md + @@ -4,13 +4,13 @@ + + ### Basic Usage + + -``` sql + +```sql + CREATE TABLE hits (url String, from IPv6) ENGINE = MergeTree() ORDER BY url; + + DESCRIBE TABLE hits; + ``` + + -``` + +```text + ┌─name─┬─type───┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┐ + │ url │ String │ │ │ │ │ + │ from │ IPv6 │ │ │ │ │ + @@ -19,19 +19,19 @@ DESCRIBE TABLE hits; + + OR you can use `IPv6` domain as a key: + + -``` sql + +```sql + CREATE TABLE hits (url String, from IPv6) ENGINE = MergeTree() ORDER BY from; + ... MORE + + OPTIONS: + -h, --help show this help message and exit + --no-pager use stdout as difference result output +''' + +SCRIPT_PATH = os.path.abspath(__file__) +CLICKHOUSE_REPO_HOME = os.path.join(os.path.dirname(SCRIPT_PATH), '..', '..') +SCRIPT_COMMAND_EXECUTOR = cmd.Git(CLICKHOUSE_REPO_HOME) + +SCRIPT_COMMAND_PARSER = argparse.ArgumentParser(add_help=False) +SCRIPT_COMMAND_PARSER.add_argument('path', type=bytes, nargs='?', default=None) +SCRIPT_COMMAND_PARSER.add_argument('--no-pager', action='store_true', default=False) +SCRIPT_COMMAND_PARSER.add_argument('-h', '--help', action='store_true', default=False) + + +def execute(commands): + return SCRIPT_COMMAND_EXECUTOR.execute(commands) + + +def get_hash(file_name): + return execute(['git', 'log', '-n', '1', '--pretty=format:"%H"', file_name]) + + +def diff_file(reference_file, working_file, out): + if not os.path.exists(reference_file): + raise RuntimeError('reference file [' + os.path.abspath(reference_file) + '] is not exists.') + + if os.path.islink(working_file): + out.writelines(["Need translate document:" + os.path.abspath(reference_file)]) + elif not os.path.exists(working_file): + out.writelines(['Need link document ' + os.path.abspath(reference_file) + ' to ' + os.path.abspath(working_file)]) + elif get_hash(working_file) != get_hash(reference_file): + out.writelines([(execute(['git', 'diff', get_hash(working_file).strip('"'), reference_file]).encode('utf-8'))]) + + return 0 + + +def diff_directory(reference_directory, working_directory, out): + if not os.path.isdir(reference_directory): + return diff_file(reference_directory, working_directory, out) + + for list_item in os.listdir(reference_directory): + working_item = os.path.join(working_directory, list_item) + reference_item = os.path.join(reference_directory, list_item) + if diff_file(reference_item, working_item, out) if os.path.isfile(reference_item) else diff_directory(reference_item, working_item, out) != 0: + return 1 + + return 0 + + +def find_language_doc(custom_document, other_language='en', children=[]): + if len(custom_document) == 0: + raise RuntimeError('The ' + os.path.join(custom_document, *children) + " is not in docs directory.") + + if os.path.samefile(os.path.join(CLICKHOUSE_REPO_HOME, 'docs'), custom_document): + return os.path.join(CLICKHOUSE_REPO_HOME, 'docs', other_language, *children[1:]) + children.insert(0, os.path.split(custom_document)[1]) + return find_language_doc(os.path.split(custom_document)[0], other_language, children) + + +class ToPager: + def __init__(self, temp_named_file): + self.temp_named_file = temp_named_file + + def writelines(self, lines): + self.temp_named_file.writelines(lines) + + def close(self): + self.temp_named_file.flush() + git_pager = execute(['git', 'var', 'GIT_PAGER']) + subprocess.check_call([git_pager, self.temp_named_file.name]) + self.temp_named_file.close() + + +class ToStdOut: + def writelines(self, lines): + self.system_stdout_stream.writelines(lines) + + def close(self): + self.system_stdout_stream.flush() + + def __init__(self, system_stdout_stream): + self.system_stdout_stream = system_stdout_stream + + +if __name__ == '__main__': + arguments = SCRIPT_COMMAND_PARSER.parse_args() + if arguments.help or not arguments.path: + sys.stdout.write(SCRIPT_DESCRIPTION) + sys.exit(0) + + working_language = os.path.join(CLICKHOUSE_REPO_HOME, 'docs', arguments.path) + with contextlib.closing(ToStdOut(sys.stdout) if arguments.no_pager else ToPager(NamedTemporaryFile('r+'))) as writer: + exit(diff_directory(find_language_doc(working_language), working_language, writer)) diff --git a/docs/tools/easy_edit.sh b/docs/tools/easy_edit.sh index 28c38453d0d..ed8a43fead7 100755 --- a/docs/tools/easy_edit.sh +++ b/docs/tools/easy_edit.sh @@ -14,7 +14,7 @@ popd rm -rf "${EDIT_DIR}" || true -for DOCS_LANG in en ru zh fa +for DOCS_LANG in en ru zh ja fa do for ARTICLE in ${ARTICLES} do diff --git a/docs/tools/github.py b/docs/tools/github.py index e07d8a0683a..d92dfe7435b 100644 --- a/docs/tools/github.py +++ b/docs/tools/github.py @@ -15,7 +15,7 @@ def choose_latest_releases(): candidates = requests.get('https://api.github.com/repos/ClickHouse/ClickHouse/tags?per_page=100').json() for tag in candidates: name = tag.get('name', '') - if 'v18' in name or 'stable' not in name: + if ('v18' in name) or ('stable' not in name) or ('prestable' in name): continue major_version = '.'.join((name.split('.', 2))[:2]) if major_version not in seen: @@ -33,6 +33,7 @@ def process_release(args, callback, release): tar.extractall(base_dir) args = copy.deepcopy(args) args.version_prefix = name + args.is_stable_release = True args.docs_dir = os.path.join(base_dir, os.listdir(base_dir)[0], 'docs') callback(args) diff --git a/docs/tools/make_links.sh b/docs/tools/make_links.sh index cca2f5feb6b..04c51424ec8 100755 --- a/docs/tools/make_links.sh +++ b/docs/tools/make_links.sh @@ -6,12 +6,12 @@ function do_make_links() { - langs=(en ru fa zh) + langs=(en ru zh ja fa) src_file="$1" for lang in "${langs[@]}" do # replacing "/./" with / - dst_file="../${lang}/${src_file}" + dst_file="../${lang}${src_file}" dst_file="${dst_file/\/\.\//\/}" mkdir -p $(dirname "${dst_file}") diff --git a/docs/tools/mkdocs-material-theme/assets/flags/ja.svg b/docs/tools/mkdocs-material-theme/assets/flags/ja.svg new file mode 100644 index 00000000000..a666c272523 --- /dev/null +++ b/docs/tools/mkdocs-material-theme/assets/flags/ja.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/docs/tools/mkdocs-material-theme/assets/javascripts/lunr/lunr.ja.js b/docs/tools/mkdocs-material-theme/assets/javascripts/lunr/lunr.ja.js new file mode 120000 index 00000000000..c20586be8a3 --- /dev/null +++ b/docs/tools/mkdocs-material-theme/assets/javascripts/lunr/lunr.ja.js @@ -0,0 +1 @@ +lunr.jp.js \ No newline at end of file diff --git a/docs/tools/mkdocs-material-theme/partials/flags.html b/docs/tools/mkdocs-material-theme/partials/flags.html index 26d6cdd8f9f..c7b06fbc4d0 100644 --- a/docs/tools/mkdocs-material-theme/partials/flags.html +++ b/docs/tools/mkdocs-material-theme/partials/flags.html @@ -1,4 +1,4 @@ -{% set alt_langs = [['en', 'English'], ['ru', 'Russian'], ['zh', 'Chinese'], ['fa', 'Farsi']] %} +{% set alt_langs = [['en', 'English'], ['ru', 'Russian'], ['zh', 'Chinese'], ['ja', 'Japanese'], ['fa', 'Farsi']] %} {% for alt_lang, alt_title in alt_langs %} diff --git a/docs/zh/data_types/special_data_types/interval.md b/docs/zh/data_types/special_data_types/interval.md new file mode 120000 index 00000000000..6829f5ced00 --- /dev/null +++ b/docs/zh/data_types/special_data_types/interval.md @@ -0,0 +1 @@ +../../../en/data_types/special_data_types/interval.md \ No newline at end of file diff --git a/docs/zh/development/build.md b/docs/zh/development/build.md index a1408fae987..3a3cdfd1b12 100644 --- a/docs/zh/development/build.md +++ b/docs/zh/development/build.md @@ -24,15 +24,9 @@ cd ClickHouse 以下教程是在 Ubuntu Linux 中进行编译的示例。 通过适当的更改，它应该可以适用于任何其他的 Linux 发行版。 -仅支持具有 SSE 4.2的 x86_64。 对 AArch64 的支持是实验性的。 +仅支持具有 x86_64、AArch64。 对 Power9 的支持是实验性的。 -测试是否支持 SSE 4.2，执行： - -```bash -grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" -``` - -## 安装 Git 和 CMake +## 安装 Git 和 CMake 和 Ninja ```bash sudo apt-get install git cmake ninja-build @@ -41,7 +35,7 @@ sudo apt-get install git cmake ninja-build Or cmake3 instead of cmake on older systems. 或者在早期版本的系统中用 cmake3 替代 cmake -## 安装 GCC 7 +## 安装 GCC 9 There are several ways to do this. @@ -51,24 +45,24 @@ There are several ways to do this. sudo apt-get install software-properties-common sudo apt-add-repository ppa:ubuntu-toolchain-r/test sudo apt-get update -sudo apt-get install gcc-7 g++-7 +sudo apt-get install gcc-9 g++-9 ``` ### 源码安装 gcc 请查看 [utils/ci/build-gcc-from-sources.sh](https://github.com/yandex/ClickHouse/blob/master/utils/ci/build-gcc-from-sources.sh) -## 使用 GCC 7 来编译 +## 使用 GCC 9 来编译 ```bash -export CC=gcc-7 -export CXX=g++-7 +export CC=gcc-9 +export CXX=g++-9 ``` ## 安装所需的工具依赖库 ```bash -sudo apt-get install libicu-dev libreadline-dev +sudo apt-get install libreadline-dev ``` ## 拉取 ClickHouse 源码 diff --git a/docs/zh/development/build_cross.md b/docs/zh/development/build_cross.md deleted file mode 120000 index f595f252de3..00000000000 --- a/docs/zh/development/build_cross.md +++ /dev/null @@ -1 +0,0 @@ -../../en/development/build_cross.md \ No newline at end of file diff --git a/docs/en/development/build_cross.md b/docs/zh/development/build_cross_osx.md similarity index 60% rename from docs/en/development/build_cross.md rename to docs/zh/development/build_cross_osx.md index 15792120158..ef31386c9f6 100644 --- a/docs/en/development/build_cross.md +++ b/docs/zh/development/build_cross_osx.md @@ -1,20 +1,22 @@ -# How to Build ClickHouse on Linux for Mac OS X +# 如何在Linux中编译Mac OS X ClickHouse -The cross-build for Mac OS X is based on the Build instructions, follow them first. +Linux机器也可以编译运行在OS X系统的`clickhouse`二进制包，这可以用于在Linux上跑持续集成测试。如果要在Mac OS X上直接构建ClickHouse，请参考另外一篇指南： https://clickhouse.yandex/docs/zh/development/build_osx/ + +Mac OS X的交叉编译基于以下构建说明，请首先遵循它们。 # Install Clang-8 -Follow the instructions from https://apt.llvm.org/ for your Ubuntu or Debian setup. -For example the commands for Bionic are like: +按照https://apt.llvm.org/中的说明进行Ubuntu或Debian安装。 +例如，安装Bionic的命令如下： ```bash sudo echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" >> /etc/apt/sources.list sudo apt-get install clang-8 ``` -# Install Cross-Compilation Toolset +# 安装交叉编译工具集 -Let's remember the path where we install `cctools` as ${CCTOOLS} +我们假设安装 `cctools` 在 ${CCTOOLS} 路径下 ```bash mkdir ${CCTOOLS} @@ -35,7 +37,7 @@ wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOS tar xJf MacOSX10.14.sdk.tar.xz ``` -# Build ClickHouse +# 编译 ClickHouse ```bash cd ClickHouse @@ -48,4 +50,4 @@ CC=clang-8 CXX=clang++-8 cmake . -Bbuild-osx -DCMAKE_SYSTEM_NAME=Darwin \ ninja -C build-osx ``` -The resulting binary will have Mach-O executable format and can't be run on Linux. +生成的二进制文件将具有Mach-O可执行格式，并且不能在Linux上运行。 \ No newline at end of file diff --git a/docs/zh/development/contrib.md b/docs/zh/development/contrib.md deleted file mode 120000 index 4749f95f9ef..00000000000 --- a/docs/zh/development/contrib.md +++ /dev/null @@ -1 +0,0 @@ -../../en/development/contrib.md \ No newline at end of file diff --git a/docs/zh/development/contrib.md b/docs/zh/development/contrib.md new file mode 100644 index 00000000000..6c5535b0d4b --- /dev/null +++ b/docs/zh/development/contrib.md @@ -0,0 +1,34 @@ +# 使用的三方库 + +| Library | License | +| ------- | ------- | +| base64 | [BSD 2-Clause License](https://github.com/aklomp/base64/blob/a27c565d1b6c676beaf297fe503c4518185666f7/LICENSE) | +| boost | [Boost Software License 1.0](https://github.com/ClickHouse-Extras/boost-extra/blob/6883b40449f378019aec792f9983ce3afc7ff16e/LICENSE_1_0.txt) | +| brotli | [MIT](https://github.com/google/brotli/blob/master/LICENSE) | +| capnproto | [MIT](https://github.com/capnproto/capnproto/blob/master/LICENSE) | +| cctz | [Apache License 2.0](https://github.com/google/cctz/blob/4f9776a310f4952454636363def82c2bf6641d5f/LICENSE.txt) | +| double-conversion | [BSD 3-Clause License](https://github.com/google/double-conversion/blob/cf2f0f3d547dc73b4612028a155b80536902ba02/LICENSE) | +| FastMemcpy | [MIT](https://github.com/yandex/ClickHouse/blob/master/libs/libmemcpy/impl/LICENSE) | +| googletest | [BSD 3-Clause License](https://github.com/google/googletest/blob/master/LICENSE) | +| hyperscan | [BSD 3-Clause License](https://github.com/intel/hyperscan/blob/master/LICENSE) | +| libbtrie | [BSD 2-Clause License](https://github.com/yandex/ClickHouse/blob/master/contrib/libbtrie/LICENSE) | +| libcxxabi | [BSD + MIT](https://github.com/yandex/ClickHouse/blob/master/libs/libglibc-compatibility/libcxxabi/LICENSE.TXT) | +| libdivide | [Zlib License](https://github.com/yandex/ClickHouse/blob/master/contrib/libdivide/LICENSE.txt) | +| libgsasl | [LGPL v2.1](https://github.com/ClickHouse-Extras/libgsasl/blob/3b8948a4042e34fb00b4fb987535dc9e02e39040/LICENSE) +| libhdfs3 | [Apache License 2.0](https://github.com/ClickHouse-Extras/libhdfs3/blob/bd6505cbb0c130b0db695305b9a38546fa880e5a/LICENSE.txt) | +| libmetrohash | [Apache License 2.0](https://github.com/yandex/ClickHouse/blob/master/contrib/libmetrohash/LICENSE) | +| libpcg-random | [Apache License 2.0](https://github.com/yandex/ClickHouse/blob/master/contrib/libpcg-random/LICENSE-APACHE.txt) | +| libressl | [OpenSSL License](https://github.com/ClickHouse-Extras/ssl/blob/master/COPYING) | +| librdkafka | [BSD 2-Clause License](https://github.com/edenhill/librdkafka/blob/363dcad5a23dc29381cc626620e68ae418b3af19/LICENSE) | +| libwidechar\_width | [CC0 1.0 Universal](https://github.com/yandex/ClickHouse/blob/master/libs/libwidechar_width/LICENSE) | +| llvm | [BSD 3-Clause License](https://github.com/ClickHouse-Extras/llvm/blob/163def217817c90fb982a6daf384744d8472b92b/llvm/LICENSE.TXT) | +| lz4 | [BSD 2-Clause License](https://github.com/lz4/lz4/blob/c10863b98e1503af90616ae99725ecd120265dfb/LICENSE) | +| mariadb-connector-c | [LGPL v2.1](https://github.com/ClickHouse-Extras/mariadb-connector-c/blob/3.1/COPYING.LIB) | +| murmurhash | [Public Domain](https://github.com/yandex/ClickHouse/blob/master/contrib/murmurhash/LICENSE) +| pdqsort | [Zlib License](https://github.com/yandex/ClickHouse/blob/master/contrib/pdqsort/license.txt) | +| poco | [Boost Software License - Version 1.0](https://github.com/ClickHouse-Extras/poco/blob/fe5505e56c27b6ecb0dcbc40c49dc2caf4e9637f/LICENSE) | +| protobuf | [BSD 3-Clause License](https://github.com/ClickHouse-Extras/protobuf/blob/12735370922a35f03999afff478e1c6d7aa917a4/LICENSE) | +| re2 | [BSD 3-Clause License](https://github.com/google/re2/blob/7cf8b88e8f70f97fd4926b56aa87e7f53b2717e0/LICENSE) | +| UnixODBC | [LGPL v2.1](https://github.com/ClickHouse-Extras/UnixODBC/tree/b0ad30f7f6289c12b76f04bfb9d466374bb32168) | +| zlib-ng | [Zlib License](https://github.com/ClickHouse-Extras/zlib-ng/blob/develop/LICENSE.md) | +| zstd | [BSD 3-Clause License](https://github.com/facebook/zstd/blob/dev/LICENSE) | diff --git a/docs/zh/development/developer_instruction.md b/docs/zh/development/developer_instruction.md new file mode 100644 index 00000000000..3f257d5a58e --- /dev/null +++ b/docs/zh/development/developer_instruction.md @@ -0,0 +1,293 @@ +ClickHose支持Linux,FreeBSD 及 Mac OS X 系统。 + +# Windows使用指引 + +如果您的系统是Windows，则需要创建Ubuntu虚拟机。可以安装VirtualBox来构建虚拟机。Ubuntu的下载链接为：https://www.ubuntu.com/#download 。请使用下载好的镜像创建一个虚拟机（请确保虚拟机有至少4GB的内存容量）。在Ubuntu中使用"terminal"程序（gnome-terminal，konsole等）运行命令行终端，或使用快捷键Ctrl+Alt+T。 + + +# 在GitHub上创建源码库 + +您需要(申请)一个GitHub账户来使用ClickHouse。 + +如果没有账户，请在https://github.com上注册一个。如果没有SSH密钥，请在本地创建密钥并将公钥上传到GitHub上。这有助于你提交更新代码。并且在不同的SSH服务端，你也可以使用相同的SSH密钥。 + +要创建ClickHouse源码库的分支，请在https://github.com/ClickHouse/ClickHouse页面上点击右上角的"fork"按钮。它会在本账户上创建您个人的ClickHouse/ClickHouse分支。 + +若要参与开发，首先请在ClickHouse的分支中提交您期望的变更，然后创建一个“pull请求”，以便这些变更能够被(ClickHouse/ClickHouse)主库接受。 + +请先安装`git`来使用git源码库。 + +请在Ubuntu终端上使用下列的指令来安装`git`: + +``` +sudo apt update +sudo apt install git +``` + +在https://services.github.com/on-demand/downloads/github-git-cheat-sheet.pdf中找到有关使用Git的简易手册。有关Git的详细手册，请参见: https://git-scm.com/book/ru/v2 。 + + +# 拷贝源码库到开发机 + +接下来，请将源码下载到开发机上。这步操作被称为“拷贝源码库”，是因为它在您的开发机上创建了源码库的本地副本。 + +在终端命令行输入下列指令： +``` +git clone --recursive git@guthub.com:your_github_username/ClickHouse.git +cd ClickHouse +``` +请注意，您需要将*your_github_username* 替换成实际使用的账户名! + +这个指令将创建一个包含项目副本的`ClickHouse`工作目录。 + +重要的是，工作目录的路径中不应包含空格，因为这可能会导致运行构建系统时出现问题。 + +请注意，ClickHouse源码库使用了`submodules`。这是对其他库的引用（即项目所依赖的外部库）。即在拷贝源码库时，需要如上述指令中那样指定`--recursive`。如果在拷贝源码库时没有包含子模块，需要执行使用下列的指令： + +``` +git submodule init +git submodule update +``` +可以通过 `git submodule status`来检查子模块的状态。 + +如果提示下列的错误信息: + +``` +Permission denied (publickey). +fatal: Could not read from remote repository. + +Please make sure you have the correct access rights +and the repository exists. +``` +这通常表示缺少用于连接GitHub的SSH密钥。这些密钥一般都在`~/.ssh`中。要接受SSH密钥，请在GitHub UI的设置页面中上传它们。 + +您还可以通过https协议来拷贝源码库: + +``` +git clone https://github.com/ClickHouse/ClickHouse.git +``` + +但是，这无法将变更提交到服务器上。您仍然可以暂时使用，并后续再添加SSH密钥，用`git remote`命令替换源码库的远程地址。 + +还可以将原始ClickHouse库的地址添加到本地库中，以便从那里获取更新： + +``` +git remote add upstream git@github.com:ClickHouse/ClickHouse.git +``` + +命令执行成功后，可以通过执行`git pull upstream master`，从ClickHouse的主分支中拉去更新。 + + +# 构建系统 + +ClickHouse使用 CMake 和 Ninja 来构建系统。 + +CMake - 一个可以生成Ninja文件的元构建系统（构建任务）。 +Ninja - 一个轻量级的构建系统，专注于速度，用于执行这些cmake生成的任务。 + +在Ubuntu,Debian或者Mint系统上执行`sudo apt install cmake ninja-build`来安装ninja。 + +在CentOS,RedHat系统上执行`sudo yum install cmake ninja-build`。 + +如果您曾经使用过Arch或Gentoo，那么也许知道如何安装CMake。 + +若要在Mac OS X上安装CMake和Ninja，请先安装Homebrew，然后再通过brew安装其他内容： + +``` +/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +brew install cmake ninja +``` + +接下来，检查CMake的版本：`cmake --version`。如果版本低于3.3，则需要从以下网站安装更新版本：https://cmake.org/download/ 。 + + +# 可供选择的外部库 + +ClickHouse使用多个外部库进行构建。大多数外部库不需要单独安装，而是和ClickHouse一起在子模块中构建。可以查看`contrib`中罗列的清单。 + +有一些库不是由源构建的，而是由系统提供，例如：Readline，也建议安装。 + +Ubuntu: `sudo apt install libreadline-dev` + +Mac OS X: `brew install readline` + +但是，这些库本身都是可选的，ClickHouse即便没有它们也可以构建。ICU用于支持`ORDER BY`中的`COLLATE`(例如，对土耳其字母进行排序)。Readline用于在clickhouse-client中更便捷的指令输入。 + + +# C++ 编译器 + +GCC编译器从版本9开始，以及Clang版本>=8都可支持构建ClickHouse。 + +Yandex官方当前使用GCC构建ClickHouse，因为它生成的机器代码性能较好（根据测评，最多可以相差几个百分点）。Clang通常可以更加便捷的开发。我们的持续集成（CI）平台会运行大约十二种构建组合的检查。 + +在Ubuntu上安装GCC，请执行：`sudo apt install gcc g++` + +请使用`gcc --version`查看gcc的版本。如果gcc版本低于9，请参考此处的指示：https://clickhouse.yandex/docs/en/development/build/#install-gcc-9 。 + +在Mac OS X上安装GCC，请执行：`brew install gcc` + +如果您决定使用Clang，还可以同时安装 `libc++`以及`lld`，前提是您也熟悉它们。此外，也推荐使用`ccache`。 + + +# 构建的过程 + +如果当前已经准备好构建ClickHouse，我们建议您在`ClickHouse`中创建一个单独的目录`build`，其中包含所有构建组件: + +``` +mkdir build +cd build +``` + +您也可以有多个不同类型的构建目录（例如，build_release, build_debug等等)。 + +在`build`目录下，通过运行CMake配置构建。 在第一次运行之前，请定义用于指定编译器的环境变量（本示例中为gcc 9 编译器）。 + +``` +export CC=gcc-9 CXX=g++-9 +cmake .. +``` + +`CC`变量指代C的编译器（C Compiler的缩写），而`CXX`变量指代要使用哪个C++编译器进行编译。 + +为了更快的构建，请使用`debug`构建类型-不含优化的构建。为此提供以下的参数`-D CMAKE_BUILD_TYPE=Debug`: + +``` +cmake -D CMAKE_BUILD_TYPE=Debug .. +``` + +您可以通过在`build`目录中运行此命令来更改构建类型。 + +运行ninja进行构建: + +``` +ninja clickhouse-server clickhouse-client +``` + +在此示例中，仅将构建所需的二进制文件。 + +如果您需要构建所有的二进制文件（utilities和tests），请运行不带参数的ninja： + +``` +ninja +``` + +全量构建需要大约30GB的可用磁盘空间或15GB的空间来构建主要的二进制文件。 + +当构建的机器上有大量内存时，可考虑设置与`-j`参数并行运行的构建任务数量： + +``` +ninja -j 1 clickhouse-server clickhouse-client +``` + +在拥有4GB内存的机器上，建议设置成1，在拥有8GB内存的机器上，建议按`-j 2`设置。 + +如果您收到以下消息： + +`ninja：error：loading'build.ninja'：No such file or directory` + +则表示生成构建配置失败，请检查上述消息。 + +成功启动构建过程后，您将看到构建进度-已处理任务的数量和任务总数。 + +在libhdfs2库中生成有关protobuf文件的消息时，可能会显示诸如`libprotobuf WARNING`。它们没有影响，可以忽略不计。 + +成功构建后，会得到一个可执行文件`ClickHouse//dbms/programs/clickhouse`: + +``` +ls -l dbms/programs/clickhouse +``` + + +# 运行ClickHouse可执行文件 + +要以当前的用户身份运行服务，请进入到`ClickHouse/dbms/programs/server/` 目录（在`build`文件夹外）并运行： + +``` +../../../build/dbms/programs/clickhouse server +``` + +在这种情况下，ClickHouse将使用位于当前目录中的配置文件。您可以从任何目录运行`Clickhouse server`，并将配置文件`--config-file`的路径指定为命令行参数。 + +在另外一个终端上连接ClickHouse的clickhouse-client客户端，请进入到`ClickHouse/build/dbms/programs/` 并运行`clickhouse client`。 + +如果您在Mac OS X 或者 FreeBSD上收到`Connection refused`的消息，请尝试指定主机地址为127.0.0.1： + +``` +clickhouse client --host 127.0.0.1 +``` + +您可以使用自定义构建的ClickHouse二进制文件替换系统中安装的ClickHouse二进制文件的生成版本。为此，请参照官方网站上的说明在计算机上安装ClickHouse。 接下来，运行以下命令： + +``` +sudo service clickhouse-server stop +sudo cp ClickHouse/build/dbms/programs/clickhouse /usr/bin/ +sudo service clickhouse-server start +``` + +请注意，`clickhouse-client`，`clickhouse-server`和其他服务通常共享`clickhouse`二进制文件的符号链接。 + +您还可以使用系统上安装的ClickHouse软件包中的配置文件运行自定义构建的ClickHouse二进制文件： + +``` +sudo service clickhouse-server stop +sudo -u clickhouse ClickHouse/build/dbms/programs/clickhouse server --config-file /etc/clickhouse-server/config.xml +``` + + +# IDE (集成开发环境) + +如果您还不知道使用哪款IDE，我们推荐使用CLion。CLion是一款商业软件，但能够有30天的免费使用时间。它同时也对学生免费。CLion可以在Linux和Mac OS X上使用。 + +KDevelop和QTCreator是另外两款适合开发ClickHouse的替代IDE。尽管不太稳定，但KDevelop还是作为一款非常便捷的IDE。如果KDevelop在打开项目后不久崩溃，则您应该在打开项目文件列表后立即单击“全部停止”按钮。按此处理后，KDevelop可以正常使用。 + +作为简易的代码编辑器，您可以使用Sublime Text或Visual Studio Code或Kate（在Linux上都可用）。 + +值得一提的是CLion会创建自己的`build`路径，它还会自行选择`debug`作为构建类型。对于配置，它使用CLion中定义的CMake版本，而不是您安装的版本。最后，CLion会使用`make`而不是`ninja`去构建任务。这属于正常的现象，请记住这一点，以免造成混淆。 + + +# 编写代码 + +ClickHouse的架构描述可以在此处查看：https://clickhouse.yandex/docs/en/development/architecture/ + +代码风格指引：https://clickhouse.yandex/docs/en/development/style/ + +编写测试用例：https://clickhouse.yandex/docs/en/development/tests/ + +任务列表：https://github.com/yandex/ClickHouse/blob/master/dbms/tests/instructions/easy_tasks_sorted_en.md + + +# 测试数据 + +开发ClickHouse通常需要加载现实的数据集，尤其是在性能测试的场景。我们可以从Yandex.Metrica获取一组特别准备的匿名数据。这些数据需要额外使用3GB的空闲磁盘空间。请注意，完成大多数开发任务并不需要此数据。 + +``` +sudo apt install wget xz-utils + +wget https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz +wget https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz + +xz -v -d hits_v1.tsv.xz +xz -v -d visits_v1.tsv.xz + +clickhouse-client + +CREATE TABLE test.hits ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree PARTITION BY toYYYYMM(EventDate) SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID), EventTime); + +CREATE TABLE test.visits ( CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8, VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32, Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String, EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32, SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32, SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16, UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16, FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8, Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), Params Array(String), `Goals.ID` Array(UInt32), `Goals.Serial` Array(UInt32), `Goals.EventTime` Array(DateTime), `Goals.Price` Array(Int64), `Goals.OrderID` Array(String), `Goals.CurrencyID` Array(UInt32), WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64, ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32, ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32, ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32, ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16, ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime, PredLastVisit Date, LastVisit Date, TotalVisits UInt32, `TraficSource.ID` Array(Int8), `TraficSource.SearchEngineID` Array(UInt16), `TraficSource.AdvEngineID` Array(UInt8), `TraficSource.PlaceID` Array(UInt16), `TraficSource.SocialSourceNetworkID` Array(UInt8), `TraficSource.Domain` Array(String), `TraficSource.SearchPhrase` Array(String), `TraficSource.SocialSourcePage` Array(String), Attendance FixedString(16), CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64, StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64, OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64, UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), `Market.Type` Array(UInt8), `Market.GoalID` Array(UInt32), `Market.OrderID` Array(String), `Market.OrderPrice` Array(Int64), `Market.PP` Array(UInt32), `Market.DirectPlaceID` Array(UInt32), `Market.DirectOrderID` Array(UInt32), `Market.DirectBannerID` Array(UInt32), `Market.GoodID` Array(String), `Market.GoodName` Array(String), `Market.GoodQuantity` Array(Int32), `Market.GoodPrice` Array(Int64), IslandID FixedString(16)) ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) SAMPLE BY intHash32(UserID) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID); + +clickhouse-client --max_insert_block_size 100000 --query "INSERT INTO test.hits FORMAT TSV" < hits_v1.tsv +clickhouse-client --max_insert_block_size 100000 --query "INSERT INTO test.visits FORMAT TSV" < visits_v1.tsv +``` + + +# 创建拉取请求 + +进入到GitHub 用户界面中的fork库。如果您已经在某个分支中进行开发，则需要选择该分支。在屏幕中有一个 "拉取请求"的按钮。实际上这等价于“创建一个请求以接受对主库的变更”。 + +即使工作尚未完成，也可以创建拉取请求。在这种情况下，请在标题的开头加上“WIP”（正在进行中），以便后续更改。这对于协同审查和讨论更改以及运行所有可用测试用例很有用。提供有关变更的简短描述很重要，这将在后续用于生成重新发布变更日志。 + +Yandex成员一旦在您的拉取请求上贴上“可以测试”标签，就会开始测试。一些初始检查项（例如，代码类型）的结果会在几分钟内反馈。构建的检查结果将在半小时内完成。而主要的测试用例集结果将在一小时内报告给您。 + +系统将分别为您的拉取请求准备ClickHouse二进制版本。若要检索这些构建信息，请在检查列表中单击“ ClickHouse构建检查”旁边的“详细信息”链接。在这里，您会找到指向ClickHouse的.deb软件包的直接链接，此外，甚至可以将其部署在生产服务器上（如果您不担心）。 + +某些构建项很可能会在首次构建时失败。这是因为我们同时检查了基于gcc和clang的构建，几乎所有现有的被clang启用的警告（总是带有`-Werror`标志）。在同一页面上，您可以找到所有构建的日志，因此不必以所有可能的方式构建ClickHouse。 diff --git a/docs/zh/development/tests.md b/docs/zh/development/tests.md index 2b5fb7ca0e6..2861697fb0c 100644 --- a/docs/zh/development/tests.md +++ b/docs/zh/development/tests.md @@ -166,7 +166,7 @@ clickhouse benchmark --concurrency 16 < queries.tsv 当我们扩展 ClickHouse 网络协议时，我们手动测试旧的 clickhouse-client 与新的 clickhouse-server 和新的clickhouse-client 一起使用旧的 clickhouse-server (只需从相应的包中运行二进制文件) -## 来自编译器的帮助 +## 来自编译器的提示 ClickHouse 主要的代码 (位于`dbms`目录中) 使用 `-Wall -Wextra -Werror` 构建，并带有一些其他已启用的警告。 虽然没有为第三方库启用这些选项。 diff --git a/docs/zh/getting_started/example_datasets/amplab_benchmark.md b/docs/zh/getting_started/example_datasets/amplab_benchmark.md index 415457c9403..5afd7dfd705 100644 --- a/docs/zh/getting_started/example_datasets/amplab_benchmark.md +++ b/docs/zh/getting_started/example_datasets/amplab_benchmark.md @@ -7,21 +7,21 @@ 在控制台运行以下命令： ```bash -sudo apt-get install s3cmd -mkdir tiny; cd tiny; -s3cmd sync s3://big-data-benchmark/pavlo/text-deflate/tiny/ . -cd .. -mkdir 1node; cd 1node; -s3cmd sync s3://big-data-benchmark/pavlo/text-deflate/1node/ . -cd .. -mkdir 5nodes; cd 5nodes; -s3cmd sync s3://big-data-benchmark/pavlo/text-deflate/5nodes/ . -cd .. +$ sudo apt-get install s3cmd +$ mkdir tiny; cd tiny; +$ s3cmd sync s3://big-data-benchmark/pavlo/text-deflate/tiny/ . +$ cd .. +$ mkdir 1node; cd 1node; +$ s3cmd sync s3://big-data-benchmark/pavlo/text-deflate/1node/ . +$ cd .. +$ mkdir 5nodes; cd 5nodes; +$ s3cmd sync s3://big-data-benchmark/pavlo/text-deflate/5nodes/ . +$ cd .. ``` 在ClickHouse运行如下查询： -``` sql +```sql CREATE TABLE rankings_tiny ( pageURL String, @@ -86,12 +86,12 @@ CREATE TABLE uservisits_5nodes_on_single 回到控制台运行如下命令： ```bash -for i in tiny/rankings/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO rankings_tiny FORMAT CSV"; done -for i in tiny/uservisits/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO uservisits_tiny FORMAT CSV"; done -for i in 1node/rankings/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO rankings_1node FORMAT CSV"; done -for i in 1node/uservisits/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO uservisits_1node FORMAT CSV"; done -for i in 5nodes/rankings/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO rankings_5nodes_on_single FORMAT CSV"; done -for i in 5nodes/uservisits/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO uservisits_5nodes_on_single FORMAT CSV"; done +$ for i in tiny/rankings/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO rankings_tiny FORMAT CSV"; done +$ for i in tiny/uservisits/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO uservisits_tiny FORMAT CSV"; done +$ for i in 1node/rankings/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO rankings_1node FORMAT CSV"; done +$ for i in 1node/uservisits/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO uservisits_1node FORMAT CSV"; done +$ for i in 5nodes/rankings/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO rankings_5nodes_on_single FORMAT CSV"; done +$ for i in 5nodes/uservisits/*.deflate; do echo $i; zlib-flate -uncompress < $i | clickhouse-client --host=example-perftest01j --query="INSERT INTO uservisits_5nodes_on_single FORMAT CSV"; done ``` 简单的查询示例： diff --git a/docs/zh/getting_started/example_datasets/criteo.md b/docs/zh/getting_started/example_datasets/criteo.md index 9914bb8720c..3a86f630d7b 100644 --- a/docs/zh/getting_started/example_datasets/criteo.md +++ b/docs/zh/getting_started/example_datasets/criteo.md @@ -4,14 +4,14 @@ 创建原始数据对应的表结构： -``` sql +```sql CREATE TABLE criteo_log (date Date, clicked UInt8, int1 Int32, int2 Int32, int3 Int32, int4 Int32, int5 Int32, int6 Int32, int7 Int32, int8 Int32, int9 Int32, int10 Int32, int11 Int32, int12 Int32, int13 Int32, cat1 String, cat2 String, cat3 String, cat4 String, cat5 String, cat6 String, cat7 String, cat8 String, cat9 String, cat10 String, cat11 String, cat12 String, cat13 String, cat14 String, cat15 String, cat16 String, cat17 String, cat18 String, cat19 String, cat20 String, cat21 String, cat22 String, cat23 String, cat24 String, cat25 String, cat26 String) ENGINE = Log ``` 下载数据： ```bash -for i in {00..23}; do echo $i; zcat datasets/criteo/day_${i#0}.gz | sed -r 's/^/2000-01-'${i/00/24}'\t/' | clickhouse-client --host=example-perftest01j --query="INSERT INTO criteo_log FORMAT TabSeparated"; done +$ for i in {00..23}; do echo $i; zcat datasets/criteo/day_${i#0}.gz | sed -r 's/^/2000-01-'${i/00/24}'\t/' | clickhouse-client --host=example-perftest01j --query="INSERT INTO criteo_log FORMAT TabSeparated"; done ``` 创建转换后的数据对应的表结构： @@ -65,7 +65,7 @@ CREATE TABLE criteo 将第一张表中的原始数据转化写入到第二张表中去： -``` sql +```sql INSERT INTO criteo SELECT date, clicked, int1, int2, int3, int4, int5, int6, int7, int8, int9, int10, int11, int12, int13, reinterpretAsUInt32(unhex(cat1)) AS icat1, reinterpretAsUInt32(unhex(cat2)) AS icat2, reinterpretAsUInt32(unhex(cat3)) AS icat3, reinterpretAsUInt32(unhex(cat4)) AS icat4, reinterpretAsUInt32(unhex(cat5)) AS icat5, reinterpretAsUInt32(unhex(cat6)) AS icat6, reinterpretAsUInt32(unhex(cat7)) AS icat7, reinterpretAsUInt32(unhex(cat8)) AS icat8, reinterpretAsUInt32(unhex(cat9)) AS icat9, reinterpretAsUInt32(unhex(cat10)) AS icat10, reinterpretAsUInt32(unhex(cat11)) AS icat11, reinterpretAsUInt32(unhex(cat12)) AS icat12, reinterpretAsUInt32(unhex(cat13)) AS icat13, reinterpretAsUInt32(unhex(cat14)) AS icat14, reinterpretAsUInt32(unhex(cat15)) AS icat15, reinterpretAsUInt32(unhex(cat16)) AS icat16, reinterpretAsUInt32(unhex(cat17)) AS icat17, reinterpretAsUInt32(unhex(cat18)) AS icat18, reinterpretAsUInt32(unhex(cat19)) AS icat19, reinterpretAsUInt32(unhex(cat20)) AS icat20, reinterpretAsUInt32(unhex(cat21)) AS icat21, reinterpretAsUInt32(unhex(cat22)) AS icat22, reinterpretAsUInt32(unhex(cat23)) AS icat23, reinterpretAsUInt32(unhex(cat24)) AS icat24, reinterpretAsUInt32(unhex(cat25)) AS icat25, reinterpretAsUInt32(unhex(cat26)) AS icat26 FROM criteo_log; DROP TABLE criteo_log; diff --git a/docs/zh/getting_started/example_datasets/metrica.md b/docs/zh/getting_started/example_datasets/metrica.md new file mode 120000 index 00000000000..984023973eb --- /dev/null +++ b/docs/zh/getting_started/example_datasets/metrica.md @@ -0,0 +1 @@ +../../../en/getting_started/example_datasets/metrica.md \ No newline at end of file diff --git a/docs/zh/getting_started/example_datasets/nyc_taxi.md b/docs/zh/getting_started/example_datasets/nyc_taxi.md index 16adae18120..338ac5ba0b7 100644 --- a/docs/zh/getting_started/example_datasets/nyc_taxi.md +++ b/docs/zh/getting_started/example_datasets/nyc_taxi.md @@ -1,5 +1,10 @@ # 纽约市出租车数据 +纽约市出租车数据有以下两个方式获取： + +从原始数据导入 +下载预处理好的分区数据 + ## 怎样导入原始数据 可以参考和中的关于数据集结构描述与数据下载指令说明。 @@ -24,8 +29,8 @@ mv data/yellow_tripdata_2010-03.csv_ data/yellow_tripdata_2010-03.csv 您可以按如下方式检查下载的行数： -``` -time psql nyc-taxi-data -c "SELECT count(*) FROM trips;" +```bash +$ time psql nyc-taxi-data -c "SELECT count(*) FROM trips;" ## Count 1298979494 (1 row) @@ -39,7 +44,7 @@ PostgreSQL处理这些数据大概需要370GB的磁盘空间。 从PostgreSQL中导出数据： -``` sql +```sql COPY ( SELECT trips.id, @@ -114,7 +119,7 @@ COPY 在ClickHouse中创建临时表： -``` sql +```sql CREATE TABLE trips ( trip_id UInt32, @@ -173,8 +178,8 @@ dropoff_puma Nullable(String) 接下来,需要将字段转换为更正确的数据类型，并且在可能的情况下，消除NULL。 -``` -time clickhouse-client --query="INSERT INTO trips FORMAT TabSeparated" < trips.tsv +```bash +$ time clickhouse-client --query="INSERT INTO trips FORMAT TabSeparated" < trips.tsv real 75m56.214s ``` @@ -191,7 +196,7 @@ real 75m56.214s 创建表结构并写入数据： -``` +```sql CREATE TABLE trips_mergetree ENGINE = MergeTree(pickup_date, pickup_datetime, 8192) AS SELECT @@ -258,13 +263,10 @@ FROM trips 这个表需要使用126GB的磁盘空间。 +```sql +SELECT formatReadableSize(sum(bytes)) FROM system.parts WHERE table = 'trips_mergetree' AND active ``` -:) SELECT formatReadableSize(sum(bytes)) FROM system.parts WHERE table = 'trips_mergetree' AND active - -SELECT formatReadableSize(sum(bytes)) -FROM system.parts -WHERE (table = 'trips_mergetree') AND active - +```text ┌─formatReadableSize(sum(bytes))─┐ │ 126.18 GiB │ └────────────────────────────────┘ @@ -272,11 +274,26 @@ WHERE (table = 'trips_mergetree') AND active 除此之外，你还可以在MergeTree上运行OPTIMIZE查询来进行优化。但这不是必须的，因为即使在没有进行优化的情况下它的表现依然是很好的。 +## 下载预处理好的分区数据 + +```bash +$ curl -O https://clickhouse-datasets.s3.yandex.net/trips_mergetree/partitions/trips_mergetree.tar +$ tar xvf trips_mergetree.tar -C /var/lib/clickhouse # path to ClickHouse data directory +$ # check permissions of unpacked data, fix if required +$ sudo service clickhouse-server restart +$ clickhouse-client --query "select count(*) from datasets.trips_mergetree" +``` + +!!!info + 如果要运行下面的SQL查询，必须使用完整的表名， + `datasets.trips_mergetree`。 + + ## 单台服务器运行结果 Q1: -``` sql +```sql SELECT cab_type, count(*) FROM trips_mergetree GROUP BY cab_type ``` @@ -284,7 +301,7 @@ SELECT cab_type, count(*) FROM trips_mergetree GROUP BY cab_type Q2: -``` sql +```sql SELECT passenger_count, avg(total_amount) FROM trips_mergetree GROUP BY passenger_count ``` @@ -292,7 +309,7 @@ SELECT passenger_count, avg(total_amount) FROM trips_mergetree GROUP BY passenge Q3: -``` sql +```sql SELECT passenger_count, toYear(pickup_date) AS year, count(*) FROM trips_mergetree GROUP BY passenger_count, year ``` @@ -300,7 +317,7 @@ SELECT passenger_count, toYear(pickup_date) AS year, count(*) FROM trips_mergetr Q4: -``` sql +```sql SELECT passenger_count, toYear(pickup_date) AS year, round(trip_distance) AS distance, count(*) FROM trips_mergetree GROUP BY passenger_count, year, distance @@ -319,19 +336,19 @@ Two Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz, 16 physical kernels total,128 GiB 在每台服务器中运行： -``` +```sql CREATE TABLE default.trips_mergetree_third ( trip_id UInt32, vendor_id Enum8('1' = 1, '2' = 2, 'CMT' = 3, 'VTS' = 4, 'DDS' = 5, 'B02512' = 10, 'B02598' = 11, 'B02617' = 12, 'B02682' = 13, 'B02764' = 14), pickup_date Date, pickup_datetime DateTime, dropoff_date Date, dropoff_datetime DateTime, store_and_fwd_flag UInt8, rate_code_id UInt8, pickup_longitude Float64, pickup_latitude Float64, dropoff_longitude Float64, dropoff_latitude Float64, passenger_count UInt8, trip_distance Float64, fare_amount Float32, extra Float32, mta_tax Float32, tip_amount Float32, tolls_amount Float32, ehail_fee Float32, improvement_surcharge Float32, total_amount Float32, payment_type_ Enum8('UNK' = 0, 'CSH' = 1, 'CRE' = 2, 'NOC' = 3, 'DIS' = 4), trip_type UInt8, pickup FixedString(25), dropoff FixedString(25), cab_type Enum8('yellow' = 1, 'green' = 2, 'uber' = 3), pickup_nyct2010_gid UInt8, pickup_ctlabel Float32, pickup_borocode UInt8, pickup_boroname Enum8('' = 0, 'Manhattan' = 1, 'Bronx' = 2, 'Brooklyn' = 3, 'Queens' = 4, 'Staten Island' = 5), pickup_ct2010 FixedString(6), pickup_boroct2010 FixedString(7), pickup_cdeligibil Enum8(' ' = 0, 'E' = 1, 'I' = 2), pickup_ntacode FixedString(4), pickup_ntaname Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195), pickup_puma UInt16, dropoff_nyct2010_gid UInt8, dropoff_ctlabel Float32, dropoff_borocode UInt8, dropoff_boroname Enum8('' = 0, 'Manhattan' = 1, 'Bronx' = 2, 'Brooklyn' = 3, 'Queens' = 4, 'Staten Island' = 5), dropoff_ct2010 FixedString(6), dropoff_boroct2010 FixedString(7), dropoff_cdeligibil Enum8(' ' = 0, 'E' = 1, 'I' = 2), dropoff_ntacode FixedString(4), dropoff_ntaname Enum16('' = 0, 'Airport' = 1, 'Allerton-Pelham Gardens' = 2, 'Annadale-Huguenot-Prince\'s Bay-Eltingville' = 3, 'Arden Heights' = 4, 'Astoria' = 5, 'Auburndale' = 6, 'Baisley Park' = 7, 'Bath Beach' = 8, 'Battery Park City-Lower Manhattan' = 9, 'Bay Ridge' = 10, 'Bayside-Bayside Hills' = 11, 'Bedford' = 12, 'Bedford Park-Fordham North' = 13, 'Bellerose' = 14, 'Belmont' = 15, 'Bensonhurst East' = 16, 'Bensonhurst West' = 17, 'Borough Park' = 18, 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' = 19, 'Briarwood-Jamaica Hills' = 20, 'Brighton Beach' = 21, 'Bronxdale' = 22, 'Brooklyn Heights-Cobble Hill' = 23, 'Brownsville' = 24, 'Bushwick North' = 25, 'Bushwick South' = 26, 'Cambria Heights' = 27, 'Canarsie' = 28, 'Carroll Gardens-Columbia Street-Red Hook' = 29, 'Central Harlem North-Polo Grounds' = 30, 'Central Harlem South' = 31, 'Charleston-Richmond Valley-Tottenville' = 32, 'Chinatown' = 33, 'Claremont-Bathgate' = 34, 'Clinton' = 35, 'Clinton Hill' = 36, 'Co-op City' = 37, 'College Point' = 38, 'Corona' = 39, 'Crotona Park East' = 40, 'Crown Heights North' = 41, 'Crown Heights South' = 42, 'Cypress Hills-City Line' = 43, 'DUMBO-Vinegar Hill-Downtown Brooklyn-Boerum Hill' = 44, 'Douglas Manor-Douglaston-Little Neck' = 45, 'Dyker Heights' = 46, 'East Concourse-Concourse Village' = 47, 'East Elmhurst' = 48, 'East Flatbush-Farragut' = 49, 'East Flushing' = 50, 'East Harlem North' = 51, 'East Harlem South' = 52, 'East New York' = 53, 'East New York (Pennsylvania Ave)' = 54, 'East Tremont' = 55, 'East Village' = 56, 'East Williamsburg' = 57, 'Eastchester-Edenwald-Baychester' = 58, 'Elmhurst' = 59, 'Elmhurst-Maspeth' = 60, 'Erasmus' = 61, 'Far Rockaway-Bayswater' = 62, 'Flatbush' = 63, 'Flatlands' = 64, 'Flushing' = 65, 'Fordham South' = 66, 'Forest Hills' = 67, 'Fort Greene' = 68, 'Fresh Meadows-Utopia' = 69, 'Ft. Totten-Bay Terrace-Clearview' = 70, 'Georgetown-Marine Park-Bergen Beach-Mill Basin' = 71, 'Glen Oaks-Floral Park-New Hyde Park' = 72, 'Glendale' = 73, 'Gramercy' = 74, 'Grasmere-Arrochar-Ft. Wadsworth' = 75, 'Gravesend' = 76, 'Great Kills' = 77, 'Greenpoint' = 78, 'Grymes Hill-Clifton-Fox Hills' = 79, 'Hamilton Heights' = 80, 'Hammels-Arverne-Edgemere' = 81, 'Highbridge' = 82, 'Hollis' = 83, 'Homecrest' = 84, 'Hudson Yards-Chelsea-Flatiron-Union Square' = 85, 'Hunters Point-Sunnyside-West Maspeth' = 86, 'Hunts Point' = 87, 'Jackson Heights' = 88, 'Jamaica' = 89, 'Jamaica Estates-Holliswood' = 90, 'Kensington-Ocean Parkway' = 91, 'Kew Gardens' = 92, 'Kew Gardens Hills' = 93, 'Kingsbridge Heights' = 94, 'Laurelton' = 95, 'Lenox Hill-Roosevelt Island' = 96, 'Lincoln Square' = 97, 'Lindenwood-Howard Beach' = 98, 'Longwood' = 99, 'Lower East Side' = 100, 'Madison' = 101, 'Manhattanville' = 102, 'Marble Hill-Inwood' = 103, 'Mariner\'s Harbor-Arlington-Port Ivory-Graniteville' = 104, 'Maspeth' = 105, 'Melrose South-Mott Haven North' = 106, 'Middle Village' = 107, 'Midtown-Midtown South' = 108, 'Midwood' = 109, 'Morningside Heights' = 110, 'Morrisania-Melrose' = 111, 'Mott Haven-Port Morris' = 112, 'Mount Hope' = 113, 'Murray Hill' = 114, 'Murray Hill-Kips Bay' = 115, 'New Brighton-Silver Lake' = 116, 'New Dorp-Midland Beach' = 117, 'New Springville-Bloomfield-Travis' = 118, 'North Corona' = 119, 'North Riverdale-Fieldston-Riverdale' = 120, 'North Side-South Side' = 121, 'Norwood' = 122, 'Oakland Gardens' = 123, 'Oakwood-Oakwood Beach' = 124, 'Ocean Hill' = 125, 'Ocean Parkway South' = 126, 'Old Astoria' = 127, 'Old Town-Dongan Hills-South Beach' = 128, 'Ozone Park' = 129, 'Park Slope-Gowanus' = 130, 'Parkchester' = 131, 'Pelham Bay-Country Club-City Island' = 132, 'Pelham Parkway' = 133, 'Pomonok-Flushing Heights-Hillcrest' = 134, 'Port Richmond' = 135, 'Prospect Heights' = 136, 'Prospect Lefferts Gardens-Wingate' = 137, 'Queens Village' = 138, 'Queensboro Hill' = 139, 'Queensbridge-Ravenswood-Long Island City' = 140, 'Rego Park' = 141, 'Richmond Hill' = 142, 'Ridgewood' = 143, 'Rikers Island' = 144, 'Rosedale' = 145, 'Rossville-Woodrow' = 146, 'Rugby-Remsen Village' = 147, 'Schuylerville-Throgs Neck-Edgewater Park' = 148, 'Seagate-Coney Island' = 149, 'Sheepshead Bay-Gerritsen Beach-Manhattan Beach' = 150, 'SoHo-TriBeCa-Civic Center-Little Italy' = 151, 'Soundview-Bruckner' = 152, 'Soundview-Castle Hill-Clason Point-Harding Park' = 153, 'South Jamaica' = 154, 'South Ozone Park' = 155, 'Springfield Gardens North' = 156, 'Springfield Gardens South-Brookville' = 157, 'Spuyten Duyvil-Kingsbridge' = 158, 'St. Albans' = 159, 'Stapleton-Rosebank' = 160, 'Starrett City' = 161, 'Steinway' = 162, 'Stuyvesant Heights' = 163, 'Stuyvesant Town-Cooper Village' = 164, 'Sunset Park East' = 165, 'Sunset Park West' = 166, 'Todt Hill-Emerson Hill-Heartland Village-Lighthouse Hill' = 167, 'Turtle Bay-East Midtown' = 168, 'University Heights-Morris Heights' = 169, 'Upper East Side-Carnegie Hill' = 170, 'Upper West Side' = 171, 'Van Cortlandt Village' = 172, 'Van Nest-Morris Park-Westchester Square' = 173, 'Washington Heights North' = 174, 'Washington Heights South' = 175, 'West Brighton' = 176, 'West Concourse' = 177, 'West Farms-Bronx River' = 178, 'West New Brighton-New Brighton-St. George' = 179, 'West Village' = 180, 'Westchester-Unionport' = 181, 'Westerleigh' = 182, 'Whitestone' = 183, 'Williamsbridge-Olinville' = 184, 'Williamsburg' = 185, 'Windsor Terrace' = 186, 'Woodhaven' = 187, 'Woodlawn-Wakefield' = 188, 'Woodside' = 189, 'Yorkville' = 190, 'park-cemetery-etc-Bronx' = 191, 'park-cemetery-etc-Brooklyn' = 192, 'park-cemetery-etc-Manhattan' = 193, 'park-cemetery-etc-Queens' = 194, 'park-cemetery-etc-Staten Island' = 195), dropoff_puma UInt16) ENGINE = MergeTree(pickup_date, pickup_datetime, 8192) ``` 在之前的服务器中运行： -``` sql +```sql CREATE TABLE trips_mergetree_x3 AS trips_mergetree_third ENGINE = Distributed(perftest, default, trips_mergetree_third, rand()) ``` 运行如下查询重新分布数据： -``` sql +```sql INSERT INTO trips_mergetree_x3 SELECT * FROM trips_mergetree ``` diff --git a/docs/zh/getting_started/example_datasets/ontime.md b/docs/zh/getting_started/example_datasets/ontime.md index ed81e2459e7..ec4053490a5 100644 --- a/docs/zh/getting_started/example_datasets/ontime.md +++ b/docs/zh/getting_started/example_datasets/ontime.md @@ -1,6 +1,13 @@ # 航班飞行数据 +航班飞行数据有以下两个方式获取： + +- 从原始数据导入 +- 下载预处理好的分区数据 + +## 从原始数据导入 + 下载数据： ```bash @@ -134,39 +141,75 @@ CREATE TABLE `ontime` ( 加载数据： ```bash -for i in *.zip; do echo $i; unzip -cq $i '*.csv' | sed 's/\.00//g' | clickhouse-client --host=example-perftest01j --query="INSERT INTO ontime FORMAT CSVWithNames"; done +$ for i in *.zip; do echo $i; unzip -cq $i '*.csv' | sed 's/\.00//g' | clickhouse-client --host=example-perftest01j --query="INSERT INTO ontime FORMAT CSVWithNames"; done ``` -查询： +## 下载预处理好的分区数据 + +```bash +$ curl -O https://clickhouse-datasets.s3.yandex.net/ontime/partitions/ontime.tar +$ tar xvf ontime.tar -C /var/lib/clickhouse # path to ClickHouse data directory +$ # check permissions of unpacked data, fix if required +$ sudo service clickhouse-server restart +$ clickhouse-client --query "select count(*) from datasets.ontime" +``` +!!!info + 如果要运行下面的SQL查询，必须使用完整的表名， + `datasets.ontime`。 + +## 查询： Q0. ```sql -select avg(c1) from (select Year, Month, count(*) as c1 from ontime group by Year, Month); +SELECT avg(c1) +FROM +( + SELECT Year, Month, count(*) AS c1 + FROM ontime + GROUP BY Year, Month +); ``` Q1. 查询从2000年到2008年每天的航班数 ```sql -SELECT DayOfWeek, count(*) AS c FROM ontime WHERE Year >= 2000 AND Year <= 2008 GROUP BY DayOfWeek ORDER BY c DESC; +SELECT DayOfWeek, count(*) AS c +FROM ontime +WHERE Year>=2000 AND Year<=2008 +GROUP BY DayOfWeek +ORDER BY c DESC; ``` Q2. 查询从2000年到2008年每周延误超过10分钟的航班数。 ```sql -SELECT DayOfWeek, count(*) AS c FROM ontime WHERE DepDelay>10 AND Year >= 2000 AND Year <= 2008 GROUP BY DayOfWeek ORDER BY c DESC +SELECT DayOfWeek, count(*) AS c +FROM ontime +WHERE DepDelay>10 AND Year>=2000 AND Year<=2008 +GROUP BY DayOfWeek +ORDER BY c DESC; ``` Q3. 查询2000年到2008年每个机场延误超过10分钟以上的次数 ```sql -SELECT Origin, count(*) AS c FROM ontime WHERE DepDelay>10 AND Year >= 2000 AND Year <= 2008 GROUP BY Origin ORDER BY c DESC LIMIT 10 +SELECT Origin, count(*) AS c +FROM ontime +WHERE DepDelay>10 AND Year>=2000 AND Year<=2008 +GROUP BY Origin +ORDER BY c DESC +LIMIT 10; ``` Q4. 查询2007年各航空公司延误超过10分钟以上的次数 ```sql -SELECT Carrier, count(*) FROM ontime WHERE DepDelay>10 AND Year = 2007 GROUP BY Carrier ORDER BY count(*) DESC +SELECT Carrier, count(*) +FROM ontime +WHERE DepDelay>10 AND Year=2007 +GROUP BY Carrier +ORDER BY count(*) DESC; ``` Q5. 查询2007年各航空公司延误超过10分钟以上的百分比 @@ -198,7 +241,11 @@ ORDER BY c3 DESC; 更好的查询版本： ```sql -SELECT Carrier, avg(DepDelay > 10) * 100 AS c3 FROM ontime WHERE Year = 2007 GROUP BY Carrier ORDER BY Carrier +SELECT Carrier, avg(DepDelay>10)*100 AS c3 +FROM ontime +WHERE Year=2007 +GROUP BY Carrier +ORDER BY Carrier ``` Q6. 同上一个查询一致,只是查询范围扩大到2000年到2008年 @@ -212,7 +259,7 @@ FROM count(*) AS c FROM ontime WHERE DepDelay>10 - AND Year >= 2000 AND Year <= 2008 + AND Year>=2000 AND Year<=2008 GROUP BY Carrier ) ANY INNER JOIN @@ -221,7 +268,7 @@ ANY INNER JOIN Carrier, count(*) AS c2 FROM ontime - WHERE Year >= 2000 AND Year <= 2008 + WHERE Year>=2000 AND Year<=2008 GROUP BY Carrier ) USING Carrier ORDER BY c3 DESC; @@ -230,7 +277,11 @@ ORDER BY c3 DESC; 更好的查询版本： ```sql -SELECT Carrier, avg(DepDelay > 10) * 100 AS c3 FROM ontime WHERE Year >= 2000 AND Year <= 2008 GROUP BY Carrier ORDER BY Carrier +SELECT Carrier, avg(DepDelay>10)*100 AS c3 +FROM ontime +WHERE Year>=2000 AND Year<=2008 +GROUP BY Carrier +ORDER BY Carrier; ``` Q7. 每年航班延误超过10分钟的百分比 @@ -254,41 +305,50 @@ ANY INNER JOIN from ontime GROUP BY Year ) USING (Year) -ORDER BY Year +ORDER BY Year; ``` 更好的查询版本： ```sql -SELECT Year, avg(DepDelay > 10) FROM ontime GROUP BY Year ORDER BY Year +SELECT Year, avg(DepDelay>10) +FROM ontime +GROUP BY Year +ORDER BY Year; ``` Q8. 每年更受人们喜爱的目的地 ```sql -SELECT DestCityName, uniqExact(OriginCityName) AS u FROM ontime WHERE Year >= 2000 and Year <= 2010 GROUP BY DestCityName ORDER BY u DESC LIMIT 10; +SELECT DestCityName, uniqExact(OriginCityName) AS u +FROM ontime +WHERE Year >= 2000 and Year <= 2010 +GROUP BY DestCityName +ORDER BY u DESC LIMIT 10; ``` Q9. ```sql -select Year, count(*) as c1 from ontime group by Year; +SELECT Year, count(*) AS c1 +FROM ontime +GROUP BY Year; ``` Q10. ```sql -select - min(Year), max(Year), Carrier, count(*) as cnt, - sum(ArrDelayMinutes>30) as flights_delayed, - round(sum(ArrDelayMinutes>30)/count(*),2) as rate +SELECT + min(Year), max(Year), Carrier, count(*) AS cnt, + sum(ArrDelayMinutes>30) AS flights_delayed, + round(sum(ArrDelayMinutes>30)/count(*),2) AS rate FROM ontime WHERE - DayOfWeek not in (6,7) and OriginState not in ('AK', 'HI', 'PR', 'VI') - and DestState not in ('AK', 'HI', 'PR', 'VI') - and FlightDate < '2010-01-01' + DayOfWeek NOT IN (6,7) AND OriginState NOT IN ('AK', 'HI', 'PR', 'VI') + AND DestState NOT IN ('AK', 'HI', 'PR', 'VI') + AND FlightDate < '2010-01-01' GROUP by Carrier -HAVING cnt > 100000 and max(Year) > 1990 +HAVING cnt>100000 and max(Year)>1990 ORDER by rate DESC LIMIT 1000; ``` @@ -296,15 +356,39 @@ LIMIT 1000; Bonus: ```sql -SELECT avg(cnt) FROM (SELECT Year,Month,count(*) AS cnt FROM ontime WHERE DepDel15=1 GROUP BY Year,Month) +SELECT avg(cnt) +FROM +( + SELECT Year,Month,count(*) AS cnt + FROM ontime + WHERE DepDel15=1 + GROUP BY Year,Month +); -select avg(c1) from (select Year,Month,count(*) as c1 from ontime group by Year,Month) +SELECT avg(c1) FROM +( + SELECT Year,Month,count(*) AS c1 + FROM ontime + GROUP BY Year,Month +); -SELECT DestCityName, uniqExact(OriginCityName) AS u FROM ontime GROUP BY DestCityName ORDER BY u DESC LIMIT 10; +SELECT DestCityName, uniqExact(OriginCityName) AS u +FROM ontime +GROUP BY DestCityName +ORDER BY u DESC +LIMIT 10; -SELECT OriginCityName, DestCityName, count() AS c FROM ontime GROUP BY OriginCityName, DestCityName ORDER BY c DESC LIMIT 10; +SELECT OriginCityName, DestCityName, count() AS c +FROM ontime +GROUP BY OriginCityName, DestCityName +ORDER BY c DESC +LIMIT 10; -SELECT OriginCityName, count() AS c FROM ontime GROUP BY OriginCityName ORDER BY c DESC LIMIT 10; +SELECT OriginCityName, count() AS c +FROM ontime +GROUP BY OriginCityName +ORDER BY c DESC +LIMIT 10; ``` 这个性能测试由Vadim Tkachenko提供。参考： diff --git a/docs/zh/getting_started/example_datasets/star_schema.md b/docs/zh/getting_started/example_datasets/star_schema.md index 1d8af3b29a5..865327b50ec 100644 --- a/docs/zh/getting_started/example_datasets/star_schema.md +++ b/docs/zh/getting_started/example_datasets/star_schema.md @@ -1,26 +1,26 @@ # Star Schema Benchmark -Compiling dbgen: +编译 dbgen: -``` -git clone git@github.com:vadimtk/ssb-dbgen.git -cd ssb-dbgen -make +```bash +$ git clone git@github.com:vadimtk/ssb-dbgen.git +$ cd ssb-dbgen +$ make ``` -Generating data: +开始生成数据： -``` -./dbgen -s 1000 -T c -./dbgen -s 1000 -T l -./dbgen -s 1000 -T p -./dbgen -s 1000 -T s -./dbgen -s 1000 -T d +```bash +$ ./dbgen -s 1000 -T c +$ ./dbgen -s 1000 -T l +$ ./dbgen -s 1000 -T p +$ ./dbgen -s 1000 -T s +$ ./dbgen -s 1000 -T d ``` -Creating tables in ClickHouse: +在ClickHouse中创建表结构： -``` +```sql CREATE TABLE customer ( C_CUSTKEY UInt32, @@ -83,73 +83,85 @@ CREATE TABLE supplier ENGINE = MergeTree ORDER BY S_SUPPKEY; ``` -Inserting data: +写入数据： -``` -clickhouse-client --query "INSERT INTO customer FORMAT CSV" < customer.tbl -clickhouse-client --query "INSERT INTO part FORMAT CSV" < part.tbl -clickhouse-client --query "INSERT INTO supplier FORMAT CSV" < supplier.tbl -clickhouse-client --query "INSERT INTO lineorder FORMAT CSV" < lineorder.tbl +```bash +$ clickhouse-client --query "INSERT INTO customer FORMAT CSV" < customer.tbl +$ clickhouse-client --query "INSERT INTO part FORMAT CSV" < part.tbl +$ clickhouse-client --query "INSERT INTO supplier FORMAT CSV" < supplier.tbl +$ clickhouse-client --query "INSERT INTO lineorder FORMAT CSV" < lineorder.tbl ``` -Converting "star schema" to denormalized "flat schema": +将“星型模型”转换为非规范化的“平面模型”： -``` +```sql SET max_memory_usage = 20000000000, allow_experimental_multiple_joins_emulation = 1; CREATE TABLE lineorder_flat ENGINE = MergeTree PARTITION BY toYear(LO_ORDERDATE) ORDER BY (LO_ORDERDATE, LO_ORDERKEY) AS -SELECT * -FROM lineorder -ANY INNER JOIN customer ON LO_CUSTKEY = C_CUSTKEY -ANY INNER JOIN supplier ON LO_SUPPKEY = S_SUPPKEY -ANY INNER JOIN part ON LO_PARTKEY = P_PARTKEY; +SELECT l.*, c.*, s.*, p.* +FROM lineorder l + ANY INNER JOIN customer c ON (c.C_CUSTKEY = l.LO_CUSTKEY) + ANY INNER JOIN supplier s ON (s.S_SUPPKEY = l.LO_SUPPKEY) + ANY INNER JOIN part p ON (p.P_PARTKEY = l.LO_PARTKEY); ALTER TABLE lineorder_flat DROP COLUMN C_CUSTKEY, DROP COLUMN S_SUPPKEY, DROP COLUMN P_PARTKEY; ``` Running the queries: -``` Q1.1 +```sql SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue FROM lineorder_flat WHERE toYear(LO_ORDERDATE) = 1993 AND LO_DISCOUNT BETWEEN 1 AND 3 AND LO_QUANTITY < 25; - +``` Q1.2 +```sql SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue FROM lineorder_flat WHERE toYYYYMM(LO_ORDERDATE) = 199401 AND LO_DISCOUNT BETWEEN 4 AND 6 AND LO_QUANTITY BETWEEN 26 AND 35; - +``` Q1.3 +```sql SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue FROM lineorder_flat WHERE toISOWeek(LO_ORDERDATE) = 6 AND toYear(LO_ORDERDATE) = 1994 AND LO_DISCOUNT BETWEEN 5 AND 7 AND LO_QUANTITY BETWEEN 26 AND 35; - +``` Q2.1 +```sql SELECT sum(LO_REVENUE), toYear(LO_ORDERDATE) AS year, P_BRAND FROM lineorder_flat WHERE P_CATEGORY = 'MFGR#12' AND S_REGION = 'AMERICA' GROUP BY year, P_BRAND ORDER BY year, P_BRAND; - +``` Q2.2 +```sql SELECT sum(LO_REVENUE), toYear(LO_ORDERDATE) AS year, P_BRAND FROM lineorder_flat WHERE P_BRAND BETWEEN 'MFGR#2221' AND 'MFGR#2228' AND S_REGION = 'ASIA' GROUP BY year, P_BRAND ORDER BY year, P_BRAND; - +``` Q2.3 +```sql SELECT sum(LO_REVENUE), toYear(LO_ORDERDATE) AS year, P_BRAND FROM lineorder_flat WHERE P_BRAND = 'MFGR#2239' AND S_REGION = 'EUROPE' GROUP BY year, P_BRAND ORDER BY year, P_BRAND; - +``` Q3.1 +```sql SELECT C_NATION, S_NATION, toYear(LO_ORDERDATE) AS year, sum(LO_REVENUE) AS revenue FROM lineorder_flat WHERE C_REGION = 'ASIA' AND S_REGION = 'ASIA' AND year >= 1992 AND year <= 1997 GROUP BY C_NATION, S_NATION, year ORDER BY year asc, revenue desc; - +``` Q3.2 +```sql SELECT C_CITY, S_CITY, toYear(LO_ORDERDATE) AS year, sum(LO_REVENUE) AS revenue FROM lineorder_flat WHERE C_NATION = 'UNITED STATES' AND S_NATION = 'UNITED STATES' AND year >= 1992 AND year <= 1997 GROUP BY C_CITY, S_CITY, year ORDER BY year asc, revenue desc; - +``` Q3.3 +```sql SELECT C_CITY, S_CITY, toYear(LO_ORDERDATE) AS year, sum(LO_REVENUE) AS revenue FROM lineorder_flat WHERE (C_CITY = 'UNITED KI1' OR C_CITY = 'UNITED KI5') AND (S_CITY = 'UNITED KI1' OR S_CITY = 'UNITED KI5') AND year >= 1992 AND year <= 1997 GROUP BY C_CITY, S_CITY, year ORDER BY year asc, revenue desc; - +``` Q3.4 +```sql SELECT C_CITY, S_CITY, toYear(LO_ORDERDATE) AS year, sum(LO_REVENUE) AS revenue FROM lineorder_flat WHERE (C_CITY = 'UNITED KI1' OR C_CITY = 'UNITED KI5') AND (S_CITY = 'UNITED KI1' OR S_CITY = 'UNITED KI5') AND toYYYYMM(LO_ORDERDATE) = '199712' GROUP BY C_CITY, S_CITY, year ORDER BY year asc, revenue desc; - +``` Q4.1 +```sql SELECT toYear(LO_ORDERDATE) AS year, C_NATION, sum(LO_REVENUE - LO_SUPPLYCOST) AS profit FROM lineorder_flat WHERE C_REGION = 'AMERICA' AND S_REGION = 'AMERICA' AND (P_MFGR = 'MFGR#1' OR P_MFGR = 'MFGR#2') GROUP BY year, C_NATION ORDER BY year, C_NATION; - +``` Q4.2 +```sql SELECT toYear(LO_ORDERDATE) AS year, S_NATION, P_CATEGORY, sum(LO_REVENUE - LO_SUPPLYCOST) AS profit FROM lineorder_flat WHERE C_REGION = 'AMERICA' AND S_REGION = 'AMERICA' AND (year = 1997 OR year = 1998) AND (P_MFGR = 'MFGR#1' OR P_MFGR = 'MFGR#2') GROUP BY year, S_NATION, P_CATEGORY ORDER BY year, S_NATION, P_CATEGORY; - +``` Q4.3 +```sql SELECT toYear(LO_ORDERDATE) AS year, S_CITY, P_BRAND, sum(LO_REVENUE - LO_SUPPLYCOST) AS profit FROM lineorder_flat WHERE S_NATION = 'UNITED STATES' AND (year = 1997 OR year = 1998) AND P_CATEGORY = 'MFGR#14' GROUP BY year, S_CITY, P_BRAND ORDER BY year, S_CITY, P_BRAND; ``` diff --git a/docs/zh/getting_started/example_datasets/wikistat.md b/docs/zh/getting_started/example_datasets/wikistat.md index c306c644551..ee3b800f47b 100644 --- a/docs/zh/getting_started/example_datasets/wikistat.md +++ b/docs/zh/getting_started/example_datasets/wikistat.md @@ -4,7 +4,7 @@ 创建表结构： -``` sql +```sql CREATE TABLE wikistat ( date Date, @@ -20,9 +20,9 @@ CREATE TABLE wikistat 加载数据： ```bash -for i in {2007..2016}; do for j in {01..12}; do echo $i-$j >&2; curl -sSL "http://dumps.wikimedia.org/other/pagecounts-raw/$i/$i-$j/" | grep -oE 'pagecounts-[0-9]+-[0-9]+\.gz'; done; done | sort | uniq | tee links.txt -cat links.txt | while read link; do wget http://dumps.wikimedia.org/other/pagecounts-raw/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1/')/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1-\2/')/$link; done -ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikistat/$i | ./wikistat-loader --time="$(echo -n $i | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})([0-9]{2})-([0-9]{2})([0-9]{2})([0-9]{2})\.gz/\1-\2-\3 \4-00-00/')" | clickhouse-client --query="INSERT INTO wikistat FORMAT TabSeparated"; done +$ for i in {2007..2016}; do for j in {01..12}; do echo $i-$j >&2; curl -sSL "http://dumps.wikimedia.org/other/pagecounts-raw/$i/$i-$j/" | grep -oE 'pagecounts-[0-9]+-[0-9]+\.gz'; done; done | sort | uniq | tee links.txt +$ cat links.txt | while read link; do wget http://dumps.wikimedia.org/other/pagecounts-raw/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1/')/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1-\2/')/$link; done +$ ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikistat/$i | ./wikistat-loader --time="$(echo -n $i | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})([0-9]{2})-([0-9]{2})([0-9]{2})([0-9]{2})\.gz/\1-\2-\3 \4-00-00/')" | clickhouse-client --query="INSERT INTO wikistat FORMAT TabSeparated"; done ``` diff --git a/docs/zh/getting_started/index.md b/docs/zh/getting_started/index.md index 5e1a5777292..c73181a6068 100644 --- a/docs/zh/getting_started/index.md +++ b/docs/zh/getting_started/index.md @@ -1,158 +1,10 @@ -# 入门指南 +# 入门 -## 系统要求 +如果您是ClickHouse的新手，并希望亲身体验它的性能，首先您需要通过 [安装过程](install.md). -如果从官方仓库安装，需要确保您使用的是x86\_64处理器构架的Linux并且支持SSE 4.2指令集 +之后，您可以选择以下选项之一： -检查是否支持SSE 4.2： +* [通过详细的教程](tutorial.md) +* [试验示例数据集](example_datasets/ontime.md) -```bash -grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" -``` - -我们推荐使用Ubuntu或者Debian。终端必须使用UTF-8编码。 - -基于rpm的系统,你可以使用第三方的安装包：https://packagecloud.io/altinity/clickhouse 或者直接安装debian安装包。 - -ClickHouse还可以在FreeBSD与Mac OS X上工作。同时它可以在不支持SSE 4.2的x86\_64构架和AArch64 CPUs上编译。 - -## 安装 - -### 为Debian/Ubuntu安装 - -在`/etc/apt/sources.list` (或创建`/etc/apt/sources.list.d/clickhouse.list`文件)中添加仓库： - -```text -deb http://repo.yandex.ru/clickhouse/deb/stable/ main/ -``` - -如果你想使用最新的测试版本，请使用'testing'替换'stable'。 - -然后运行： - -```bash -sudo apt-get install dirmngr # optional -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 # optional -sudo apt-get update -sudo apt-get install clickhouse-client clickhouse-server -``` - -你也可以从这里手动下载安装包：。 - -ClickHouse包含访问控制配置，它们位于`users.xml`文件中(与'config.xml'同目录)。 -默认情况下，允许从任何地方使用默认的‘default’用户无密码的访问ClickHouse。参考‘user/default/networks’。 -有关更多信息，请参考"Configuration files"部分。 - -###来自RPM包 - -Yandex ClickHouse团队建议使用官方预编译的`rpm`软件包，用于CentOS，RedHat和所有其他基于rpm的Linux发行版。 - -首先，您需要添加官方存储库： - -```bash -sudo yum install yum-utils -sudo rpm --import https://repo.yandex.ru/clickhouse/CLICKHOUSE-KEY.GPG -sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/x86_64 -``` - -如果您想使用最新版本，请将`stable`替换为`testing`（建议您在测试环境中使用）。 - -然后运行这些命令以实际安装包： - -```bash -sudo yum install clickhouse-server clickhouse-client -``` - -您也可以从此处手动下载和安装软件包：。 - -###来自Docker - -要在Docker中运行ClickHouse，请遵循[Docker Hub](https://hub.docker.com/r/yandex/clickhouse-server/)上的指南。这些镜像使用官方的`deb`包构建。 - -### 使用源码安装 - -具体编译方式可以参考build.md。 - -你可以编译并安装它们。 -你也可以直接使用而不进行安装。 - -```text -Client: dbms/programs/clickhouse-client -Server: dbms/programs/clickhouse-server -``` - -在服务器中为数据创建如下目录： - -```text -/opt/clickhouse/data/default/ -/opt/clickhouse/metadata/default/ -``` - -(它们可以在server config中配置。) -为需要的用户运行‘chown’ - -日志的路径可以在server config (src/dbms/programs/server/config.xml)中配置。 - -## 启动 - -可以运行如下命令在后台启动服务： - -```bash -sudo service clickhouse-server start -``` - -可以在`/var/log/clickhouse-server/`目录中查看日志。 - -如果服务没有启动，请检查配置文件 `/etc/clickhouse-server/config.xml`。 - -你也可以在控制台中直接启动服务： - -```bash -clickhouse-server --config-file=/etc/clickhouse-server/config.xml -``` - -在这种情况下，日志将被打印到控制台中，这在开发过程中很方便。 -如果配置文件在当前目录中，你可以不指定‘--config-file’参数。它默认使用‘./config.xml’。 - -你可以使用命令行客户端连接到服务： - -```bash -clickhouse-client -``` - -默认情况下它使用‘default’用户无密码的与localhost:9000服务建立连接。 -客户端也可以用于连接远程服务，例如： - -```bash -clickhouse-client --host=example.com -``` - -有关更多信息，请参考"Command-line client"部分。 - -检查系统是否工作： - -```bash -milovidov@hostname:~/work/metrica/src/dbms/src/Client$ ./clickhouse-client -ClickHouse client version 0.0.18749. -Connecting to localhost:9000. -Connected to ClickHouse server version 0.0.18749. - -:) SELECT 1 - -SELECT 1 - -┌─1─┐ -│ 1 │ -└───┘ - -1 rows in set. Elapsed: 0.003 sec. - -:) -``` - -**恭喜，系统已经工作了!** - -为了继续进行实验，你可以尝试下载测试数据集。 - - -[Original article](https://clickhouse.yandex/docs/en/getting_started/) +[来源文章](https://clickhouse.yandex/docs/zh/getting_started/) diff --git a/docs/zh/getting_started/install.md b/docs/zh/getting_started/install.md new file mode 100644 index 00000000000..aa3cb816218 --- /dev/null +++ b/docs/zh/getting_started/install.md @@ -0,0 +1,152 @@ +## 系统要求 + +ClickHouse可以在任何具有x86\_64，AArch64或PowerPC64LE CPU架构的Linux，FreeBSD或Mac OS X上运行。 + +虽然预构建的二进制文件通常是为x86 \ _64编译并利用SSE 4.2指令集，但除非另有说明，否则使用支持它的CPU将成为额外的系统要求。这是检查当前CPU是否支持SSE 4.2的命令： + +``` bash +$ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" +``` + +要在不支持SSE 4.2或具有AArch64或PowerPC64LE体系结构的处理器上运行ClickHouse，您应该[通过源构建ClickHouse](#from-sources)进行适当的配置调整。 + +##可用的安装选项 + +### 为Debian/Ubuntu安装 {#from-deb-packages} + +在`/etc/apt/sources.list` (或创建`/etc/apt/sources.list.d/clickhouse.list`文件)中添加仓库： + +```text +deb http://repo.yandex.ru/clickhouse/deb/stable/ main/ +``` + +如果你想使用最新的测试版本，请使用'testing'替换'stable'。 + +然后运行： + +```bash +sudo apt-get install dirmngr # optional +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 # optional +sudo apt-get update +sudo apt-get install clickhouse-client clickhouse-server +``` + +你也可以从这里手动下载安装包：。 + +ClickHouse包含访问控制配置，它们位于`users.xml`文件中(与'config.xml'同目录)。 +默认情况下，允许从任何地方使用默认的‘default’用户无密码的访问ClickHouse。参考‘user/default/networks’。 +有关更多信息，请参考"Configuration files"部分。 + +###来自RPM包 {#from-rpm-packages} + +Yandex ClickHouse团队建议使用官方预编译的`rpm`软件包，用于CentOS，RedHat和所有其他基于rpm的Linux发行版。 + +首先，您需要添加官方存储库： + +```bash +sudo yum install yum-utils +sudo rpm --import https://repo.yandex.ru/clickhouse/CLICKHOUSE-KEY.GPG +sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/x86_64 +``` + +如果您想使用最新版本，请将`stable`替换为`testing`（建议您在测试环境中使用）。 + +然后运行这些命令以实际安装包： + +```bash +sudo yum install clickhouse-server clickhouse-client +``` + +您也可以从此处手动下载和安装软件包：。 + +###来自Docker {#from-docker-image} + +要在Docker中运行ClickHouse，请遵循[Docker Hub](https://hub.docker.com/r/yandex/clickhouse-server/)上的指南。那些图像使用官方的`deb`包。 + +### 使用源码安装 {#from-sources} + +具体编译方式可以参考build.md。 + +你可以编译并安装它们。 +你也可以直接使用而不进行安装。 + +```text +Client: dbms/programs/clickhouse-client +Server: dbms/programs/clickhouse-server +``` + +在服务器中为数据创建如下目录： + +```text +/opt/clickhouse/data/default/ +/opt/clickhouse/metadata/default/ +``` + +(它们可以在server config中配置。) +为需要的用户运行‘chown’ + +日志的路径可以在server config (src/dbms/programs/server/config.xml)中配置。 + +## 启动 + +可以运行如下命令在后台启动服务： + +```bash +sudo service clickhouse-server start +``` + +可以在`/var/log/clickhouse-server/`目录中查看日志。 + +如果服务没有启动，请检查配置文件 `/etc/clickhouse-server/config.xml`。 + +你也可以在控制台中直接启动服务： + +```bash +clickhouse-server --config-file=/etc/clickhouse-server/config.xml +``` + +在这种情况下，日志将被打印到控制台中，这在开发过程中很方便。 +如果配置文件在当前目录中，你可以不指定‘--config-file’参数。它默认使用‘./config.xml’。 + +你可以使用命令行客户端连接到服务： + +```bash +clickhouse-client +``` + +默认情况下它使用‘default’用户无密码的与localhost:9000服务建立连接。 +客户端也可以用于连接远程服务，例如： + +```bash +clickhouse-client --host=example.com +``` + +有关更多信息，请参考"Command-line client"部分。 + +检查系统是否工作： + +```bash +milovidov@hostname:~/work/metrica/src/dbms/src/Client$ ./clickhouse-client +ClickHouse client version 0.0.18749. +Connecting to localhost:9000. +Connected to ClickHouse server version 0.0.18749. + +:) SELECT 1 + +SELECT 1 + +┌─1─┐ +│ 1 │ +└───┘ + +1 rows in set. Elapsed: 0.003 sec. + +:) +``` + +**恭喜，系统已经工作了!** + +为了继续进行实验，你可以尝试下载测试数据集。 + + +[Original article](https://clickhouse.yandex/docs/en/getting_started/install/) diff --git a/docs/zh/getting_started/tutorial.md b/docs/zh/getting_started/tutorial.md new file mode 120000 index 00000000000..8bc40816ab2 --- /dev/null +++ b/docs/zh/getting_started/tutorial.md @@ -0,0 +1 @@ +../../en/getting_started/tutorial.md \ No newline at end of file diff --git a/docs/zh/interfaces/cli.md b/docs/zh/interfaces/cli.md index 933b43c9469..bc9be960c48 100644 --- a/docs/zh/interfaces/cli.md +++ b/docs/zh/interfaces/cli.md @@ -89,13 +89,13 @@ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMA - `--vertical, -E` – 如果指定，默认情况下使用垂直格式输出结果。这与 '--format=Vertical' 相同。在这种格式中，每个值都在单独的行上打印，这种方式对显示宽表很有帮助。 - `--time, -t` – 如果指定，非交互模式下会打印查询执行的时间到 'stderr' 中。 - `--stacktrace` – 如果指定，如果出现异常，会打印堆栈跟踪信息。 -- `-config-file` – 配置文件的名称。 +- `--config-file` – 配置文件的名称。 ### 配置文件 `clickhouse-client` 使用一下第一个存在的文件： -- 通过 `-config-file` 参数指定的文件. +- 通过 `--config-file` 参数指定的文件. - `./clickhouse-client.xml` - `\~/.clickhouse-client/config.xml` - `/etc/clickhouse-client/config.xml` diff --git a/docs/zh/interfaces/cpp.md b/docs/zh/interfaces/cpp.md new file mode 100644 index 00000000000..ff791b38d13 --- /dev/null +++ b/docs/zh/interfaces/cpp.md @@ -0,0 +1,5 @@ +# C ++客户端库 + +请参阅以下网站的自述文件[clickhouse-cpp](https://github.com/ClickHouse/clickhouse-cpp)资料库。 + +[Original article](https://clickhouse.yandex/docs/zh/interfaces/cpp/) diff --git a/docs/zh/interfaces/index.md b/docs/zh/interfaces/index.md index 12b61c3f9fd..3336aa4d105 100644 --- a/docs/zh/interfaces/index.md +++ b/docs/zh/interfaces/index.md @@ -3,12 +3,13 @@ ClickHouse提供了两个网络接口（两者都可以选择包装在TLS中以提高安全性）： * [HTTP](http.md)，记录在案，易于使用. -* [本地人TCP](tcp.md)，这有较少的开销. +* [本地TCP](tcp.md)，这有较少的开销. 在大多数情况下，建议使用适当的工具或库，而不是直接与这些工具或库进行交互。 Yandex的官方支持如下： * [命令行客户端](cli.md) * [JDBC驱动程序](jdbc.md) * [ODBC驱动程序](odbc.md) +* [C++客户端库](cpp.md) 还有许多第三方库可供使用ClickHouse： * [客户端库](third-party/client_libraries.md) diff --git a/docs/zh/interfaces/third-party/client_libraries.md b/docs/zh/interfaces/third-party/client_libraries.md index 2175f8c89eb..a8625c0d4ac 100644 --- a/docs/zh/interfaces/third-party/client_libraries.md +++ b/docs/zh/interfaces/third-party/client_libraries.md @@ -26,7 +26,7 @@ - [HTTP-ClickHouse](https://metacpan.org/release/HTTP-ClickHouse) - [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse) - Ruby - - [clickhouse (Ruby)](https://github.com/archan937/clickhouse) + - [ClickHouse (Ruby)](https://github.com/shlima/click_house) - R - [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r) - [RClickhouse](https://github.com/IMSMWU/RClickhouse) @@ -39,8 +39,6 @@ - C# - [ClickHouse.Ado](https://github.com/killwort/ClickHouse-Net) - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) -- C++ - - [clickhouse-cpp](https://github.com/artpaul/clickhouse-cpp/) - Elixir - [clickhousex](https://github.com/appodeal/clickhousex/) - Nim diff --git a/docs/zh/interfaces/third-party/gui.md b/docs/zh/interfaces/third-party/gui.md index 31a533e229d..b8143792981 100644 --- a/docs/zh/interfaces/third-party/gui.md +++ b/docs/zh/interfaces/third-party/gui.md @@ -63,6 +63,7 @@ ClickHouse Web 界面 [Tabix](https://github.com/tabixio/tabix). [clickhouse-cli](https://github.com/hatarist/clickhouse-cli) 是ClickHouse的替代命令行客户端，用Python 3编写。 特征： + - 自动完成。 - 查询和数据输出的语法高亮显示。 - 寻呼机支持数据输出。 @@ -74,6 +75,18 @@ ClickHouse Web 界面 [Tabix](https://github.com/tabixio/tabix). ## 商业 +### Holistics Software + +[Holistics](https://www.holistics.io/) 在2019年被Gartner FrontRunners列为可用性最高排名第二的商业智能工具之一。 Holistics是一个基于SQL的全栈数据平台和商业智能工具，用于设置您的分析流程。 + +特征： + +-自动化的电子邮件，Slack和Google表格报告时间表。 +-强大的SQL编辑器，具有版本控制，自动完成，可重用的查询组件和动态过滤器。 +-通过iframe在自己的网站或页面中嵌入仪表板。 +-数据准备和ETL功能。 +-SQL数据建模支持数据的关系映射。 + ### DataGrip [DataGrip](https://www.jetbrains.com/datagrip/) 是JetBrains的数据库IDE，专门支持ClickHouse。 它还嵌入到其他基于IntelliJ的工具中：PyCharm，IntelliJ IDEA，GoLand，PhpStorm等。 diff --git a/docs/zh/introduction/features_considered_disadvantages.md b/docs/zh/introduction/features_considered_disadvantages.md index 9aa6e90a4d8..015f1481b80 100644 --- a/docs/zh/introduction/features_considered_disadvantages.md +++ b/docs/zh/introduction/features_considered_disadvantages.md @@ -1,4 +1,4 @@ -# ClickHouse可以考虑缺点的功能 +# ClickHouse可以认为是缺点的功能 1. 没有完整的事务支持。 2. 缺少高频率，低延迟的修改或删除已存在数据的能力。仅能用于批量删除或修改数据，但这符合 [GDPR](https://gdpr-info.eu)。 diff --git a/docs/zh/introduction/ya_metrika_task.md b/docs/zh/introduction/history.md similarity index 99% rename from docs/zh/introduction/ya_metrika_task.md rename to docs/zh/introduction/history.md index da4b18826e0..86fe02f84d5 100644 --- a/docs/zh/introduction/ya_metrika_task.md +++ b/docs/zh/introduction/history.md @@ -1,4 +1,4 @@ -# Yandex.Metrica的使用案例 +# ClickHouse历史 ClickHouse最初是为 [Yandex.Metrica](https://metrica.yandex.com/) [世界第二大Web分析平台](http://w3techs.com/technologies/overview/traffic_analysis/all) 而开发的。多年来一直作为该系统的核心组件被该系统持续使用着。目前为止，该系统在ClickHouse中有超过13万亿条记录，并且每天超过200多亿个事件被处理。它允许直接从原始数据中动态查询并生成报告。本文简要介绍了ClickHouse在其早期发展阶段的目标。 diff --git a/docs/zh/operations/monitoring.md b/docs/zh/operations/monitoring.md deleted file mode 120000 index 515ae8b4fff..00000000000 --- a/docs/zh/operations/monitoring.md +++ /dev/null @@ -1 +0,0 @@ -../../en/operations/monitoring.md \ No newline at end of file diff --git a/docs/zh/operations/monitoring.md b/docs/zh/operations/monitoring.md new file mode 100644 index 00000000000..5ad0a1846a2 --- /dev/null +++ b/docs/zh/operations/monitoring.md @@ -0,0 +1,37 @@ +# 监控 + +可以监控到： + +- 硬件资源的利用率。 +- ClickHouse 服务的指标。 + +## 硬件资源利用率 + +ClickHouse 本身不会去监控硬件资源的状态。 + +强烈推荐监控以下监控项： + +- 处理器上的负载和温度。 + + 可以使用 [dmesg](https://en.wikipedia.org/wiki/Dmesg), [turbostat](https://www.linux.org/docs/man8/turbostat.html) 或者其他工具。 + +- 磁盘存储，RAM和网络的使用率。 + +## ClickHouse 服务的指标。 + +ClickHouse服务本身具有用于自我状态监视指标。 + +要跟踪服务器事件，请观察服务器日志。 请参阅配置文件的[logger]（server_settings/settings.md#server_settings-logger）部分。 + +ClickHouse 收集的指标项： + +- 服务用于计算的资源占用的各种指标。 +- 关于查询处理的常见统计信息。 + +可以在 [system.metrics](system_tables.md#system_tables-metrics) ，[system.events](system_tables.md#system_tables-events) 以及[system.asynchronous_metrics](system_tables.md#system_tables-asynchronous_metrics) 等系统表查看所有的指标项。 + +可以配置ClickHouse 往 [Graphite](https://github.com/graphite-project)导入指标。 参考 [Graphite section](server_settings/settings.md#server_settings-graphite) 配置文件。在配置指标导出之前，需要参考Graphite[官方教程](https://graphite.readthedocs.io/en/latest/install.html)搭建服务。 + +此外，您可以通过HTTP API监视服务器可用性。 将HTTP GET请求发送到 `/`。 如果服务器可用，它将以 `200 OK` 响应。 + +要监视服务器集群的配置中，应设置[max_replica_delay_for_distributed_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries)参数并使用HTTP资源`/replicas_status`。 如果副本可用，并且不延迟在其他副本之后，则对`/replicas_status`的请求将返回200 OK。 如果副本被延迟，它将返回有关延迟信息。 diff --git a/docs/zh/operations/table_engines/mergetree.md b/docs/zh/operations/table_engines/mergetree.md index 4c35f3cf6b9..fc7b4967571 100644 --- a/docs/zh/operations/table_engines/mergetree.md +++ b/docs/zh/operations/table_engines/mergetree.md @@ -70,8 +70,14 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] - `SETTINGS` — 影响 `MergeTree` 性能的额外参数： - `index_granularity` — 索引粒度。即索引中相邻『标记』间的数据行数。默认值，8192 。该列表中所有可用的参数可以从这里查看 [MergeTreeSettings.h](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/src/Storages/MergeTree/MergeTreeSettings.h) 。 + - `index_granularity_bytes` — 索引粒度，以字节为单位，默认值: 10Mb。如果仅按数据行数限制索引粒度, 请设置为0(不建议)。 + - `enable_mixed_granularity_parts` — 启用或禁用通过 `index_granularity_bytes` 控制索引粒度的大小。在19.11版本之前, 只有 `index_granularity` 配置能够用于限制索引粒度的大小。当从大表(数十或数百兆)中查询数据时候，`index_granularity_bytes` 配置能够提升ClickHouse的性能。如果你的表内数据量很大，可以开启这项配置用以提升`SELECT` 查询的性能。 - `use_minimalistic_part_header_in_zookeeper` — 数据片段头在 ZooKeeper 中的存储方式。如果设置了 `use_minimalistic_part_header_in_zookeeper=1` ，ZooKeeper 会存储更少的数据。更多信息参考『服务配置参数』这章中的 [设置描述](../server_settings/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) 。 - `min_merge_bytes_to_use_direct_io` — 使用直接 I/O 来操作磁盘的合并操作时要求的最小数据量。合并数据片段时，ClickHouse 会计算要被合并的所有数据的总存储空间。如果大小超过了 `min_merge_bytes_to_use_direct_io` 设置的字节数，则 ClickHouse 将使用直接 I/O 接口（`O_DIRECT` 选项）对磁盘读写。如果设置 `min_merge_bytes_to_use_direct_io = 0` ，则会禁用直接 I/O。默认值：`10 * 1024 * 1024 * 1024` 字节。 + + - `merge_with_ttl_timeout` — TTL合并频率的最小间隔时间。默认值: 86400 (1 天)。 + - `write_final_mark` — 启用或禁用在数据片段尾部写入最终索引标记。默认值: 1（不建议更改）。 + - `storage_policy` — 存储策略。 参见 [使用多个区块装置进行数据存储](#table_engine-mergetree-multiple-volumes). **示例配置** @@ -115,7 +121,7 @@ MergeTree(EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID) 对于主要的配置方法，这里 `MergeTree` 引擎跟前面的例子一样，可以以同样的方式配置。 -## 数据存储 +## 数据存储 {#mergetree-data-storage} 表由按主键排序的数据 *片段* 组成。 @@ -297,4 +303,101 @@ INDEX sample_index3 (lower(str), str) TYPE ngrambf_v1(3, 256, 2, 0) GRANULARITY 对表的读操作是自动并行的。 +## 列和表的TTL {#table_engine-mergetree-ttl} + +TTL可以设置值的生命周期，它既可以为整张表设置，也可以为每个列字段单独设置。如果`TTL`同时作用于表和字段，ClickHouse会使用先到期的那个。 + +被设置TTL的表，必须拥有[Date](../../data_types/date.md) 或 [DateTime](../../data_types/datetime.md) 类型的字段。要定义数据的生命周期，需要在这个日期字段上使用操作符，例如: + +```sql +TTL time_column +TTL time_column + interval +``` + +要定义`interval`, 需要使用 [time interval](../../query_language/operators.md#operators-datetime) 操作符。 + +```sql +TTL date_time + INTERVAL 1 MONTH +TTL date_time + INTERVAL 15 HOUR +``` + +**列字段 TTL** + +当列字段中的值过期时, ClickHouse会将它们替换成数据类型的默认值。如果分区内，某一列的所有值均已过期，则ClickHouse会从文件系统中删除这个分区目录下的列文件。 + +`TTL`子句不能被用于主键字段。 + +示例说明: + +创建一张包含 `TTL` 的表 + +```sql +CREATE TABLE example_table +( + d DateTime, + a Int TTL d + INTERVAL 1 MONTH, + b Int TTL d + INTERVAL 1 MONTH, + c String +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(d) +ORDER BY d; +``` + +为表中已存在的列字段添加 `TTL` + +```sql +ALTER TABLE example_table + MODIFY COLUMN + c String TTL d + INTERVAL 1 DAY; +``` + +修改列字段的 `TTL` + +```sql +ALTER TABLE example_table + MODIFY COLUMN + c String TTL d + INTERVAL 1 MONTH; +``` + +**表 TTL** + +当表内的数据过期时, ClickHouse会删除所有对应的行。 + +举例说明: + +创建一张包含 `TTL` 的表 + +```sql +CREATE TABLE example_table +( + d DateTime, + a Int +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(d) +ORDER BY d +TTL d + INTERVAL 1 MONTH; +``` + +修改表的 `TTL` + +```sql +ALTER TABLE example_table + MODIFY TTL d + INTERVAL 1 DAY; +``` + +**删除数据** + +当ClickHouse合并数据分区时, 会删除TTL过期的数据。 + +当ClickHouse发现数据过期时, 它将会执行一个计划外的合并。要控制这类合并的频率, 你可以设置 [merge_with_ttl_timeout](#mergetree_setting-merge_with_ttl_timeout)。如果该值被设置的太低, 它将导致执行许多的计划外合并，这可能会消耗大量资源。 + +如果在合并的时候执行`SELECT` 查询, 则可能会得到过期的数据。为了避免这种情况，可以在`SELECT`之前使用 [OPTIMIZE](../../query_language/misc.md#misc_operations-optimize) 查询。 + + +## Using Multiple Block Devices for Data Storage {#table_engine-mergetree-multiple-volumes} + +### Configuration {#table_engine-mergetree-multiple-volumes_configure} + [来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/mergetree/) diff --git a/docs/zh/query_language/create.md b/docs/zh/query_language/create.md index 62630673540..54ef6ecb0b8 100644 --- a/docs/zh/query_language/create.md +++ b/docs/zh/query_language/create.md @@ -82,11 +82,9 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name ENGINE = engine AS SELECT ... ### Constraints {#constraints} -WARNING: This feature is experimental. Correct work is not guaranteed on non-MergeTree family engines. - Along with columns descriptions constraints could be defined: -``sql +```sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ( name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1], @@ -100,15 +98,15 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] Adding large amount of constraints can negatively affect performance of big `INSERT` queries. -### TTL expression +### TTL Expression Defines storage time for values. Can be specified only for MergeTree-family tables. For the detailed description, see [TTL for columns and tables](../operations/table_engines/mergetree.md#table_engine-mergetree-ttl). -## Column Compression Codecs +### Column Compression Codecs {#codecs} -By default, ClickHouse applies to columns the compression method, defined in [server settings](../operations/server_settings/settings.md#compression). Also, you can define compression method for each individual column in the `CREATE TABLE` query. +By default, ClickHouse applies the compression method, defined in [server settings](../operations/server_settings/settings.md#compression), to columns. You can also define the compression method for each individual column in the `CREATE TABLE` query. -``` +```sql CREATE TABLE codec_example ( dt Date CODEC(ZSTD), @@ -121,23 +119,23 @@ ENGINE = ... ``` -If a codec is specified, the default codec doesn't apply. Codecs can be combined in a pipeline, for example, `CODEC(Delta, ZSTD)`. To select the best codecs combination for you project, pass benchmarks, similar to described in the Altinity [New Encodings to Improve ClickHouse Efficiency](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse) article. +If a codec is specified, the default codec doesn't apply. Codecs can be combined in a pipeline, for example, `CODEC(Delta, ZSTD)`. To select the best codec combination for you project, pass benchmarks similar to described in the Altinity [New Encodings to Improve ClickHouse Efficiency](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse) article. -!!!warning - You cannot decompress ClickHouse database files with external utilities, for example, `lz4`. Use the special utility, [clickhouse-compressor](https://github.com/ClickHouse/ClickHouse/tree/master/dbms/programs/compressor). +!!!warning "Warning" + You can't decompress ClickHouse database files with external utilities like `lz4`. Instead, use the special [clickhouse-compressor](https://github.com/yandex/ClickHouse/tree/master/dbms/programs/compressor) utility. -Compression is supported for the table engines: +Compression is supported for the following table engines: -- [*MergeTree](../operations/table_engines/mergetree.md) family -- [*Log](../operations/table_engines/log_family.md) family +- [MergeTree](../operations/table_engines/mergetree.md) family +- [Log](../operations/table_engines/log_family.md) family - [Set](../operations/table_engines/set.md) - [Join](../operations/table_engines/join.md) ClickHouse supports common purpose codecs and specialized codecs. -### Specialized codecs {#create-query-specialized-codecs} +#### Specialized Codecs {#create-query-specialized-codecs} -These codecs are designed to make compression more effective using specifities of the data. Some of this codecs don't compress data by itself, but they prepare data to be compressed better by common purpose codecs. +These codecs are designed to make compression more effective by using specific features of data. Some of these codecs don't compress data themself. Instead, they prepare the data for a common purpose codec, which compresses it better than without this preparation. Specialized codecs: @@ -157,7 +155,7 @@ CREATE TABLE codec_example ENGINE = MergeTree() ``` -### Common purpose codecs {#create-query-common-purpose-codecs} +#### Common purpose codecs {#create-query-common-purpose-codecs} Codecs: @@ -166,7 +164,8 @@ Codecs: - `LZ4HC[(level)]` — LZ4 HC (high compression) algorithm with configurable level. Default level: 9. Setting `level <= 0` applies the default level. Possible levels: [1, 12]. Recommended level range: [4, 9]. - `ZSTD[(level)]` — [ZSTD compression algorithm](https://en.wikipedia.org/wiki/Zstandard) with configurable `level`. Possible levels: [1, 22]. Default value: 1. -High compression levels useful for asymmetric scenarios, like compress once, decompress a lot of times. Greater levels stands for better compression and higher CPU usage. +High compression levels are useful for asymmetric scenarios, like compress once, decompress repeatedly. Higher levels mean better compression and higher CPU usage. + ## 临时表 @@ -202,7 +201,6 @@ CREATE TABLE IF NOT EXISTS all_hits ON CLUSTER cluster (p Date, i Int32) ENGINE 为了能够正确的运行这种查询，每台主机必须具有相同的cluster声明（为了简化配置的同步，你可以使用zookeeper的方式进行配置）。同时这些主机还必须链接到zookeeper服务器。 这个查询将最终在集群的每台主机上运行，即使一些主机当前处于不可用状态。同时它还保证了所有的查询在单台主机中的执行顺序。 -replicated系列表还没有支持`ALTER`查询。 ## CREATE VIEW @@ -249,3 +247,19 @@ SELECT a, b, c FROM (SELECT ...) 没有单独的删除视图的语法。如果要删除视图，请使用`DROP TABLE`。 [来源文章](https://clickhouse.yandex/docs/en/query_language/create/) + +## CREATE DICTIONARY {#create-dictionary-query} + +```sql +CREATE DICTIONARY [IF NOT EXISTS] [db.]dictionary_name +( + key1 type1 [DEFAULT|EXPRESSION expr1] [HIERARCHICAL|INJECTIVE|IS_OBJECT_ID], + key2 type2 [DEFAULT|EXPRESSION expr2] [HIERARCHICAL|INJECTIVE|IS_OBJECT_ID], + attr1 type2 [DEFAULT|EXPRESSION expr3], + attr2 type2 [DEFAULT|EXPRESSION expr4] +) +PRIMARY KEY key1, key2 +SOURCE(SOURCE_NAME([param1 value1 ... paramN valueN])) +LAYOUT(LAYOUT_NAME([param_name param_value])) +LIFETIME([MIN val1] MAX val2) +``` diff --git a/docs/zh/query_language/functions/array_functions.md b/docs/zh/query_language/functions/array_functions.md index e655ce03427..9b760c7b6ae 100644 --- a/docs/zh/query_language/functions/array_functions.md +++ b/docs/zh/query_language/functions/array_functions.md @@ -384,7 +384,7 @@ arrayPushFront(array, single_value) **示例** ``` sql -SELECT arrayPushBack(['b'], 'a') AS res +SELECT arrayPushFront(['b'], 'a') AS res ``` ``` diff --git a/docs/zh/query_language/functions/geo.md b/docs/zh/query_language/functions/geo.md index 486457e34b3..2182aaa2fcb 100644 --- a/docs/zh/query_language/functions/geo.md +++ b/docs/zh/query_language/functions/geo.md @@ -4,7 +4,7 @@ 使用[great-circle distance公式](https://en.wikipedia.org/wiki/Great-circle_distance)计算地球表面两点之间的距离。 -``` +```sql greatCircleDistance(lon1Deg, lat1Deg, lon2Deg, lat2Deg) ``` @@ -25,11 +25,11 @@ greatCircleDistance(lon1Deg, lat1Deg, lon2Deg, lat2Deg) **示例** -``` sql +```sql SELECT greatCircleDistance(55.755831, 37.617673, -55.755831, -37.617673) ``` -``` +```text ┌─greatCircleDistance(55.755831, 37.617673, -55.755831, -37.617673)─┐ │ 14132374.194975413 │ └───────────────────────────────────────────────────────────────────┘ @@ -37,9 +37,10 @@ SELECT greatCircleDistance(55.755831, 37.617673, -55.755831, -37.617673) ## pointInEllipses -检查指定的点是否至少包含在一个指定的椭圆中。 +检查指定的点是否至少包含在指定的一个椭圆中。 +下述中的坐标是几何图形在笛卡尔坐标系中的位置。 -``` +```sql pointInEllipses(x, y, x₀, y₀, a₀, b₀,...,xₙ, yₙ, aₙ, bₙ) ``` @@ -47,7 +48,7 @@ pointInEllipses(x, y, x₀, y₀, a₀, b₀,...,xₙ, yₙ, aₙ, bₙ) - `x, y` — 平面上某个点的坐标。 - `xᵢ, yᵢ` — 第i个椭圆的中心坐标。 -- `aᵢ, bᵢ` — 第i个椭圆的轴，单位：米。 +- `aᵢ, bᵢ` — 以x, y坐标为单位的第i个椭圆的轴。 输入参数的个数必须是`2+4⋅n`，其中`n`是椭圆的数量。 @@ -57,11 +58,11 @@ pointInEllipses(x, y, x₀, y₀, a₀, b₀,...,xₙ, yₙ, aₙ, bₙ) **示例** -``` sql +```sql SELECT pointInEllipses(55.755831, 37.617673, 55.755831, 37.617673, 1.0, 2.0) ``` -``` +```text ┌─pointInEllipses(55.755831, 37.617673, 55.755831, 37.617673, 1., 2.)─┐ │ 1 │ └─────────────────────────────────────────────────────────────────────┘ @@ -71,7 +72,7 @@ SELECT pointInEllipses(55.755831, 37.617673, 55.755831, 37.617673, 1.0, 2.0) 检查指定的点是否包含在指定的多边形中。 -``` +```sql pointInPolygon((x, y), [(a, b), (c, d) ...], ...) ``` @@ -88,11 +89,11 @@ pointInPolygon((x, y), [(a, b), (c, d) ...], ...) **示例** -``` sql +```sql SELECT pointInPolygon((3., 3.), [(6, 0), (8, 4), (5, 8), (0, 2)]) AS res ``` -``` +```text ┌─res─┐ │ 1 │ └─────┘ @@ -101,7 +102,7 @@ SELECT pointInPolygon((3., 3.), [(6, 0), (8, 4), (5, 8), (0, 2)]) AS res ## geohashEncode 将经度和纬度编码为geohash-string，请参阅（http://geohash.org/,https://en.wikipedia.org/wiki/Geohash）。 -``` +```sql geohashEncode(longitude, latitude, [precision]) ``` @@ -117,11 +118,11 @@ geohashEncode(longitude, latitude, [precision]) **示例** -``` sql +```sql SELECT geohashEncode(-5.60302734375, 42.593994140625, 0) AS res ``` -``` +```text ┌─res──────────┐ │ ezs42d000000 │ └──────────────┘ @@ -141,14 +142,79 @@ SELECT geohashEncode(-5.60302734375, 42.593994140625, 0) AS res **示例** -``` sql +```sql SELECT geohashDecode('ezs42') AS res ``` -``` +```text ┌─res─────────────────────────────┐ │ (-5.60302734375,42.60498046875) │ └─────────────────────────────────┘ ``` + +## geoToH3 + +计算指定的分辨率的[H3](https://uber.github.io/h3/#/documentation/overview/introduction)索引`(lon, lat)`。 + +```sql +geoToH3(lon, lat, resolution) +``` + +**输入值** + +- `lon` — 经度。 [Float64](../../data_types/float.md)类型。 +- `lat` — 纬度。 [Float64](../../data_types/float.md)类型。 +- `resolution` — 索引的分辨率。 取值范围为: `[0, 15]`。 [UInt8](../../data_types/int_uint.md)类型。 + +**返回值** + +- H3中六边形的索引值。 +- 发生异常时返回0。 + +[UInt64](../../data_types/int_uint.md)类型。 + +**示例** + +```sql +SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index +``` +```text +┌────────────h3Index─┐ +│ 644325524701193974 │ +└────────────────────┘ +``` + +## geohashesInBox + +计算在指定精度下计算最小包含指定的经纬范围的最小图形的geohash数组。 + +**输入值** + +- longitude_min - 最小经度。其值应在`[-180°，180°]`范围内 +- latitude_min - 最小纬度。其值应在`[-90°，90°]`范围内 +- longitude_max - 最大经度。其值应在`[-180°，180°]`范围内 +- latitude_max - 最大纬度。其值应在`[-90°，90°]`范围内 +- precision - geohash的精度。其值应在`[1, 12]`内的`UInt8`类型的数字 + +请注意，上述所有的坐标参数必须同为`Float32`或`Float64`中的一种类型。 + +**返回值** + +- 包含指定范围内的指定精度的geohash字符串数组。注意，您不应该依赖返回数组中geohash的顺序。 +- [] - 当传入的最小经纬度大于最大经纬度时将返回一个空数组。 + +请注意，如果生成的数组长度超过10000时，则函数将抛出异常。 + +**示例** + +```sql +SELECT geohashesInBox(24.48, 40.56, 24.785, 40.81, 4) AS thasos +``` +```text +┌─thasos──────────────────────────────────────┐ +│ ['sx1q','sx1r','sx32','sx1w','sx1x','sx38'] │ +└─────────────────────────────────────────────┘ +``` + [来源文章](https://clickhouse.yandex/docs/en/query_language/functions/geo/) diff --git a/docs/zh/query_language/functions/higher_order_functions.md b/docs/zh/query_language/functions/higher_order_functions.md index e64db0bc8d3..39c6770e5b8 100644 --- a/docs/zh/query_language/functions/higher_order_functions.md +++ b/docs/zh/query_language/functions/higher_order_functions.md @@ -12,7 +12,7 @@ 除了'arrayMap'和'arrayFilter'以外的所有其他函数，都可以省略第一个参数（lambda函数）。在这种情况下，默认返回数组元素本身。 -### arrayMap(func, arr1, ...) +### arrayMap(func, arr1, ...) {#higher_order_functions-array-map} 将arr 将从'func'函数的原始应用程序获得的数组返回到'arr'数组中的每个元素。 diff --git a/docs/zh/query_language/functions/introspection.md b/docs/zh/query_language/functions/introspection.md new file mode 120000 index 00000000000..b1a487e9c77 --- /dev/null +++ b/docs/zh/query_language/functions/introspection.md @@ -0,0 +1 @@ +../../../en/query_language/functions/introspection.md \ No newline at end of file diff --git a/docs/zh/query_language/functions/type_conversion_functions.md b/docs/zh/query_language/functions/type_conversion_functions.md index 70ccc191e19..3cabf07643b 100644 --- a/docs/zh/query_language/functions/type_conversion_functions.md +++ b/docs/zh/query_language/functions/type_conversion_functions.md @@ -145,7 +145,7 @@ SELECT toTypeName(CAST(x, 'Nullable(UInt16)')) FROM t_null └─────────────────────────────────────────┘ ``` -## toIntervalYear, toIntervalQuarter, toIntervalMonth, toIntervalWeek, toIntervalDay, toIntervalHour, toIntervalMinute, toIntervalSecond +## toIntervalYear, toIntervalQuarter, toIntervalMonth, toIntervalWeek, toIntervalDay, toIntervalHour, toIntervalMinute, toIntervalSecond {#function-tointerval} 将数字类型参数转换为Interval类型（时间区间）。 Interval类型实际上是非常有用的，您可以使用此类型的数据直接与Date或DateTime执行算术运算。同时，ClickHouse为Interval类型数据的声明提供了更方便的语法。例如： diff --git a/docs/zh/query_language/operators.md b/docs/zh/query_language/operators.md index 2e1dec00897..9dbddd681ed 100644 --- a/docs/zh/query_language/operators.md +++ b/docs/zh/query_language/operators.md @@ -82,6 +82,92 @@ 条件运算符会先计算表达式b和表达式c的值，再根据表达式a的真假，返回相应的值。如果表达式b和表达式c是 [arrayJoin()](functions/array_join.md#functions_arrayjoin) 函数，则不管表达式a是真是假，每行都会被复制展开。 + +## Operators for Working with Dates and Times {#operators-datetime} + +### EXTRACT {#operator-extract} + +```sql +EXTRACT(part FROM date); +``` + +Extracts a part from a given date. For example, you can retrieve a month from a given date, or a second from a time. + +The `part` parameter specifies which part of the date to retrieve. The following values are available: + +- `DAY` — The day of the month. Possible values: 1–31. +- `MONTH` — The number of a month. Possible values: 1–12. +- `YEAR` — The year. +- `SECOND` — The second. Possible values: 0–59. +- `MINUTE` — The minute. Possible values: 0–59. +- `HOUR` — The hour. Possible values: 0–23. + +The `part` parameter is case-insensitive. + +The `date` parameter specifies the date or the time to process. Either [Date](../data_types/date.md) or [DateTime](../data_types/datetime.md) type is supported. + +Examples: + +```sql +SELECT EXTRACT(DAY FROM toDate('2017-06-15')); +SELECT EXTRACT(MONTH FROM toDate('2017-06-15')); +SELECT EXTRACT(YEAR FROM toDate('2017-06-15')); +``` + +In the following example we create a table and insert into it a value with the `DateTime` type. + +```sql +CREATE TABLE test.Orders +( + OrderId UInt64, + OrderName String, + OrderDate DateTime +) +ENGINE = Log; +``` + +```sql +INSERT INTO test.Orders VALUES (1, 'Jarlsberg Cheese', toDateTime('2008-10-11 13:23:44')); +``` +```sql +SELECT + toYear(OrderDate) AS OrderYear, + toMonth(OrderDate) AS OrderMonth, + toDayOfMonth(OrderDate) AS OrderDay, + toHour(OrderDate) AS OrderHour, + toMinute(OrderDate) AS OrderMinute, + toSecond(OrderDate) AS OrderSecond +FROM test.Orders; +``` +```text +┌─OrderYear─┬─OrderMonth─┬─OrderDay─┬─OrderHour─┬─OrderMinute─┬─OrderSecond─┐ +│ 2008 │ 10 │ 11 │ 13 │ 23 │ 44 │ +└───────────┴────────────┴──────────┴───────────┴─────────────┴─────────────┘ +``` + +You can see more examples in [tests](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00619_extract.sql). + +### INTERVAL {#operator-interval} + +Creates an [Interval](../data_types/special_data_types/interval.md)-type value that should be used in arithmetical operations with [Date](../data_types/date.md) and [DateTime](../data_types/datetime.md)-type values. + +Example: + +```sql +SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL 3 HOUR +``` +```text +┌───current_date_time─┬─plus(plus(now(), toIntervalDay(4)), toIntervalHour(3))─┐ +│ 2019-10-23 11:16:28 │ 2019-10-27 14:16:28 │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +``` + +**See Also** + +- [Interval](../data_types/special_data_types/interval.md) data type +- [toInterval](functions/type_conversion_functions.md#function-tointerval) type convertion functions + + ## CASE条件表达式 {#operator_case} ``` sql diff --git a/docs/zh/roadmap.md b/docs/zh/roadmap.md deleted file mode 120000 index 24df86352b3..00000000000 --- a/docs/zh/roadmap.md +++ /dev/null @@ -1 +0,0 @@ -../en/roadmap.md \ No newline at end of file diff --git a/docs/zh/roadmap.md b/docs/zh/roadmap.md new file mode 100644 index 00000000000..3be2aa01533 --- /dev/null +++ b/docs/zh/roadmap.md @@ -0,0 +1,16 @@ +# 规划 + +## Q3 2019 + +- 字典表的DDL +- 与类S3对象存储集成 +- 冷热数据存储分离，支持JBOD + +## Q4 2019 + +- JOIN 不受可用内存限制 +- 更精确的用户资源池，可以在用户之间合理分配集群资源 +- 细粒度的授权管理 +- 与外部认证服务集成 + +[来源文章](https://clickhouse.yandex/docs/en/roadmap/) diff --git a/docs/zh/security_changelog.md b/docs/zh/security_changelog.md deleted file mode 120000 index 101a4f4e48c..00000000000 --- a/docs/zh/security_changelog.md +++ /dev/null @@ -1 +0,0 @@ -../en/security_changelog.md \ No newline at end of file diff --git a/docs/zh/security_changelog.md b/docs/zh/security_changelog.md new file mode 100644 index 00000000000..f4e5910c6d2 --- /dev/null +++ b/docs/zh/security_changelog.md @@ -0,0 +1,39 @@ +## 修复于 ClickHouse Release 18.12.13, 2018-09-10 + +### CVE-2018-14672 + +加载CatBoost模型的功能，允许遍历路径并通过错误消息读取任意文件。 + +来源: Yandex信息安全团队的Andrey Krasichkov + +## 修复于 ClickHouse Release 18.10.3, 2018-08-13 + +### CVE-2018-14671 + +unixODBC允许从文件系统加载任意共享对象，从而导致“远程执行代码”漏洞。 + +来源：Yandex信息安全团队的Andrey Krasichkov和Evgeny Sidorov + +## 修复于 ClickHouse Release 1.1.54388, 2018-06-28 + +### CVE-2018-14668 +远程表函数功能允许在 "user", "password" 及 "default_database" 字段中使用任意符号，从而导致跨协议请求伪造攻击。 + +来源：Yandex信息安全团队的Andrey Krasichkov + +## 修复于 ClickHouse Release 1.1.54390, 2018-07-06 + +### CVE-2018-14669 +ClickHouse MySQL客户端启用了 "LOAD DATA LOCAL INFILE" 功能，该功能允许恶意MySQL数据库从连接的ClickHouse服务器读取任意文件。 + +来源：Yandex信息安全团队的Andrey Krasichkov和Evgeny Sidorov + +## 修复于 ClickHouse Release 1.1.54131, 2017-01-10 + +### CVE-2018-14670 + +deb软件包中的错误配置可能导致使用未经授权的数据库。 + +来源：英国国家网络安全中心（NCSC） + +[来源文章](https://clickhouse.yandex/docs/en/security_changelog/) diff --git a/libs/consistent-hashing/bitops.h b/libs/consistent-hashing/bitops.h index 697063ee77e..e07d5045cd3 100644 --- a/libs/consistent-hashing/bitops.h +++ b/libs/consistent-hashing/bitops.h @@ -26,7 +26,7 @@ inline uint32_t HI_32(uint64_t x) { return static_cast(x >> 32); } return std::numeric_limits::digits - __builtin_clzll(value); } #else - /// Stupid realization for non GCC-like compilers. Can use BSR from x86 instructions set. + /// Stupid implementation for non GCC-like compilers. Can use BSR from x86 instructions set. template inline unsigned GetValueBitCountImpl(T value) noexcept { unsigned result = 1; // result == 0 - impossible value, since value cannot be zero diff --git a/libs/libcommon/CMakeLists.txt b/libs/libcommon/CMakeLists.txt index 357e457b240..3e58cba0164 100644 --- a/libs/libcommon/CMakeLists.txt +++ b/libs/libcommon/CMakeLists.txt @@ -53,6 +53,7 @@ add_library (common include/common/phdr_cache.h include/ext/bit_cast.h + include/ext/chrono_io.h include/ext/collection_cast.h include/ext/enumerate.h include/ext/function_traits.h diff --git a/libs/libcommon/cmake/find_gperftools.cmake b/libs/libcommon/cmake/find_gperftools.cmake deleted file mode 100644 index 73e8182d390..00000000000 --- a/libs/libcommon/cmake/find_gperftools.cmake +++ /dev/null @@ -1,31 +0,0 @@ -if (OS_FREEBSD OR ARCH_32) - option (USE_INTERNAL_GPERFTOOLS_LIBRARY "Set to FALSE to use system gperftools (tcmalloc) library instead of bundled" OFF) -else () - option (USE_INTERNAL_GPERFTOOLS_LIBRARY "Set to FALSE to use system gperftools (tcmalloc) library instead of bundled" ${NOT_UNBUNDLED}) -endif () - -option (ENABLE_TCMALLOC "Set to TRUE to enable tcmalloc" OFF) -option (DEBUG_TCMALLOC "Set to TRUE to use debug version of libtcmalloc" OFF) - -if (ENABLE_TCMALLOC) - #contrib/libtcmalloc doesnt build debug version, try find in system - if (DEBUG_TCMALLOC OR NOT USE_INTERNAL_GPERFTOOLS_LIBRARY) - find_package (Gperftools) - endif () - - if (NOT (GPERFTOOLS_FOUND AND GPERFTOOLS_INCLUDE_DIR AND GPERFTOOLS_TCMALLOC_MINIMAL) AND NOT (OS_FREEBSD OR ARCH_32)) - set (USE_INTERNAL_GPERFTOOLS_LIBRARY 1) - set (GPERFTOOLS_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libtcmalloc/include") - set (GPERFTOOLS_TCMALLOC_MINIMAL tcmalloc_minimal_internal) - endif () - - if (GPERFTOOLS_FOUND OR USE_INTERNAL_GPERFTOOLS_LIBRARY) - set (USE_TCMALLOC 1) - endif () - - if (SANITIZE) - message (FATAL_ERROR "ENABLE_TCMALLOC is set to true, but it cannot be used with sanitizers") - endif () - - message (STATUS "Using tcmalloc=${USE_TCMALLOC}: ${GPERFTOOLS_INCLUDE_DIR} : ${GPERFTOOLS_TCMALLOC_MINIMAL}") -endif () diff --git a/libs/libcommon/include/common/DateLUTImpl.h b/libs/libcommon/include/common/DateLUTImpl.h index ef50d6ede3f..7f1e8c74313 100644 --- a/libs/libcommon/include/common/DateLUTImpl.h +++ b/libs/libcommon/include/common/DateLUTImpl.h @@ -666,7 +666,7 @@ public: inline DayNum makeDayNum(UInt16 year, UInt8 month, UInt8 day_of_month) const { if (unlikely(year < DATE_LUT_MIN_YEAR || year > DATE_LUT_MAX_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31)) - return DayNum(0); + return DayNum(0); // TODO (nemkov, DateTime64 phase 2): implement creating real date for year outside of LUT range. return DayNum(years_months_lut[(year - DATE_LUT_MIN_YEAR) * 12 + month - 1] + day_of_month - 1); } diff --git a/libs/libcommon/include/common/Types.h b/libs/libcommon/include/common/Types.h index 70c9c3d2f3c..5d933f218c1 100644 --- a/libs/libcommon/include/common/Types.h +++ b/libs/libcommon/include/common/Types.h @@ -1,5 +1,8 @@ #pragma once #include +#include +#include +#include using Int8 = int8_t; using Int16 = int16_t; @@ -10,3 +13,43 @@ using UInt8 = uint8_t; using UInt16 = uint16_t; using UInt32 = uint32_t; using UInt64 = uint64_t; + +/// The standard library type traits, such as std::is_arithmetic, with one exception +/// (std::common_type), are "set in stone". Attempting to specialize them causes undefined behavior. +/// So instead of using the std type_traits, we use our own version which allows extension. +template +struct is_signed +{ + static constexpr bool value = std::is_signed_v; +}; + +template +inline constexpr bool is_signed_v = is_signed::value; + +template +struct is_unsigned +{ + static constexpr bool value = std::is_unsigned_v; +}; + +template +inline constexpr bool is_unsigned_v = is_unsigned::value; + +template +struct is_integral +{ + static constexpr bool value = std::is_integral_v; +}; + +template +inline constexpr bool is_integral_v = is_integral::value; + +template +struct is_arithmetic +{ + static constexpr bool value = std::is_arithmetic_v; +}; + +template +inline constexpr bool is_arithmetic_v = is_arithmetic::value; + diff --git a/libs/libcommon/include/common/iostream_debug_helpers.h b/libs/libcommon/include/common/iostream_debug_helpers.h index 9149ffb5ed0..72891ed03a5 100644 --- a/libs/libcommon/include/common/iostream_debug_helpers.h +++ b/libs/libcommon/include/common/iostream_debug_helpers.h @@ -140,7 +140,7 @@ Out & dump(Out & out, const char * name, T && x) #pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments" #endif -#define DUMPVAR(VAR) dump(std::cerr, #VAR, (VAR)); std::cerr << "; "; +#define DUMPVAR(VAR) ::dump(std::cerr, #VAR, (VAR)); std::cerr << "; "; #define DUMPHEAD std::cerr << __FILE__ << ':' << __LINE__ << " [ " << getThreadNumber() << " ] "; #define DUMPTAIL std::cerr << '\n'; diff --git a/libs/libcommon/include/common/logger_useful.h b/libs/libcommon/include/common/logger_useful.h index b4693115cb3..ea1a25cc8fa 100644 --- a/libs/libcommon/include/common/logger_useful.h +++ b/libs/libcommon/include/common/logger_useful.h @@ -6,7 +6,6 @@ #include #include #include -#include #include #ifndef QUERY_PREVIEW_LENGTH diff --git a/libs/libcommon/include/ext/chrono_io.h b/libs/libcommon/include/ext/chrono_io.h new file mode 100644 index 00000000000..8fa448b9e6a --- /dev/null +++ b/libs/libcommon/include/ext/chrono_io.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include + + +namespace ext +{ + template + std::string to_string(const std::chrono::time_point & tp) + { + return DateLUT::instance().timeToString(std::chrono::system_clock::to_time_t(tp)); + } + + template > + std::string to_string(const std::chrono::duration & dur) + { + auto seconds_as_int = std::chrono::duration_cast(dur); + if (seconds_as_int == dur) + return std::to_string(seconds_as_int.count()) + "s"; + auto seconds_as_double = std::chrono::duration_cast>(dur); + return std::to_string(seconds_as_double.count()) + "s"; + } + + template + std::ostream & operator<<(std::ostream & o, const std::chrono::time_point & tp) + { + return o << to_string(tp); + } + + template > + std::ostream & operator<<(std::ostream & o, const std::chrono::duration & dur) + { + return o << to_string(dur); + } +} diff --git a/libs/libcommon/include/ext/range.h b/libs/libcommon/include/ext/range.h index 61b644c2ce5..c379d453f7b 100644 --- a/libs/libcommon/include/ext/range.h +++ b/libs/libcommon/include/ext/range.h @@ -1,46 +1,42 @@ #pragma once #include -#include -#include -#include +#include +#include -/** Numeric range iterator, used to represent a half-closed interval [begin, end). - * In conjunction with std::reverse_iterator allows for forward and backward iteration - * over corresponding interval. - */ namespace ext { - template - using range_iterator = boost::counting_iterator; - - /** Range-based for loop adapter for (reverse_)range_iterator. - * By and large should be in conjunction with ext::range and ext::reverse_range. - */ - template - struct range_wrapper + /// For loop adaptor which is used to iterate through a half-closed interval [begin, end). + template + inline auto range(BeginType begin, EndType end) { - using value_type = typename std::remove_reference::type; - using iterator = range_iterator; + using CommonType = typename std::common_type::type; + return boost::counting_range(begin, end); + } - value_type begin_; - value_type end_; - - iterator begin() const { return iterator(begin_); } - iterator end() const { return iterator(end_); } - }; - - /** Constructs range_wrapper for forward-iteration over [begin, end) in range-based for loop. - * Usage example: - * for (const auto i : ext::range(0, 4)) print(i); - * Output: - * 0 1 2 3 - */ - template - inline range_wrapper::type> range(T1 begin, T2 end) + template + inline auto range(Type end) { - using common_type = typename std::common_type::type; - return { static_cast(begin), static_cast(end) }; + return range(static_cast(0), end); + } + + /// The same as range(), but every value is casted statically to a specified `ValueType`. + /// This is useful to iterate through all constants of a enum. + template + inline auto range_with_static_cast(BeginType begin, EndType end) + { + using CommonType = typename std::common_type::type; + if constexpr (std::is_same_v) + return boost::counting_range(begin, end); + else + return boost::counting_range(begin, end) + | boost::adaptors::transformed([](CommonType x) -> ValueType { return static_cast(x); }); + } + + template + inline auto range_with_static_cast(EndType end) + { + return range_with_static_cast(static_cast(0), end); } } diff --git a/libs/libcommon/include/ext/shared_ptr_helper.h b/libs/libcommon/include/ext/shared_ptr_helper.h index ca7219e6261..df132382fa6 100644 --- a/libs/libcommon/include/ext/shared_ptr_helper.h +++ b/libs/libcommon/include/ext/shared_ptr_helper.h @@ -20,4 +20,20 @@ struct shared_ptr_helper } }; + +template +struct is_shared_ptr +{ + static constexpr bool value = false; +}; + + +template +struct is_shared_ptr> +{ + static constexpr bool value = true; +}; + +template +inline constexpr bool is_shared_ptr_v = is_shared_ptr::value; } diff --git a/libs/libdaemon/src/BaseDaemon.cpp b/libs/libdaemon/src/BaseDaemon.cpp index 931d91bd8b5..15b61c9b454 100644 --- a/libs/libdaemon/src/BaseDaemon.cpp +++ b/libs/libdaemon/src/BaseDaemon.cpp @@ -110,7 +110,7 @@ static void faultSignalHandler(int sig, siginfo_t * info, void * context) out.next(); - if (sig != SIGPROF) /// This signal is used for debugging. + if (sig != SIGTSTP) /// This signal is used for debugging. { /// The time that is usually enough for separate thread to print info into log. ::sleep(10); @@ -719,9 +719,9 @@ void BaseDaemon::initializeTerminationAndSignalProcessing() } }; - /// SIGPROF is added for debugging purposes. To output a stack trace of any running thread at anytime. + /// SIGTSTP is added for debugging purposes. To output a stack trace of any running thread at anytime. - add_signal_handler({SIGABRT, SIGSEGV, SIGILL, SIGBUS, SIGSYS, SIGFPE, SIGPIPE, SIGPROF}, faultSignalHandler); + add_signal_handler({SIGABRT, SIGSEGV, SIGILL, SIGBUS, SIGSYS, SIGFPE, SIGPIPE, SIGTSTP}, faultSignalHandler); add_signal_handler({SIGHUP, SIGUSR1}, closeLogsSignalHandler); add_signal_handler({SIGINT, SIGQUIT, SIGTERM}, terminateRequestedSignalHandler); @@ -731,7 +731,6 @@ void BaseDaemon::initializeTerminationAndSignalProcessing() signal_listener.reset(new SignalListener(*this)); signal_listener_thread.start(*signal_listener); - } void BaseDaemon::logRevision() const @@ -891,4 +890,3 @@ void BaseDaemon::waitForTerminationRequest() std::unique_lock lock(signal_handler_mutex); signal_event.wait(lock, [this](){ return terminate_signals_counter > 0; }); } - diff --git a/libs/libglibc-compatibility/CMakeLists.txt b/libs/libglibc-compatibility/CMakeLists.txt index 2dbec5fa772..8405c9450a5 100644 --- a/libs/libglibc-compatibility/CMakeLists.txt +++ b/libs/libglibc-compatibility/CMakeLists.txt @@ -31,8 +31,17 @@ if (GLIBC_COMPATIBILITY) list(APPEND glibc_compatibility_sources libcxxabi/cxa_thread_atexit.cpp) endif() + # Need to omit frame pointers to match the performance of glibc + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fomit-frame-pointer") + add_library(glibc-compatibility STATIC ${glibc_compatibility_sources}) + if (COMPILER_CLANG) + target_compile_options(glibc-compatibility PRIVATE -Wno-unused-command-line-argument) + elseif (COMPILER_GCC) + target_compile_options(glibc-compatibility PRIVATE -Wno-unused-but-set-variable) + endif () + target_include_directories(glibc-compatibility PRIVATE libcxxabi ${musl_arch_include_dir}) if (NOT USE_STATIC_LIBRARIES AND NOT MAKE_STATIC_LIBRARIES) diff --git a/libs/libglibc-compatibility/musl/aarch64/syscall_arch.h b/libs/libglibc-compatibility/musl/aarch64/syscall_arch.h new file mode 100644 index 00000000000..0588c15484c --- /dev/null +++ b/libs/libglibc-compatibility/musl/aarch64/syscall_arch.h @@ -0,0 +1,3 @@ +#define VDSO_USEFUL +#define VDSO_CGT_SYM "__kernel_clock_gettime" +#define VDSO_CGT_VER "LINUX_2.6.39" diff --git a/libs/libglibc-compatibility/musl/clock_gettime.c b/libs/libglibc-compatibility/musl/clock_gettime.c new file mode 100644 index 00000000000..574f9b83d15 --- /dev/null +++ b/libs/libglibc-compatibility/musl/clock_gettime.c @@ -0,0 +1,108 @@ +#include +#include +#include +#include "atomic.h" +#include "musl_features.h" +#include "syscall.h" + +#ifdef VDSO_CGT_SYM + +static void *volatile vdso_func; + +#ifdef VDSO_CGT32_SYM +static void *volatile vdso_func_32; +static int cgt_time32_wrap(clockid_t clk, struct timespec *ts) +{ + long ts32[2]; + int (*f)(clockid_t, long[2]) = + (int (*)(clockid_t, long[2]))vdso_func_32; + int r = f(clk, ts32); + if (!r) { + /* Fallback to syscalls if time32 overflowed. Maybe + * we lucked out and somehow migrated to a kernel with + * time64 syscalls available. */ + if (ts32[0] < 0) { + a_cas_p(&vdso_func, (void *)cgt_time32_wrap, 0); + return -ENOSYS; + } + ts->tv_sec = ts32[0]; + ts->tv_nsec = ts32[1]; + } + return r; +} +#endif + +static int cgt_init(clockid_t clk, struct timespec *ts) +{ + void *p = __vdsosym(VDSO_CGT_VER, VDSO_CGT_SYM); +#ifdef VDSO_CGT32_SYM + if (!p) { + void *q = __vdsosym(VDSO_CGT32_VER, VDSO_CGT32_SYM); + if (q) { + a_cas_p(&vdso_func_32, 0, q); + p = cgt_time32_wrap; + } + } +#endif + int (*f)(clockid_t, struct timespec *) = + (int (*)(clockid_t, struct timespec *))p; + a_cas_p(&vdso_func, (void *)cgt_init, p); + return f ? f(clk, ts) : -ENOSYS; +} + +static void *volatile vdso_func = (void *)cgt_init; + +#endif + +int __clock_gettime(clockid_t clk, struct timespec *ts) +{ + int r; + +#ifdef VDSO_CGT_SYM + int (*f)(clockid_t, struct timespec *) = + (int (*)(clockid_t, struct timespec *))vdso_func; + if (f) { + r = f(clk, ts); + if (!r) return r; + if (r == -EINVAL) return __syscall_ret(r); + /* Fall through on errors other than EINVAL. Some buggy + * vdso implementations return ENOSYS for clocks they + * can't handle, rather than making the syscall. This + * also handles the case where cgt_init fails to find + * a vdso function to use. */ + } +#endif + +#ifdef SYS_clock_gettime64 + r = -ENOSYS; + if (sizeof(time_t) > 4) + r = __syscall(SYS_clock_gettime64, clk, ts); + if (SYS_clock_gettime == SYS_clock_gettime64 || r!=-ENOSYS) + return __syscall_ret(r); + long ts32[2]; + r = __syscall(SYS_clock_gettime, clk, ts32); + if (r==-ENOSYS && clk==CLOCK_REALTIME) { + r = __syscall(SYS_gettimeofday, ts32, 0); + ts32[1] *= 1000; + } + if (!r) { + ts->tv_sec = ts32[0]; + ts->tv_nsec = ts32[1]; + return r; + } + return __syscall_ret(r); +#else + r = __syscall(SYS_clock_gettime, clk, ts); + if (r == -ENOSYS) { + if (clk == CLOCK_REALTIME) { + __syscall(SYS_gettimeofday, ts, 0); + ts->tv_nsec = (int)ts->tv_nsec * 1000; + return 0; + } + r = -EINVAL; + } + return __syscall_ret(r); +#endif +} + +weak_alias(__clock_gettime, clock_gettime); diff --git a/libs/libglibc-compatibility/musl/clock_nanosleep.c b/libs/libglibc-compatibility/musl/clock_nanosleep.c new file mode 100644 index 00000000000..bf71a5e84ac --- /dev/null +++ b/libs/libglibc-compatibility/musl/clock_nanosleep.c @@ -0,0 +1,27 @@ +#include +#include +#include +#include "musl_features.h" +#include "syscall.h" + +int __clock_nanosleep(clockid_t clk, int flags, const struct timespec * req, struct timespec * rem) +{ + if (clk == CLOCK_THREAD_CPUTIME_ID) + return EINVAL; + int old_cancel_type; + int status; + /// We cannot port __syscall_cp because musl has very limited cancellation point implementation. + /// For example, c++ destructors won't get called and exception unwinding isn't implemented. + /// Instead, we use normal __syscall here and turn on the asynchrous cancel mode to allow + /// cancel. This works because nanosleep doesn't contain any resource allocations or + /// deallocations. + pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_cancel_type); + if (clk == CLOCK_REALTIME && !flags) + status = -__syscall(SYS_nanosleep, req, rem); + else + status = -__syscall(SYS_clock_nanosleep, clk, flags, req, rem); + pthread_setcanceltype(old_cancel_type, NULL); + return status; +} + +weak_alias(__clock_nanosleep, clock_nanosleep); diff --git a/libs/libglibc-compatibility/musl/log2f.c b/libs/libglibc-compatibility/musl/log2f.c new file mode 100644 index 00000000000..c368f88f33f --- /dev/null +++ b/libs/libglibc-compatibility/musl/log2f.c @@ -0,0 +1,72 @@ +/* + * Single-precision log2 function. + * + * Copyright (c) 2017-2018, Arm Limited. + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include "libm.h" +#include "log2f_data.h" + +/* +LOG2F_TABLE_BITS = 4 +LOG2F_POLY_ORDER = 4 + +ULP error: 0.752 (nearest rounding.) +Relative error: 1.9 * 2^-26 (before rounding.) +*/ + +#define N (1 << LOG2F_TABLE_BITS) +#define T __log2f_data.tab +#define A __log2f_data.poly +#define OFF 0x3f330000 + +float log2f(float x) +{ + double_t z, r, r2, p, y, y0, invc, logc; + uint32_t ix, iz, top, tmp; + int k, i; + + ix = asuint(x); + /* Fix sign of zero with downward rounding when x==1. */ + if (WANT_ROUNDING && predict_false(ix == 0x3f800000)) + return 0; + if (predict_false(ix - 0x00800000 >= 0x7f800000 - 0x00800000)) { + /* x < 0x1p-126 or inf or nan. */ + if (ix * 2 == 0) + return __math_divzerof(1); + if (ix == 0x7f800000) /* log2(inf) == inf. */ + return x; + if ((ix & 0x80000000) || ix * 2 >= 0xff000000) + return __math_invalidf(x); + /* x is subnormal, normalize it. */ + ix = asuint(x * 0x1p23f); + ix -= 23 << 23; + } + + /* x = 2^k z; where z is in range [OFF,2*OFF] and exact. + The range is split into N subintervals. + The ith subinterval contains z and c is near its center. */ + tmp = ix - OFF; + i = (tmp >> (23 - LOG2F_TABLE_BITS)) % N; + top = tmp & 0xff800000; + iz = ix - top; + k = (int32_t)tmp >> 23; /* arithmetic shift */ + invc = T[i].invc; + logc = T[i].logc; + z = (double_t)asfloat(iz); + + /* log2(x) = log1p(z/c-1)/ln2 + log2(c) + k */ + r = z * invc - 1; + y0 = logc + (double_t)k; + + /* Pipelined polynomial evaluation to approximate log1p(r)/ln2. */ + r2 = r * r; + y = A[1] * r + A[2]; + y = A[0] * r2 + y; + p = A[3] * r + y0; + y = y * r2 + p; + return eval_as_float(y); +} diff --git a/libs/libglibc-compatibility/musl/log2f_data.c b/libs/libglibc-compatibility/musl/log2f_data.c new file mode 100644 index 00000000000..24e450f1ec3 --- /dev/null +++ b/libs/libglibc-compatibility/musl/log2f_data.c @@ -0,0 +1,33 @@ +/* + * Data definition for log2f. + * + * Copyright (c) 2017-2018, Arm Limited. + * SPDX-License-Identifier: MIT + */ + +#include "log2f_data.h" + +const struct log2f_data __log2f_data = { + .tab = { + { 0x1.661ec79f8f3bep+0, -0x1.efec65b963019p-2 }, + { 0x1.571ed4aaf883dp+0, -0x1.b0b6832d4fca4p-2 }, + { 0x1.49539f0f010bp+0, -0x1.7418b0a1fb77bp-2 }, + { 0x1.3c995b0b80385p+0, -0x1.39de91a6dcf7bp-2 }, + { 0x1.30d190c8864a5p+0, -0x1.01d9bf3f2b631p-2 }, + { 0x1.25e227b0b8eap+0, -0x1.97c1d1b3b7afp-3 }, + { 0x1.1bb4a4a1a343fp+0, -0x1.2f9e393af3c9fp-3 }, + { 0x1.12358f08ae5bap+0, -0x1.960cbbf788d5cp-4 }, + { 0x1.0953f419900a7p+0, -0x1.a6f9db6475fcep-5 }, + { 0x1p+0, 0x0p+0 }, + { 0x1.e608cfd9a47acp-1, 0x1.338ca9f24f53dp-4 }, + { 0x1.ca4b31f026aap-1, 0x1.476a9543891bap-3 }, + { 0x1.b2036576afce6p-1, 0x1.e840b4ac4e4d2p-3 }, + { 0x1.9c2d163a1aa2dp-1, 0x1.40645f0c6651cp-2 }, + { 0x1.886e6037841edp-1, 0x1.88e9c2c1b9ff8p-2 }, + { 0x1.767dcf5534862p-1, 0x1.ce0a44eb17bccp-2 }, + }, + .poly = { + -0x1.712b6f70a7e4dp-2, 0x1.ecabf496832ep-2, -0x1.715479ffae3dep-1, + 0x1.715475f35c8b8p0, + } +}; diff --git a/libs/libglibc-compatibility/musl/log2f_data.h b/libs/libglibc-compatibility/musl/log2f_data.h new file mode 100644 index 00000000000..91d781c10fe --- /dev/null +++ b/libs/libglibc-compatibility/musl/log2f_data.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2017-2018, Arm Limited. + * SPDX-License-Identifier: MIT + */ +#ifndef _LOG2F_DATA_H +#define _LOG2F_DATA_H + +#include "musl_features.h" + +#define LOG2F_TABLE_BITS 4 +#define LOG2F_POLY_ORDER 4 +extern hidden const struct log2f_data { + struct { + double invc, logc; + } tab[1 << LOG2F_TABLE_BITS]; + double poly[LOG2F_POLY_ORDER]; +} __log2f_data; + +#endif diff --git a/libs/libglibc-compatibility/musl/sched_getcpu.c b/libs/libglibc-compatibility/musl/sched_getcpu.c index d0e171dac95..4ec5eaf6796 100644 --- a/libs/libglibc-compatibility/musl/sched_getcpu.c +++ b/libs/libglibc-compatibility/musl/sched_getcpu.c @@ -1,14 +1,11 @@ #define _GNU_SOURCE #include #include -#include #include "syscall.h" #include "atomic.h" #ifdef VDSO_GETCPU_SYM -void *__vdsosym(const char *, const char *); - static void *volatile vdso_func; typedef long (*getcpu_f)(unsigned *, unsigned *, void *); diff --git a/libs/libglibc-compatibility/musl/syscall.h b/libs/libglibc-compatibility/musl/syscall.h index 49edd7244d3..70b4688f642 100644 --- a/libs/libglibc-compatibility/musl/syscall.h +++ b/libs/libglibc-compatibility/musl/syscall.h @@ -1,5 +1,8 @@ #pragma once +#include +#include + typedef long syscall_arg_t; __attribute__((visibility("hidden"))) @@ -7,3 +10,6 @@ long __syscall_ret(unsigned long); __attribute__((visibility("hidden"))) long __syscall(syscall_arg_t, ...); + +__attribute__((visibility("hidden"))) +void *__vdsosym(const char *, const char *); diff --git a/libs/libglibc-compatibility/musl/vdso.c b/libs/libglibc-compatibility/musl/vdso.c new file mode 100644 index 00000000000..c0dd0f33e4e --- /dev/null +++ b/libs/libglibc-compatibility/musl/vdso.c @@ -0,0 +1,105 @@ +#include +#include +#include +#include +#include +#include +#include "syscall.h" + +#ifdef VDSO_USEFUL + +#if ULONG_MAX == 0xffffffff +typedef Elf32_Ehdr Ehdr; +typedef Elf32_Phdr Phdr; +typedef Elf32_Sym Sym; +typedef Elf32_Verdef Verdef; +typedef Elf32_Verdaux Verdaux; +#else +typedef Elf64_Ehdr Ehdr; +typedef Elf64_Phdr Phdr; +typedef Elf64_Sym Sym; +typedef Elf64_Verdef Verdef; +typedef Elf64_Verdaux Verdaux; +#endif + +static int checkver(Verdef *def, int vsym, const char *vername, char *strings) +{ + vsym &= 0x7fff; + for (;;) { + if (!(def->vd_flags & VER_FLG_BASE) + && (def->vd_ndx & 0x7fff) == vsym) + break; + if (def->vd_next == 0) + return 0; + def = (Verdef *)((char *)def + def->vd_next); + } + Verdaux *aux = (Verdaux *)((char *)def + def->vd_aux); + return !strcmp(vername, strings + aux->vda_name); +} + +#define OK_TYPES (1<e_phoff); + size_t *dynv=0, base=-1; + for (i=0; ie_phnum; i++, ph=(void *)((char *)ph+eh->e_phentsize)) { + if (ph->p_type == PT_LOAD) + base = (size_t)eh + ph->p_offset - ph->p_vaddr; + else if (ph->p_type == PT_DYNAMIC) + dynv = (void *)((char *)eh + ph->p_offset); + } + if (!dynv || base==(size_t)-1) return 0; + + char *strings = 0; + Sym *syms = 0; + Elf_Symndx *hashtab = 0; + uint16_t *versym = 0; + Verdef *verdef = 0; + + for (i=0; dynv[i]; i+=2) { + void *p = (void *)(base + dynv[i+1]); + switch(dynv[i]) { + case DT_STRTAB: strings = p; break; + case DT_SYMTAB: syms = p; break; + case DT_HASH: hashtab = p; break; + case DT_VERSYM: versym = p; break; + case DT_VERDEF: verdef = p; break; + } + } + + if (!strings || !syms || !hashtab) return 0; + if (!verdef) versym = 0; + + for (i=0; i>4) & OK_BINDS)) continue; + if (!syms[i].st_shndx) continue; + if (strcmp(name, strings+syms[i].st_name)) continue; + if (versym && !checkver(verdef, versym[i], vername, strings)) + continue; + return (void *)(base + syms[i].st_value); + } + + return 0; +} + +#endif diff --git a/libs/libglibc-compatibility/musl/x86_64/syscall_arch.h b/libs/libglibc-compatibility/musl/x86_64/syscall_arch.h new file mode 100644 index 00000000000..b20e812aa1a --- /dev/null +++ b/libs/libglibc-compatibility/musl/x86_64/syscall_arch.h @@ -0,0 +1,5 @@ +#define VDSO_USEFUL +#define VDSO_CGT_SYM "__vdso_clock_gettime" +#define VDSO_CGT_VER "LINUX_2.6" +#define VDSO_GETCPU_SYM "__vdso_getcpu" +#define VDSO_GETCPU_VER "LINUX_2.6" diff --git a/libs/libwidechar_width/widechar_width.h b/libs/libwidechar_width/widechar_width.h index 39cf0ded05b..3007a112886 100644 --- a/libs/libwidechar_width/widechar_width.h +++ b/libs/libwidechar_width/widechar_width.h @@ -500,7 +500,7 @@ bool widechar_in_table(const Collection &arr, int32_t c) { } /* Return the width of character c, or a special negative value. */ -int widechar_wcwidth(wchar_t c) { +inline int widechar_wcwidth(wchar_t c) { if (widechar_in_table(widechar_private_table, c)) return widechar_private_use; if (widechar_in_table(widechar_nonprint_table, c)) diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index b3df25d13e6..fcf56e82b52 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -21,7 +21,6 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS) add_subdirectory (corrector_utf8) add_subdirectory (zookeeper-cli) add_subdirectory (zookeeper-dump-tree) - add_subdirectory (zookeeper-copy-tree) add_subdirectory (zookeeper-remove-by-list) add_subdirectory (zookeeper-create-entry-to-download-part) add_subdirectory (zookeeper-adjust-block-numbers-to-parts) diff --git a/utils/build/build_no_submodules.sh b/utils/build/build_no_submodules.sh index 63349882128..fae10ab3270 100755 --- a/utils/build/build_no_submodules.sh +++ b/utils/build/build_no_submodules.sh @@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) cd ${CUR_DIR}/../.. BRANCH=`git rev-parse --abbrev-ref HEAD` BRANCH=${BRANCH:=master} -ROOT_DIR=${CUR_DIR}/../build_no_submodules +ROOT_DIR=${CUR_DIR}/../../build_no_submodules mkdir -p $ROOT_DIR cd $ROOT_DIR URL=`git remote get-url origin | sed 's/.git$//'` diff --git a/utils/grammar/ClickHouseLexer.g4 b/utils/grammar/ClickHouseLexer.g4 new file mode 100644 index 00000000000..766e30d2850 --- /dev/null +++ b/utils/grammar/ClickHouseLexer.g4 @@ -0,0 +1,232 @@ +lexer grammar ClickHouseLexer; + +LINE_COMMENT + : '--' ~[\r\n]* -> channel(HIDDEN) + ; + + // TOKENS, KEYWORDS + +K_ADD : A D D; +K_AFTER : A F T E R; +K_ALL : A L L; +K_ALIAS : A L I A S; +K_ALTER : A L T E R; +K_AND : A N D; +K_ANY : A N Y; +K_ARRAY : A R R A Y; +K_AS : A S; +K_ASCENDING : A S C E N D I N G; +K_ASC : A S C; +K_ASYNC : A S Y N C; +K_ATTACH : A T T A C H; +K_BETWEEN : B E T W E E N; +K_BY : B Y; +K_CASE : C A S E; +K_CAST : C A S T; +K_CHECK : C H E C K; +K_CLUSTER : C L U S T E R; +K_COLUMN : C O L U M N; +K_COLLATE : C O L L A T E; +K_CREATE : C R E A T E; +K_CROSS : C R O S S; +K_DAY : D A Y; +K_DESCRIBE : D E S C R I B E; +K_DESCENDING : D E S C E N D I N G; +K_DESC : D E S C; +K_DATABASE : D A T A B A S E; +K_DATABASES : D A T A B A S E S; +K_DEFAULT : D E F A U L T; +K_DETACH : D E T A C H; +K_DISTINCT : D I S T I N C T; +K_DROP : D R O P; +K_ELSE : E L S E; +K_END : E N D; +K_ENGINE : E N G I N E; +K_EXISTS : E X I S T S; +K_FETCH : F E T C H; +K_FINAL : F I N A L; +K_FIRST : F I R S T; +K_FROM : F R O M; +K_FREEZE : F R E E Z E; +K_FORMAT : F O R M A T; +K_FULL : F U L L; +K_GLOBAL : G L O B A L; +K_GROUP : G R O U P; +K_HAVING : H A V I N G; +K_HOUR : H O U R; +K_ID : I D; +K_IF : I F; +K_INNER : I N N E R; +K_INSERT : I N S E R T; +K_INTERVAL : I N T E R V A L; +K_INTO : I N T O; +K_IN : I N; +K_IS : I S; +K_JOIN : J O I N; +K_KILL: K I L L; +K_LAST : L A S T; +K_LEFT : L E F T; +K_LIKE : L I K E; +K_LIMIT : L I M I T; +K_MAIN : M A I N; // not a clickhouse reverved word +K_MATERIALIZED : M A T E R I A L I Z E D; +K_MINUTE : M I N U T E; +K_MODIFY : M O D I F Y; +K_MONTH : M O N T H; +K_NOT : N O T; +K_NULL : N U L L; +K_NULLS : N U L L S; +K_OFFSET : O F F S E T; +K_ON : O N; +K_OPTIMIZE : O P T I M I Z E; +K_ORDER : O R D E R; +K_OR : O R; +K_OUTFILE : O U T F I L E; +K_PARTITION : P A R T I T I O N; +K_POPULATE : P O P U L A T E; +K_PREWHERE : P R E W H E R E; +K_PROCESSLIST : P R O C E S S L I S T; +K_QUERY : Q U E R Y; +K_RENAME : R E N A M E; +K_RETURN : R E T U R N; // not a clickhouse reverved word +K_RIGHT : R I G H T; +K_SAMPLE : S A M P L E; +K_SECOND : S E C O N D; +K_SELECT : S E L E C T; +K_SET : S E T; +K_SETTINGS : S E T T I N G S; +K_SHOW : S H O W; +K_SYNC : S Y N C; +K_TABLE : T A B L E; +K_TABLES : T A B L E S; +K_TEMPORARY : T E M P O R A R Y; +K_TEST : T E S T; +K_THEN : T H E N; +K_TOTALS : T O T A L S; +K_TO : T O; +K_OUTER: O U T E R; +K_VALUES : V A L U E S; +K_VIEW : V I E W; +K_UNION : U N I O N; +K_USE : U S E; +K_USING : U S I N G; +K_WEEK : W E E K; +K_WHEN : W H E N; +K_WHERE : W H E R E; +K_WITH : W I T H; +K_YEAR : Y E A R; + +COLON : ':' ; +COMMA : ',' ; +SEMI : ';' ; +LPAREN : '(' ; +RPAREN : ')' ; +RARROW : '->' ; +LT : '<' ; +GT : '>' ; +QUESTION : '?' ; +STAR : '*' ; +PLUS : '+' ; +CONCAT : '||' ; +OR : '|' ; +DOLLAR : '$' ; +DOT : '.' ; +PERCENT : '%' ; +MINUS : '-' ; +DIVIDE : '/' ; +EQUALS : '==' ; +ASSIGN : '=' ; +NOT_EQUALS : '!=' ; +NOT_EQUALS2 : '<>' ; +LE : '<=' ; +GE : '>=' ; +LBRAKET : '[' ; +RBRAKET : ']' ; +LCURLY : '{' ; +RCURLY : '}' ; + + +T_ARRAY : 'Array' ; +T_TUPLE : 'Tuple' ; +T_NULLABLE : 'Nullable' ; +T_FLOAT32 : 'Float32' ; +T_FLOAT64 : 'Float64' ; +T_UINT8 : 'UInt8' ; +T_UINT16 : 'UInt16' ; +T_UINT32 : 'UInt32' ; +T_UINT64 : 'UInt64' ; +T_INT8 : 'Int8' ; +T_INT16 : 'Int16' ; +T_INT32 : 'Int32' ; +T_INT64 : 'Int64' ; +T_ENUM8 : 'Enum8' ; +T_ENUM16 : 'Enum16' ; +T_UUID : 'UUID' ; +T_DATE : 'Date' ; +T_DATETIME : 'DateTime' ; +T_STRING : 'String' ; +T_FIXEDSTRING : 'FixedString' ; +T_NULL : 'Null' ; +T_INTERVAL_YEAR : 'IntervalYear' ; +T_INTERVAL_MONTH : 'IntervalMonth' ; +T_INTERVAL_WEEK : 'IntervalWeek' ; +T_INTERVAL_DAY : 'IntervalDay' ; +T_INTERVAL_HOUR : 'IntervalHour' ; +T_INTERVAL_MINUTE : 'IntervalMinute' ; +T_INTERVAL_SECOND : 'IntervalSecond' ; +T_AGGREGATE_FUNCTION : 'AggregateFunction' ; +// lambda type has unknown name. + +IDENTIFIER + : [a-zA-Z_] [a-zA-Z_0-9]* + ; + +NUMERIC_LITERAL + : DIGIT+ ( '.' DIGIT* )? ( E [-+]? DIGIT+ )? + | '.' DIGIT+ ( E [-+]? DIGIT+ )? + ; + +STRING_LITERAL + : '\'' ( ~'\'' | '\\\'' )* '\'' + ; + +QUOTED_LITERAL + : '`' ( ~'`' )* '`' + ; + +SPACES + : [ \u000B\t\r\n] -> channel(HIDDEN) + ; + +UNEXPECTED_CHAR + : . + ; + +fragment DIGIT : [0-9]; + +fragment A : [aA]; +fragment B : [bB]; +fragment C : [cC]; +fragment D : [dD]; +fragment E : [eE]; +fragment F : [fF]; +fragment G : [gG]; +fragment H : [hH]; +fragment I : [iI]; +fragment J : [jJ]; +fragment K : [kK]; +fragment L : [lL]; +fragment M : [mM]; +fragment N : [nN]; +fragment O : [oO]; +fragment P : [pP]; +fragment Q : [qQ]; +fragment R : [rR]; +fragment S : [sS]; +fragment T : [tT]; +fragment U : [uU]; +fragment V : [vV]; +fragment W : [wW]; +fragment X : [xX]; +fragment Y : [yY]; +fragment Z : [zZ]; diff --git a/utils/grammar/ClickHouseParser.g4 b/utils/grammar/ClickHouseParser.g4 new file mode 100644 index 00000000000..5cb4676fcb8 --- /dev/null +++ b/utils/grammar/ClickHouseParser.g4 @@ -0,0 +1,584 @@ +parser grammar ClickHouseParser; + +options { + tokenVocab=ClickHouseLexer; +} + +// эта грамматика написана по сорсам парсеров, имена правил примерно соответствуют парсерам в cpp. +// известные расхождения +// 1. скобки не обязательно сразу идут после имени функции. +// 2. многословные токены поделены на самостоятельные слова +// 3. для INSERT запроса не написана часть парсинга значений. +// 4. правило для expr переписано чтобы понизить глубину AST и сразу выходить на уровень expr - al + +parse + : ( query | err ) EOF + ; + +query + : show_tables_query + | select_query + | insert_query + | create_query + | rename_query + | drop_query + | alter_query + | use_query + | set_query + | optimize_query + | table_properties_query + | show_processlist_query + | check_query + | kill_query_query + ; + +// 1. QUERIES + +select_query + : select_query_main ( K_UNION K_ALL select_query_main ) * + query_outfile_step? + select_format_step? + ; + +select_query_main + : select_with_step? + select_select_step select_from_step? + K_FINAL? select_sample_step? + select_array_join_step? select_join_step? + select_prewhere_step? select_where_step? + select_groupby_step? select_having_step? + select_orderby_step? + select_limitby_step? select_limit_step? + select_settings_step? + ; + +select_with_step + : K_WITH select_expr_list + ; + +select_select_step + : K_SELECT K_DISTINCT? select_expr_list + ; + +select_from_step + : K_FROM ( full_table_name + | table_function + | subquery + ) select_alias? + ; + +select_array_join_step + : K_LEFT? K_ARRAY K_JOIN not_empty_expression_list + ; + +select_sample_step + : K_SAMPLE sample_ratio (K_OFFSET sample_ratio ) ? + ; + +sample_ratio + : NUMERIC_LITERAL ( DIVIDE NUMERIC_LITERAL ) ? + ; + +select_join_step + : K_GLOBAL? + ( K_ANY | K_ALL ) ( K_INNER | K_LEFT K_OUTER? | K_RIGHT K_OUTER? | K_FULL K_OUTER? ) K_JOIN select_join_right_part + ( K_USING LPAREN not_empty_expression_list RPAREN + | K_USING not_empty_expression_list + // | K_ON expr на самом деле нет. + ) + | K_GLOBAL? K_CROSS K_JOIN select_join_right_part + ; + +select_join_right_part + : identifier + | subquery + ; + +select_prewhere_step + : K_PREWHERE expression_with_optional_alias + ; + +select_where_step + : K_WHERE expression_with_optional_alias + ; + +select_groupby_step + : K_GROUP K_BY not_empty_expression_list ( K_WITH K_TOTALS ) ? + ; + +select_having_step + : K_HAVING expression_with_optional_alias + ; + +select_orderby_step + : K_ORDER K_BY order_by_expression_list + ; + +select_limit_step + : K_LIMIT NUMERIC_LITERAL ( COMMA NUMERIC_LITERAL )? + ; + +select_limitby_step + : K_LIMIT NUMERIC_LITERAL K_BY not_empty_expression_list + ; + +select_settings_step + : K_SETTINGS assignment_list + ; + +select_format_step + : K_FORMAT identifier + ; + +insert_query + : K_INSERT K_INTO full_table_name + ( K_ID ASSIGN STRING_LITERAL )? // wtf? + ( LPAREN column_name_list RPAREN )? + ( K_VALUES LPAREN literal (COMMA literal )* RPAREN(COMMA LPAREN literal (COMMA literal )* RPAREN)* // ch тут дальше не парсит. а я написал скобки + | K_FORMAT format_name // ch тут дальше не парсит, только доедает все пробелы или один перевод строки. pushMode() + | select_query ) + ; + +create_query + : ( K_CREATE | K_ATTACH ) K_TEMPORARY? + ( K_DATABASE ( K_IF K_NOT K_EXISTS ) ? database_name + | K_TABLE ( K_IF K_NOT K_EXISTS ) ? full_table_name ( K_ON K_CLUSTER cluster_name ) ? + ( LPAREN column_declaration_list RPAREN engine ( K_AS select_query ) ? // если VIEW - то есть и колонки и select. + | engine K_AS ( select_query + | full_table_name engine? // wtf + ) + ) + | K_MATERIALIZED? K_VIEW ( K_IF K_NOT K_EXISTS ) ? full_table_name + ( LPAREN column_declaration_list RPAREN ) ? engine? K_POPULATE? K_AS select_query + ) + ; + +rename_query + : K_RENAME K_TABLE full_table_name K_TO full_table_name ( COMMA full_table_name K_TO full_table_name )* ( K_ON K_CLUSTER cluster_name ) ? + ; + +drop_query + : ( K_DROP | K_DETACH ) + ( K_DATABASE ( K_IF K_EXISTS ) ? database_name ( K_ON K_CLUSTER cluster_name ) ? + | K_TABLE ( K_IF K_EXISTS ) ? full_table_name ( K_ON K_CLUSTER cluster_name ) ? + ) + ; + +alter_query + : K_ALTER K_TABLE full_table_name ( K_ON K_CLUSTER cluster_name ) ? + alter_query_element ( COMMA alter_query_element ) * + ; + +alter_query_element + : K_ADD K_COLUMN compound_name_type_pair ( K_AFTER column_name ) ? + | K_DROP K_COLUMN column_name + | K_MODIFY K_COLUMN compound_name_type_pair + | K_ATTACH K_PARTITION partition_name + | K_DETACH K_PARTITION partition_name + | K_DROP K_PARTITION partition_name + | K_FETCH K_PARTITION partition_name K_FROM STRING_LITERAL + | K_FREEZE K_PARTITION partition_name + ; + +clickhouse_type + : simple_type + | T_AGGREGATE_FUNCTION LPAREN function_name ( COMMA clickhouse_type ) * RPAREN + | T_ARRAY LPAREN clickhouse_type RPAREN + | T_TUPLE LPAREN clickhouse_type ( COMMA clickhouse_type ) * RPAREN + | T_NULLABLE LPAREN clickhouse_type LPAREN + ; + +simple_type + : T_UINT8 + | T_UINT16 + | T_UINT32 + | T_UINT64 + | T_INT8 + | T_INT16 + | T_INT32 + | T_INT64 + | T_FLOAT32 + | T_FLOAT64 + | T_ENUM8 LPAREN enum_entry ( COMMA enum_entry ) * LPAREN + | T_ENUM16 LPAREN enum_entry ( COMMA enum_entry ) * LPAREN + | T_UUID + | T_DATE + | T_DATETIME + | T_STRING + | T_INTERVAL_YEAR + | T_INTERVAL_MONTH + | T_INTERVAL_WEEK + | T_INTERVAL_DAY + | T_INTERVAL_HOUR + | T_INTERVAL_MINUTE + | T_INTERVAL_SECOND + | T_NULL + | T_FIXEDSTRING LPAREN NUMERIC_LITERAL LPAREN + ; + +enum_entry + : STRING_LITERAL ASSIGN NUMERIC_LITERAL + ; + +use_query + : K_USE database_name + ; + +set_query + : K_SET K_GLOBAL? assignment_list + ; + +assignment_list + : assignment ( COMMA assignment ) * + ; + +assignment + : identifier ASSIGN literal + ; + +kill_query_query + : K_KILL K_QUERY K_WHERE expression_with_optional_alias ( K_SYNC | K_ASYNC | K_TEST ) + ; + +optimize_query + : K_OPTIMIZE K_TABLE full_table_name ( K_PARTITION STRING_LITERAL ) ? K_FINAL? + ; + +table_properties_query + : ( K_EXISTS | ( K_DESCRIBE | K_DESC ) | K_SHOW K_CREATE ) K_TABLE full_table_name query_outfile_step? ( K_FORMAT format_name ) ? + ; + +show_tables_query + : K_SHOW ( K_DATABASES + | K_TABLES ( K_FROM database_name ) ? ( K_NOT? K_LIKE STRING_LITERAL ) ? ) + query_outfile_step? + ( K_FORMAT format_name ) ? + ; + +show_processlist_query + : K_SHOW K_PROCESSLIST query_outfile_step? ( K_FORMAT format_name ) ? + ; + +check_query + : K_CHECK K_TABLE full_table_name + ; + +// 2. QUERY ELEMENTS + +full_table_name + : ( database_name DOT ) ? table_name + ; + +partition_name + : identifier | STRING_LITERAL + ; + +cluster_name + : identifier | STRING_LITERAL + ; + +database_name + : identifier + ; + +table_name + : identifier + ; + +format_name + : identifier + ; + +query_outfile_step + : K_INTO K_OUTFILE STRING_LITERAL + ; + +engine + : K_ENGINE ASSIGN identifier_with_optional_parameters + ; + +identifier_with_optional_parameters + : identifier_with_parameters + | identifier + ; + +identifier_with_parameters + : function + | nested_table + ; + +order_by_expression_list + : order_by_element ( COMMA order_by_element ) * + ; + +order_by_element + : expression_with_optional_alias ( K_DESC | K_DESCENDING | K_ASC | K_ASCENDING ) ? ( K_NULLS ( K_FIRST | K_LAST ) ) ? ( K_COLLATE STRING_LITERAL ) ? + ; + +nested_table + : identifier LPAREN name_type_pair_list RPAREN + ; + +name_type_pair_list + : name_type_pair ( COMMA name_type_pair ) * + ; + +name_type_pair + : identifier column_type + ; + +compound_name_type_pair + : compound_identifier column_type + ; + +column_declaration_list + : column_declaration ( COMMA column_declaration ) * + ; + +column_declaration + : column_name + ( ( K_DEFAULT | K_MATERIALIZED | K_ALIAS ) expr + | column_type + ) + ; + +column_name + : identifier + ; + +column_type + : clickhouse_type + ; + +column_name_list + : column_name ( COMMA column_name ) * + ; + +select_expr_list + : select_expr ( COMMA select_expr) * + ; + +select_expr + : expr select_alias? + ; + +select_alias + : K_AS? alias_name + ; + +alias + : K_AS alias_name + ; + +alias_name + : identifier + ; + +table_function + : function + ; + + +subquery + : LPAREN select_query_main RPAREN + ; + +expression_with_optional_alias + : expr alias? + ; + +// EXPRESSIONS + +expr + : LPAREN expr RPAREN # ExprParen + | function # ExprFunction + | K_CASE expr? ( K_WHEN expr K_THEN expr ) ( K_WHEN expr K_THEN expr ) * K_ELSE expr K_END # ExprCase + | expr DOT expr # ExprTupleElement + | expr LBRAKET expr RBRAKET # ExprArrayElement + | MINUS expr # ExprUnaryMinus + | K_CAST LPAREN expr K_AS clickhouse_type RPAREN # ExprCast + | expr ( STAR | DIVIDE | PERCENT ) expr # ExprMul + | expr ( PLUS | MINUS ) expr # ExprAdd + | expr CONCAT expr # ExprConcat + | expr K_BETWEEN expr K_AND expr # ExprBetween + | expr ( EQUALS | ASSIGN | NOT_EQUALS | NOT_EQUALS2 | LE | GE | LT | GT | K_LIKE | K_NOT K_LIKE ) expr # ExprLogical + | expr ( K_IN | K_NOT K_IN | K_GLOBAL K_IN | K_GLOBAL K_NOT K_IN ) expr # ExprIn + | expr ( K_IS K_NULL | K_IS K_NOT K_NULL ) # ExprIsNull + | K_INTERVAL expr interval_unit # ExprInterval + | K_NOT expr # ExprNot + | expr K_AND expr # ExprAnd + | expr K_OR expr # ExprOr + | expr QUESTION expr COLON expr # ExprTernary + | ( LPAREN identifier ( COMMA identifier )* RPAREN | identifier ( COMMA identifier )* ) RARROW expr # ExprLambda + | subquery # ExprSubquery + | LPAREN not_empty_expression_list RPAREN # ExprList + | array # ExprArray + | literal # ExprLiteral + | compound_identifier # ExprId + | STAR # ExprStar + | expr alias # ExprWithAlias + ; + +interval_unit + : K_YEAR + | K_MONTH + | K_WEEK + | K_DAY + | K_HOUR + | K_MINUTE + | K_SECOND + ; +expression_list + : ( not_empty_expression_list )? + ; + +not_empty_expression_list + : expr ( COMMA expr )* + ; + +array + : LBRAKET expression_list RBRAKET + ; + +function + : function_name function_parameters? function_arguments + ; + +function_parameters + : LPAREN ( expr ( COMMA expr )* )? RPAREN + ; +function_arguments + : LPAREN ( expr ( COMMA expr )* )? RPAREN + ; + +function_name + : identifier + ; + +identifier + : QUOTED_LITERAL + | IDENTIFIER + // в данном случае мы разрешаем ключевым словам выступать в качестве имен колонок или функций. + | simple_type + | keyword + ; + +keyword + : K_ADD + | K_AFTER + | K_ALL + | K_ALIAS + | K_ALTER + | K_AND + | K_ANY + | K_ARRAY + | K_AS + | K_ASCENDING + | K_ASC + | K_ASYNC + | K_ATTACH + | K_BETWEEN + | K_BY + | K_CASE + | K_CHECK + | K_COLUMN + | K_COLLATE + | K_CREATE + | K_CROSS + | K_DESCRIBE + | K_DESCENDING + | K_DESC + | K_DATABASE + | K_DATABASES + | K_DEFAULT + | K_DETACH + | K_DISTINCT + | K_DROP + | K_ENGINE + | K_ELSE + | K_END + | K_EXISTS + | K_FINAL + | K_FIRST + | K_FROM + | K_FORMAT + | K_FULL + | K_GLOBAL + | K_GROUP + | K_HAVING + | K_ID + | K_IF + | K_INNER + | K_INSERT + | K_INTO + | K_IN + | K_IS + | K_JOIN + | K_KILL + | K_LAST + | K_LEFT + | K_LIKE + | K_LIMIT + | K_MAIN + | K_MATERIALIZED + | K_MODIFY + | K_NOT + | K_NULL + | K_NULLS + | K_OFFSET + | K_ON + | K_OPTIMIZE + | K_ORDER + | K_OR + | K_OUTFILE + | K_PARTITION + | K_POPULATE + | K_PREWHERE + | K_PROCESSLIST + | K_QUERY + | K_RENAME + | K_RETURN + | K_RIGHT + | K_SAMPLE + | K_SELECT + | K_SET + | K_SETTINGS + | K_SHOW + | K_SYNC + | K_TABLE + | K_TABLES + | K_TEMPORARY + | K_TEST + | K_THEN + | K_TOTALS + | K_TO + | K_OUTER + | K_VALUES + | K_VIEW + | K_UNION + | K_USE + | K_USING + | K_WHEN + | K_WHERE + | K_WITH + ; + +compound_identifier +: identifier DOT identifier +| identifier +; + + +literal + : K_NULL + | NUMERIC_LITERAL + | STRING_LITERAL + ; + +err + : UNEXPECTED_CHAR + { + throw new RuntimeException("UNEXPECTED_CHAR=" + $UNEXPECTED_CHAR.text); + } + ; + diff --git a/utils/grammar/README.md b/utils/grammar/README.md new file mode 100644 index 00000000000..03a611be69c --- /dev/null +++ b/utils/grammar/README.md @@ -0,0 +1,8 @@ +ClickHouse grammar for ANTLR4 +============================= + +Authors: Yuriy Galitskiy (orantius, https://github.com/duremar), Sergey Serebryanik (serebrserg, https://github.com/serebrserg), Efim Pyshnograev (graev). + +Initially developed for Yandex.Metrica product and published under Apache 2.0 license with permission from Yandex. It has also found its usage in DataGrip product. + +It is not used in ClickHouse directly and is not synchronized with ClickHouse C++ code. Neither supported or tested. Any help welcome. diff --git a/utils/kafka/consume.py b/utils/kafka/consume.py index 34a1997d3e6..c82901f9e0e 100755 --- a/utils/kafka/consume.py +++ b/utils/kafka/consume.py @@ -34,6 +34,7 @@ def main(): pprint(client.poll(10000)) client.unsubscribe() client.close() + return 0 if __name__ == "__main__": diff --git a/utils/kafka/manage.py b/utils/kafka/manage.py index 13bc2fa0388..01847c7675b 100755 --- a/utils/kafka/manage.py +++ b/utils/kafka/manage.py @@ -35,6 +35,7 @@ def main(): print(client.delete_topics(args.delete)) client.close() + return 0 if __name__ == "__main__": diff --git a/utils/kafka/produce.py b/utils/kafka/produce.py index 218471e4840..97e2e6b7705 100755 --- a/utils/kafka/produce.py +++ b/utils/kafka/produce.py @@ -77,6 +77,7 @@ def main(): client.flush() client.close() + return 0 if __name__ == "__main__": diff --git a/utils/kafka/status.py b/utils/kafka/status.py index 8331a056dff..28ba3c9c36f 100755 --- a/utils/kafka/status.py +++ b/utils/kafka/status.py @@ -5,7 +5,6 @@ import kafka # … kafka-python import argparse -from pprint import pprint def main(): @@ -46,6 +45,7 @@ def main(): consumer.close() client.close() + return 0 if __name__ == "__main__": diff --git a/utils/make_changelog.py b/utils/make_changelog.py index 40070d62693..a47706767e3 100755 --- a/utils/make_changelog.py +++ b/utils/make_changelog.py @@ -35,7 +35,7 @@ def http_get_json(url, token, max_retries, retry_timeout): logging.warning(msg) time.sleep(retry_timeout) continue - except: + except Exception: pass raise Exception(msg) @@ -60,10 +60,22 @@ def get_merge_base(first, second, project_root): sha = tuple(filter(len, text.split()))[0] check_sha(sha) return sha - except: + except Exception: logging.error('Cannot find merge base for %s and %s', first, second) raise +def rev_parse(rev, project_root): + try: + command = "git rev-parse {}".format(rev) + text = subprocess.check_output(command, shell=True, cwd=project_root) + text = text.decode('utf-8', 'ignore') + sha = tuple(filter(len, text.split()))[0] + check_sha(sha) + return sha + except Exception: + logging.error('Cannot find revision %s', rev) + raise + # Get list of commits from branch to base_sha. Update commits_info. def get_commits_from_branch(repo, branch, base_sha, commits_info, max_pages, token, max_retries, retry_timeout): @@ -198,7 +210,7 @@ def process_unknown_commits(commits, commits_info, users): # First, try get name from github user try: name = users[login]['name'] - except: + except KeyError: pass else: login = 'Unknown' @@ -207,7 +219,7 @@ def process_unknown_commits(commits, commits_info, users): if not name: try: name = info['commit']['author']['name'] - except: + except KeyError: pass author = '[{}]({})'.format(name or login, info['author']['html_url']) @@ -217,49 +229,61 @@ def process_unknown_commits(commits, commits_info, users): text = 'Commits which are not from any pull request:\n\n' return text + '\n\n'.join(texts) +# This function mirrors the PR description checks in ClickhousePullRequestTrigger. +# Returns False if the PR should not be mentioned changelog. +def parse_one_pull_request(item): + description = item['description'] + lines = [line for line in map(lambda x: x.strip(), description.split('\n')) if line] + lines = [re.sub(r'\s+', ' ', l) for l in lines] + + cat_pos = None + short_descr_pos = None + long_descr_pos = None + + if lines: + for i in range(len(lines) - 1): + if re.match(r'(?i).*category.*:$', lines[i]): + cat_pos = i + if re.match(r'(?i)^\**\s*(Short description|Change\s*log entry)', lines[i]): + short_descr_pos = i + if re.match(r'(?i)^\**\s*Detailed description', lines[i]): + long_descr_pos = i + + if cat_pos is None: + return False + cat = lines[cat_pos + 1] + cat = re.sub(r'^[-*\s]*', '', cat) + + # Filter out the PR categories that are not for changelog. + if re.match(r'(?i)doc|((non|in|not|un)[-\s]*significant)', cat): + return False + + short_descr = '' + if short_descr_pos: + short_descr_end = long_descr_pos or len(lines) + short_descr = lines[short_descr_pos + 1] + if short_descr_pos + 2 != short_descr_end: + short_descr += ' ...' + + # If we have nothing meaningful + if not re.match('\w', short_descr): + short_descr = item['title'] + + # TODO: Add detailed description somewhere + + item['entry'] = short_descr + item['category'] = cat + + return True + # List of pull requests -> text description. def process_pull_requests(pull_requests, users, repo): groups = {} for id, item in pull_requests.items(): - lines = list(filter(len, map(lambda x: x.strip(), item['description'].split('\n')))) - - cat_pos = None - short_descr_pos = None - long_descr_pos = None - - if lines: - for i in range(len(lines) - 1): - if re.match('^\**Category', lines[i]): - cat_pos = i - if re.match('^\**\s*Short description', lines[i]): - short_descr_pos = i - if re.match('^\**\s*Detailed description', lines[i]): - long_descr_pos = i - - cat = '' - if cat_pos is not None: - # TODO: Sometimes have more than one - cat = lines[cat_pos + 1] - cat = cat.strip().lstrip('-').strip() - - # We are not interested in documentation PRs in changelog. - if re.match('^\**\s*(?:Documentation|Doc\s)', cat): - continue; - - short_descr = '' - if short_descr_pos: - short_descr_end = long_descr_pos or len(lines) - short_descr = lines[short_descr_pos + 1] - if short_descr_pos + 2 != short_descr_end: - short_descr += ' ...' - - # If we have nothing meaningful - if not re.match('\w', short_descr): - short_descr = item['title'] - - # TODO: Add detailed description somewhere + if not parse_one_pull_request(item): + continue pattern = u"{} [#{}]({}) ({})" link = 'https://github.com/{}/pull/{}'.format(repo, id) @@ -269,20 +293,21 @@ def process_pull_requests(pull_requests, users, repo): user = users[item['user']] author = u'[{}]({})'.format(user['name'] or user['login'], user['html_url']) + cat = item['category'] if cat not in groups: groups[cat] = [] - groups[cat].append(pattern.format(short_descr, id, link, author)) + groups[cat].append(pattern.format(item['entry'], id, link, author)) - categories_preferred_order = ['New Feature', 'Bug Fix', 'Improvement', 'Performance Improvement', 'Build/Testing/Packaging Improvement', 'Backward Incompatible Change', 'Other'] + categories_preferred_order = ['Backward Incompatible Change', 'New Feature', 'Bug Fix', 'Improvement', 'Performance Improvement', 'Build/Testing/Packaging Improvement', 'Other'] def categories_sort_key(name): if name in categories_preferred_order: - return categories_preferred_order.index(name) + return str(categories_preferred_order.index(name)).zfill(3) else: return name.lower() texts = [] - for group, text in sorted(groups.items(), key = lambda (k, v): categories_sort_key(k)): + for group, text in sorted(groups.items(), key = lambda kv: categories_sort_key(kv[0])): items = [u'* {}'.format(pr) for pr in text] texts.append(u'### {}\n{}'.format(group if group else u'[No category]', '\n'.join(items))) @@ -447,5 +472,7 @@ if __name__ == '__main__': logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') repo_folder = os.path.expanduser(repo_folder) + new_release_tag = rev_parse(new_release_tag, repo_folder) + prev_release_tag = rev_parse(prev_release_tag, repo_folder) make_changelog(new_release_tag, prev_release_tag, pull_requests, repo, repo_folder, state_file, token, max_retry, retry_timeout) diff --git a/utils/release/push_packages b/utils/release/push_packages index c2ab47b6e85..487e939c47e 100755 --- a/utils/release/push_packages +++ b/utils/release/push_packages @@ -158,14 +158,14 @@ def transfer_packages_dupload(ssh_key, path, repo_user, repo_url, incoming_direc def clear_old_incoming_packages(ssh_connection, user): for pkg in ('deb', 'rpm', 'tgz'): - for release_type in ('stable', 'testing', 'prestable'): + for release_type in ('stable', 'testing', 'prestable', 'lts'): try: if pkg != 'tgz': ssh_connection.execute("rm /home/{user}/incoming/clickhouse/{pkg}/{release_type}/*".format( user=user, pkg=pkg, release_type=release_type)) else: ssh_connection.execute("rm /home/{user}/incoming/clickhouse/{pkg}/*".format( - user=user, pkg=pkg, release_type=release_type)) + user=user, pkg=pkg)) except Exception: logging.info("rm is not required") @@ -201,7 +201,7 @@ if __name__ == "__main__": parser.add_argument('--deb-directory') parser.add_argument('--rpm-directory') parser.add_argument('--tgz-directory') - parser.add_argument('--release-type', choices=('testing', 'stable', 'prestable'), default='testing') + parser.add_argument('--release-type', choices=('testing', 'stable', 'prestable', 'lts'), default='testing') parser.add_argument('--ssh-key-path') parser.add_argument('--gpg-passphrase', required=True) parser.add_argument('--gpg-sec-key-path') diff --git a/utils/s3tools/s3uploader b/utils/s3tools/s3uploader index 25d4abbd375..cb1cd52228e 100755 --- a/utils/s3tools/s3uploader +++ b/utils/s3tools/s3uploader @@ -92,14 +92,14 @@ if __name__ == "__main__": parser.add_argument('--tmp-prefix', default='/tmp', help='Prefix to store temporay downloaded file') data_group = parser.add_mutually_exclusive_group(required=True) - data_group.add_argument('--table-name', + table_name_argument = data_group.add_argument('--table-name', help='Name of table with database, if you are uploading partitions') data_group.add_argument('--file-path', help='Name of file, if you are uploading') args = parser.parse_args() if args.table_name is not None and args.clickhouse_data_path is None: - raise argparse.ArgumentError( + raise argparse.ArgumentError(table_name_argument, "You should specify --clickhouse-data-path to upload --table") s3_conn = S3API( diff --git a/utils/test-data-generator/MarkovModel.h b/utils/test-data-generator/MarkovModel.h index 7ef69b2a1f0..338aee2e61f 100644 --- a/utils/test-data-generator/MarkovModel.h +++ b/utils/test-data-generator/MarkovModel.h @@ -105,7 +105,7 @@ public: if (table.end() == it) return pos - data; - *pos = it->getSecond().sample(random()); + *pos = it->getMapped().sample(random()); /// Zero byte marks end of string. if (0 == *pos) @@ -125,12 +125,12 @@ public: for (auto & elem : table) { UInt32 new_total = 0; - for (auto & frequency : elem.getSecond().data) + for (auto & frequency : elem.getMapped().data) { frequency.count = transform(frequency.count); new_total += frequency.count; } - elem.getSecond().total = new_total; + elem.getMapped().total = new_total; } } @@ -142,10 +142,10 @@ public: for (const auto & elem : table) { - writeBinary(elem.getFirst(), out); - writeBinary(UInt8(elem.getSecond().data.size()), out); + writeBinary(elem.getKey(), out); + writeBinary(UInt8(elem.getMapped().data.size()), out); - for (const auto & frequency : elem.getSecond().data) + for (const auto & frequency : elem.getMapped().data) { writeBinary(frequency.byte, out); writeVarUInt(frequency.count, out); diff --git a/utils/zookeeper-copy-tree/CMakeLists.txt b/utils/zookeeper-copy-tree/CMakeLists.txt deleted file mode 100644 index c4dc88d700c..00000000000 --- a/utils/zookeeper-copy-tree/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -add_executable (zookeeper-copy-tree main.cpp ${SRCS}) -target_link_libraries(zookeeper-copy-tree PRIVATE clickhouse_common_zookeeper clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) diff --git a/utils/zookeeper-copy-tree/main.cpp b/utils/zookeeper-copy-tree/main.cpp deleted file mode 100644 index 7bc7316b4af..00000000000 --- a/utils/zookeeper-copy-tree/main.cpp +++ /dev/null @@ -1,149 +0,0 @@ -#include -#include -#include - -#include - -#include - -namespace DB -{ -namespace ErrorCodes -{ - -extern const int UNEXPECTED_NODE_IN_ZOOKEEPER; - -} -} - -int main(int argc, char ** argv) -try -{ - boost::program_options::options_description desc("Allowed options"); - desc.add_options() - ("help,h", "produce help message") - ("from", boost::program_options::value()->required(), - "addresses of source ZooKeeper instances, comma separated. Example: example01e.yandex.ru:2181") - ("from-path", boost::program_options::value()->required(), - "where to copy from") - ("to", boost::program_options::value()->required(), - "addresses of destination ZooKeeper instances, comma separated. Example: example01e.yandex.ru:2181") - ("to-path", boost::program_options::value()->required(), - "where to copy to") - ; - - boost::program_options::variables_map options; - boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options); - - if (options.count("help")) - { - std::cout << "Copy a ZooKeeper tree to another cluster." << std::endl; - std::cout << "Usage: " << argv[0] << " [options]" << std::endl; - std::cout << "WARNING: it is almost useless as it is impossible to corretly copy sequential nodes" << std::endl; - std::cout << desc << std::endl; - return 1; - } - - zkutil::ZooKeeper from_zookeeper(options.at("from").as()); - zkutil::ZooKeeper to_zookeeper(options.at("to").as()); - - std::string from_path = options.at("from-path").as(); - std::string to_path = options.at("to-path").as(); - - if (to_zookeeper.exists(to_path)) - throw DB::Exception("Destination path: " + to_path + " already exists, aborting.", - DB::ErrorCodes::UNEXPECTED_NODE_IN_ZOOKEEPER); - - struct Node - { - Node( - std::string path_, - std::future get_future_, - std::future children_future_, - Node * parent_) - : path(std::move(path_)) - , get_future(std::move(get_future_)) - , children_future(std::move(children_future_)) - , parent(parent_) - { - } - - std::string path; - std::future get_future; - std::future children_future; - - Node * parent = nullptr; - std::future create_future; - bool created = false; - bool deleted = false; - bool ephemeral = false; - }; - - std::list nodes_queue; - nodes_queue.emplace_back( - from_path, from_zookeeper.asyncGet(from_path), from_zookeeper.asyncGetChildren(from_path), nullptr); - - to_zookeeper.createAncestors(to_path); - - for (auto it = nodes_queue.begin(); it != nodes_queue.end(); ++it) - { - Coordination::GetResponse get_response; - Coordination::ListResponse children_response; - try - { - get_response = it->get_future.get(); - children_response = it->children_future.get(); - } - catch (const Coordination::Exception & e) - { - if (e.code == Coordination::ZNONODE) - { - it->deleted = true; - continue; - } - throw; - } - - if (get_response.stat.ephemeralOwner) - { - it->ephemeral = true; - continue; - } - - if (it->parent && !it->parent->created) - { - it->parent->create_future.get(); - it->parent->created = true; - std::cerr << it->parent->path << " copied!" << std::endl; - } - - std::string new_path = it->path; - new_path.replace(0, from_path.length(), to_path); - it->create_future = to_zookeeper.asyncCreate(new_path, get_response.data, zkutil::CreateMode::Persistent); - get_response.data.clear(); - get_response.data.shrink_to_fit(); - - for (const auto & name : children_response.names) - { - std::string child_path = it->path == "/" ? it->path + name : it->path + '/' + name; - nodes_queue.emplace_back( - child_path, from_zookeeper.asyncGet(child_path), from_zookeeper.asyncGetChildren(child_path), - &(*it)); - } - } - - for (auto it = nodes_queue.begin(); it != nodes_queue.end(); ++it) - { - if (!it->created && !it->deleted && !it->ephemeral) - { - it->create_future.get(); - it->created = true; - std::cerr << it->path << " copied!" << std::endl; - } - } -} -catch (...) -{ - std::cerr << DB::getCurrentExceptionMessage(true) << '\n'; - throw; -} diff --git a/website/benchmark.html b/website/benchmark.html index be7eaca1528..433a9138237 100644 --- a/website/benchmark.html +++ b/website/benchmark.html @@ -2008,7 +2008,10 @@ function calculate_totals() { var k = current_runs[current_run_idx]; var current_ratios = ratios[j][k].filter( - function(x, i) { return x && $("#query_checkbox" + i).is(':checked'); }) + function(x, i) { + return x && $("#query_checkbox" + i).is(':checked'); + } + ); var ratio = Math.pow( current_ratios.reduce( diff --git a/website/index.html b/website/index.html index fcae470547e..afe8d2abcf5 100644 --- a/website/index.html +++ b/website/index.html @@ -131,24 +131,24 @@ - - - - - - - - - - - + + + + + + + + + + + - - - - - - + + + + + + diff --git a/website/nginx/default.conf b/website/nginx/default.conf index 98edad41055..fc029323fe2 100644 --- a/website/nginx/default.conf +++ b/website/nginx/default.conf @@ -14,6 +14,8 @@ server { } rewrite ^/docs/$ https://clickhouse.yandex/docs/en/ permanent; + rewrite ^/tutorial.html$ https://clickhouse.yandex/docs/en/getting_started/tutorial/ permanent; + rewrite ^/presentations/(.*)$ https://clickhouse.github.io/clickhouse-presentations/$1 permanent; rewrite ^/reference_en.html$ https://clickhouse.yandex/docs/en/single/ permanent; rewrite ^/reference_ru.html$ https://clickhouse.yandex/docs/ru/single/ permanent; rewrite ^/presentations/(.*)$ https://clickhouse.github.io/clickhouse-presentations/$1 permanent; diff --git a/website/robots.txt b/website/robots.txt index db843cdbf06..82708ceea95 100644 --- a/website/robots.txt +++ b/website/robots.txt @@ -2,16 +2,16 @@ User-agent: * Disallow: /docs/en/single/ Disallow: /docs/ru/single/ Disallow: /docs/zh/single/ +Disallow: /docs/ja/single/ Disallow: /docs/fa/single/ Disallow: /docs/v1* Disallow: /docs/v2* Disallow: /docs/v3* Disallow: /docs/en/search.html Disallow: /docs/ru/search.html -Disallow: /docs/fa/search.html +Disallow: /docs/ja/search.html Disallow: /docs/zh/search.html -Disallow: /deprecated/reference_en.html -Disallow: /deprecated/reference_ru.html +Disallow: /docs/fa/search.html Allow: / Host: https://clickhouse.yandex Sitemap: https://clickhouse.yandex/docs/sitemap.xml diff --git a/website/sitemap.xml b/website/sitemap.xml index db7bd695b58..e9319dc8701 100644 --- a/website/sitemap.xml +++ b/website/sitemap.xml @@ -9,6 +9,9 @@ https://clickhouse.yandex/docs/zh/sitemap.xml + + https://clickhouse.yandex/docs/ja/sitemap.xml + https://clickhouse.yandex/docs/fa/sitemap.xml diff --git a/website/tutorial.html b/website/tutorial.html deleted file mode 100644 index 4a6232d788e..00000000000 --- a/website/tutorial.html +++ /dev/null @@ -1,649 +0,0 @@ - - - - - ClickHouse Quick Start Guide - - - - - - - - - - -
- -
- - -

ClickHouse

-

Tutorial

-
- -

Let's get started with sample dataset from open sources. We will use USA civil flights data since 1987 till 2015. - It's hard to call this sample a Big Data (contains 166 millions rows, 63 Gb of uncompressed data) but this - allows us to quickly get to work. Dataset is available for download here. - Also you may download it from the original datasource as described here.

- -

Firstly we will deploy ClickHouse to a single server. Below that we will also review the process of deployment to - a cluster with support for sharding and replication.

- -

On Ubuntu and Debian Linux ClickHouse can be installed from packages. - For other Linux distributions you can compile - ClickHouse from sources and then install.

- -

clickhouse-client package contains clickhouse-client application — - interactive ClickHouse client. clickhouse-common contains a clickhouse-server binary file. clickhouse-server - — contains config files for the clickhouse-server.

- -

Server config files are located in /etc/clickhouse-server/. Before getting to work please notice the path - element in config. Path determines the location for data storage. It's not really handy to directly - edit config.xml file considering package updates. Recommended way is to override the config elements in - files of config.d directory. - Also you may want to set up access - rights at the start.

- -

clickhouse-server won't be launched automatically after package installation. It won't be automatically - restarted after updates either. Start the server with: - 

sudo service clickhouse-server start
- Default location for server logs is /var/log/clickhouse-server/ - Server is ready to handle client connections once "Ready for connections" message was logged.

- -

Use clickhouse-client to connect to the server.

- -
Tips for clickhouse-client -
- Interactive mode: -
-clickhouse-client
-clickhouse-client --host=... --port=... --user=... --password=...
-
- Enable multiline queries: -
-clickhouse-client -m
-clickhouse-client --multiline
-
- Run queries in batch-mode: -
-clickhouse-client --query='SELECT 1'
-echo 'SELECT 1' | clickhouse-client
-
- Insert data from file of a specified format: -
-clickhouse-client --query='INSERT INTO table VALUES' < data.txt
-clickhouse-client --query='INSERT INTO table FORMAT TabSeparated' < data.tsv
-
-
-
- -

Create table for sample dataset

-
Create table query -
-
-$ clickhouse-client --multiline
-ClickHouse client version 0.0.53720.
-Connecting to localhost:9000.
-Connected to ClickHouse server version 0.0.53720.
-
-:) CREATE TABLE ontime
-(
-    Year UInt16,
-    Quarter UInt8,
-    Month UInt8,
-    DayofMonth UInt8,
-    DayOfWeek UInt8,
-    FlightDate Date,
-    UniqueCarrier FixedString(7),
-    AirlineID Int32,
-    Carrier FixedString(2),
-    TailNum String,
-    FlightNum String,
-    OriginAirportID Int32,
-    OriginAirportSeqID Int32,
-    OriginCityMarketID Int32,
-    Origin FixedString(5),
-    OriginCityName String,
-    OriginState FixedString(2),
-    OriginStateFips String,
-    OriginStateName String,
-    OriginWac Int32,
-    DestAirportID Int32,
-    DestAirportSeqID Int32,
-    DestCityMarketID Int32,
-    Dest FixedString(5),
-    DestCityName String,
-    DestState FixedString(2),
-    DestStateFips String,
-    DestStateName String,
-    DestWac Int32,
-    CRSDepTime Int32,
-    DepTime Int32,
-    DepDelay Int32,
-    DepDelayMinutes Int32,
-    DepDel15 Int32,
-    DepartureDelayGroups String,
-    DepTimeBlk String,
-    TaxiOut Int32,
-    WheelsOff Int32,
-    WheelsOn Int32,
-    TaxiIn Int32,
-    CRSArrTime Int32,
-    ArrTime Int32,
-    ArrDelay Int32,
-    ArrDelayMinutes Int32,
-    ArrDel15 Int32,
-    ArrivalDelayGroups Int32,
-    ArrTimeBlk String,
-    Cancelled UInt8,
-    CancellationCode FixedString(1),
-    Diverted UInt8,
-    CRSElapsedTime Int32,
-    ActualElapsedTime Int32,
-    AirTime Int32,
-    Flights Int32,
-    Distance Int32,
-    DistanceGroup UInt8,
-    CarrierDelay Int32,
-    WeatherDelay Int32,
-    NASDelay Int32,
-    SecurityDelay Int32,
-    LateAircraftDelay Int32,
-    FirstDepTime String,
-    TotalAddGTime String,
-    LongestAddGTime String,
-    DivAirportLandings String,
-    DivReachedDest String,
-    DivActualElapsedTime String,
-    DivArrDelay String,
-    DivDistance String,
-    Div1Airport String,
-    Div1AirportID Int32,
-    Div1AirportSeqID Int32,
-    Div1WheelsOn String,
-    Div1TotalGTime String,
-    Div1LongestGTime String,
-    Div1WheelsOff String,
-    Div1TailNum String,
-    Div2Airport String,
-    Div2AirportID Int32,
-    Div2AirportSeqID Int32,
-    Div2WheelsOn String,
-    Div2TotalGTime String,
-    Div2LongestGTime String,
-    Div2WheelsOff String,
-    Div2TailNum String,
-    Div3Airport String,
-    Div3AirportID Int32,
-    Div3AirportSeqID Int32,
-    Div3WheelsOn String,
-    Div3TotalGTime String,
-    Div3LongestGTime String,
-    Div3WheelsOff String,
-    Div3TailNum String,
-    Div4Airport String,
-    Div4AirportID Int32,
-    Div4AirportSeqID Int32,
-    Div4WheelsOn String,
-    Div4TotalGTime String,
-    Div4LongestGTime String,
-    Div4WheelsOff String,
-    Div4TailNum String,
-    Div5Airport String,
-    Div5AirportID Int32,
-    Div5AirportSeqID Int32,
-    Div5WheelsOn String,
-    Div5TotalGTime String,
-    Div5LongestGTime String,
-    Div5WheelsOff String,
-    Div5TailNum String
-)
-ENGINE = MergeTree(FlightDate, (Year, FlightDate), 8192);
-
-
-
- -

Now we have a table of MergeTree type. - MergeTree table type is recommended for usage in production. Table of this kind has a primary key used for - incremental sort of table data. This allows fast execution of queries in ranges of a primary key.

- - -

Note - We store ad network banners impressions logs in ClickHouse. Each table entry looks like: - [Advertiser ID, Impression ID, attribute1, attribute2, …]. - Let assume that our aim is to provide a set of reports for each advertiser. Common and frequently demanded query - would be to count impressions for a specific Advertiser ID. This means that table primary key should start with - Advertiser ID. In this case ClickHouse needs to read smaller amount of data to perform the query for a - given Advertiser ID. -

- -

Load data

- 
xz -v -c -d < ontime.csv.xz | clickhouse-client --query="INSERT INTO ontime FORMAT CSV"
-

ClickHouse INSERT query allows to load data in any supported - format. Data load requires just O(1) RAM consumption. INSERT query can receive any data volume as input. - It's strongly recommended to insert data with not too small - size blocks. Notice that insert of blocks with size up to max_insert_block_size (= 1 048 576 - rows by default) is an atomic operation: data block will be inserted completely or not inserted at all. In case - of disconnect during insert operation you may not know if the block was inserted successfully. To achieve - exactly-once semantics ClickHouse supports idempotency for replicated tables. This means - that you may retry insert of the same data block (possibly on a different replicas) but this block will be - inserted just once. Anyway in this guide we will load data from our localhost so we may not take care about data - blocks generation and exactly-once semantics.

- -

INSERT query into tables of MergeTree type is non-blocking (so does a SELECT query). You can execute SELECT - queries right after of during insert operation.

- -

Our sample dataset is a bit not optimal. There are two reasons.

- -

The first is that String data type is used in cases when Enum or numeric type would fit best.

- -

⚖ When set of possible values is determined and known to be small. (E.g. OS name, browser - vendors etc.) it's recommended to use Enums or numbers to improve performance. - When set of possible values is not limited (search query, URL, etc.) just go ahead with String.

- -

The second is that dataset contains redundant fields like Year, Quarter, Month, DayOfMonth, DayOfWeek. In fact a - single FlightDate would be enough. Most likely they have been added to improve performance for other DBMS'es - which DateTime handling functions may be not efficient.

- -

✯ ClickHouse functions - for operating with DateTime fields are well-optimized so such redundancy is not required. Anyway much - columns is not a reason to worry — ClickHouse is a column-oriented - DBMS. This allows you to have as much fields as you need. Hundreds of columns in a table is fine for - ClickHouse.

- -

Querying the sample dataset

- -

Here are some examples of the queries from our test data.

- -
    -
  • -
    the most popular destinations in 2015; -
    -
    -SELECT
-    OriginCityName,
-    DestCityName,
-    count(*) AS flights,
-    bar(flights, 0, 20000, 40)
-FROM ontime WHERE Year = 2015 GROUP BY OriginCityName, DestCityName ORDER BY flights DESC LIMIT 20
-
    - -
    -SELECT
-    OriginCityName < DestCityName ? OriginCityName : DestCityName AS a,
-    OriginCityName < DestCityName ? DestCityName : OriginCityName AS b,
-    count(*) AS flights,
-    bar(flights, 0, 40000, 40)
-FROM ontime WHERE Year = 2015 GROUP BY a, b ORDER BY flights DESC LIMIT 20
-
    -
    -
    -
    • -
  • -
    the most popular cities of departure; -
    -
    -SELECT OriginCityName, count(*) AS flights
-FROM ontime GROUP BY OriginCityName ORDER BY flights DESC LIMIT 20
-
    -
    -
    -
    • -
  • -
    cities of departure which offer maximum variety of - destinations; -
    -
    -SELECT OriginCityName, uniq(Dest) AS u
-FROM ontime GROUP BY OriginCityName ORDER BY u DESC LIMIT 20
-
    -
    -
    -
    • -
  • -
    flight delay dependence on the day of week; -
    -
    -SELECT DayOfWeek, count() AS c, avg(DepDelay >  60) AS delays
-FROM ontime GROUP BY DayOfWeek ORDER BY DayOfWeek
-
    -
    -
    -
    • -
  • -
    cities of departure with most frequent delays for 1 hour or - longer; -
    -
    -SELECT OriginCityName, count() AS c, avg(DepDelay >  60) AS delays
-FROM ontime
-GROUP BY OriginCityName
-HAVING c >  100000
-ORDER BY delays DESC
-LIMIT 20
-
    -
    -
    -
    • -
  • -
    flights of maximum duration; -
    -
    -SELECT OriginCityName, DestCityName, count(*) AS flights, avg(AirTime) AS duration
-FROM ontime
-GROUP BY OriginCityName, DestCityName
-ORDER BY duration DESC
-LIMIT 20
-
    -
    -
    -
    • -
  • -
    distribution of arrival time delays split by aircompanies; -
    -
    -SELECT Carrier, count() AS c, round(quantileTDigest(0.99)(DepDelay), 2) AS q
-FROM ontime GROUP BY Carrier ORDER BY q DESC
-
    -
    -
    -
    • -
  • -
    aircompanies who stopped flights operation; -
    -
    -SELECT Carrier, min(Year), max(Year), count()
-FROM ontime GROUP BY Carrier HAVING max(Year) < 2015 ORDER BY count() DESC
-
    -
    -
    -
    • -
  • -
    most trending destination cities in 2015; -
    -
    -SELECT
-    DestCityName,
-    sum(Year = 2014) AS c2014,
-    sum(Year = 2015) AS c2015,
-    c2015 / c2014 AS diff
-FROM ontime
-WHERE Year IN (2014, 2015)
-GROUP BY DestCityName
-HAVING c2014 >  10000 AND c2015 >  1000 AND diff >  1
-ORDER BY diff DESC
-
    -
    -
    -
    • -
  • -
    destination cities with maximum popularity-season - dependency. -
    -
    -SELECT
-    DestCityName,
-    any(total),
-    avg(abs(monthly * 12 - total) / total) AS avg_month_diff
-FROM
-(
-    SELECT DestCityName, count() AS total
-    FROM ontime GROUP BY DestCityName HAVING total > 100000
-)
-ALL INNER JOIN
-(
-    SELECT DestCityName, Month, count() AS monthly
-    FROM ontime GROUP BY DestCityName, Month HAVING monthly > 10000
-)
-USING DestCityName
-GROUP BY DestCityName
-ORDER BY avg_month_diff DESC
-LIMIT 20
-
    -
    -
    -
    • -
- -

ClickHouse deployment to cluster

-

ClickHouse cluster is a homogenous cluster. Steps to set up: -

    -
  1. Install ClickHouse server on all machines of the cluster
    2. -
  2. Set up cluster configs in configuration file
    3. -
  3. Create local tables on each instance
    4. -
  4. Create a Distributed table
    5. -
-

- -

Distributed-table is actually a kind of - "view" to local tables of ClickHouse cluster. SELECT query from a distributed table will be executed using - resources of all cluster's shards. You may specify configs for multiple clusters and create multiple - Distributed-tables providing views to different clusters.

- -
Config for cluster of three shards. Each shard stores data on a single - replica -
-
-<remote_servers>
-    <perftest_3shards_1replicas>
-        <shard>
-            <replica>
-                <host>example-perftest01j.yandex.ru</host>
-                <port>9000</port>
-            </replica>
-        </shard>
-        <shard>
-            <replica>
-                <host>example-perftest02j.yandex.ru</host>
-                <port>9000</port>
-            </replica>
-        </shard>
-        <shard>
-            <replica>
-                <host>example-perftest03j.yandex.ru</host>
-                <port>9000</port>
-            </replica>
-        </shard>
-    </perftest_3shards_1replicas>
-</remote_servers>
-
-
-
- Creating a local table: - 
CREATE TABLE ontime_local (...) ENGINE = MergeTree(FlightDate, (Year, FlightDate), 8192);
- Creating a distributed table providing a view into local tables of the cluster: -
CREATE TABLE ontime_all AS ontime_local
-    ENGINE = Distributed(perftest_3shards_1replicas, default, ontime_local, rand());
- -

You can create a Distributed table on all machines in the cluster. This would allow to run distributed queries on - any machine of the cluster. Besides distributed table you can also use *remote* table function.

- -

Let's run INSERT SELECT into Distributed table - to spread the table to multiple servers.

- - 
INSERT INTO ontime_all SELECT * FROM ontime;
- -

⚠ Worth to notice that the approach given above wouldn't fit for sharding of large - tables.

- -

As you could expect heavy queries are executed N times faster being launched on 3 servers instead of one.

-
See here -
- - -

You may have noticed that quantiles calculation are slightly different. This happens due to t-digest - algorithm implementation which is non-deterministic — it depends on the order of data processing.

-
-
- -

In this case we have used a cluster with 3 shards each contains a single replica.

- -

To provide for resilience in production environment we recommend that each shard should contain 2-3 replicas - distributed between multiple data-centers. Note that ClickHouse supports unlimited number of replicas.

- -
Config for cluster of one shard containing three replicas -
-
-<remote_servers>
-    ...
-    <perftest_1shards_3replicas>
-        <shard>
-            <replica>
-                <host>example-perftest01j.yandex.ru</host>
-                <port>9000</port>
-             </replica>
-             <replica>
-                <host>example-perftest02j.yandex.ru</host>
-                <port>9000</port>
-             </replica>
-             <replica>
-                <host>example-perftest03j.yandex.ru</host>
-                <port>9000</port>
-             </replica>
-        </shard>
-    </perftest_1shards_3replicas>
-</remote_servers>
-
-
-
- -

To enable replication ZooKeeper is required. - ClickHouse will take care of data consistency on all replicas and run restore procedure after failure - automatically. It's recommended to deploy ZooKeeper cluster to separate servers.

- -

ZooKeeper is not a requirement — in some simple cases you can duplicate the data by writing it into all the - replicas from your application code. This approach is not recommended — in this case ClickHouse is not able to - guarantee data consistency on all replicas. This remains the responsibility of your application.

- -
Set ZooKeeper locations in configuration file -
-
-<zookeeper-servers>
-    <node>
-        <host>zoo01.yandex.ru</host>
-        <port>2181</port>
-    </node>
-    <node>
-        <host>zoo02.yandex.ru</host>
-        <port>2181</port>
-    </node>
-    <node>
-        <host>zoo03.yandex.ru</host>
-        <port>2181</port>
-    </node>
-</zookeeper-servers>
-
-
-
- -

Also we need to set macros for identifying shard and replica — it will be used on table creation

-
-<macros>
-    <shard>01</shard>
-    <replica>01</replica>
-</macros>
-
-

If there are no replicas at the moment on replicated table creation — a new first replica will be instantiated. - If there are already live replicas — new replica will clone the data from existing ones. You have an option to - create all replicated tables first and that insert data to it. Another option is to create some replicas and add - the others after or during data insertion.

- -
-CREATE TABLE ontime_replica (...)
-ENGINE = ReplicatedMergeTree(
-    '/clickhouse_perftest/tables/{shard}/ontime',
-    '{replica}',
-    FlightDate,
-    (Year, FlightDate),
-    8192);
-
-

Here we use ReplicatedMergeTree - table type. In parameters we specify ZooKeeper path containing shard and replica identifiers.

- - 
INSERT INTO ontime_replica SELECT * FROM ontime;
-

Replication operates in multi-master mode. Data can be loaded into any replica — it will be synced with other - instances automatically. Replication is asynchronous so at a given moment of time not all replicas may contain - recently inserted data. To allow data insertion at least one replica should be up. Others will sync up data and - repair consistency once they will become active again. Please notice that such scheme allows for the possibility - of just appended data loss.

- -

- ClickHouse source code is published under Apache 2.0 License. Software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied.

- - - -
- - - - - - - - - - -