From 4b46a9ca281a29b5525788ca9f8cb14f441f0e47 Mon Sep 17 00:00:00 2001 From: KochetovNicolai Date: Wed, 1 Aug 2018 13:37:35 +0300 Subject: [PATCH 01/20] Update CHANGELOG.draft.md --- CHANGELOG.draft.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.draft.md b/CHANGELOG.draft.md index 8b137891791..046f0475be1 100644 --- a/CHANGELOG.draft.md +++ b/CHANGELOG.draft.md @@ -1 +1,6 @@ +## RU +### Новые возможности: +* Добавлена поддержка ON выражений для JOIN ON синтаксиса: +`JOIN ON Expr([table.]column, ...) = Expr([table.]column, ...) [AND Expr([table.]column, ...) = Expr([table.]column, ...) ...]` +Выражение должно представлять из себя цепочку равенств, объединенных оператором AND. Каждая часть равенства может являться произвольным выражением над столбцами одной из таблиц. Поддержана возможность использования fully qualified имен столбцов (`table.name`, `database.table.name`, `table_alias.name`, `subquery_alias.name`) для правой таблицы. From 13f0dff3ed2ed9f87c84098eba1f38b0f04c6da4 Mon Sep 17 00:00:00 2001 From: KochetovNicolai Date: Wed, 1 Aug 2018 14:32:02 +0300 Subject: [PATCH 02/20] Update CHANGELOG.draft.md --- CHANGELOG.draft.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.draft.md b/CHANGELOG.draft.md index 046f0475be1..3ec4386ac76 100644 --- a/CHANGELOG.draft.md +++ b/CHANGELOG.draft.md @@ -3,4 +3,4 @@ ### Новые возможности: * Добавлена поддержка ON выражений для JOIN ON синтаксиса: `JOIN ON Expr([table.]column, ...) = Expr([table.]column, ...) [AND Expr([table.]column, ...) = Expr([table.]column, ...) ...]` -Выражение должно представлять из себя цепочку равенств, объединенных оператором AND. Каждая часть равенства может являться произвольным выражением над столбцами одной из таблиц. Поддержана возможность использования fully qualified имен столбцов (`table.name`, `database.table.name`, `table_alias.name`, `subquery_alias.name`) для правой таблицы. +Выражение должно представлять из себя цепочку равенств, объединенных оператором AND. Каждая часть равенства может являться произвольным выражением над столбцами одной из таблиц. Поддержана возможность использования fully qualified имен столбцов (`table.name`, `database.table.name`, `table_alias.name`, `subquery_alias.name`) для правой таблицы. [#2742](https://github.com/yandex/ClickHouse/pull/2742) From dc5249d0c1ebe86efc3a734ab7ce4175f2b00039 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 1 Aug 2018 16:20:46 +0300 Subject: [PATCH 03/20] Auto version update to [18.7.0] [54402] --- dbms/cmake/version.cmake | 10 +++++----- debian/changelog | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index 03a581d2a3a..fd1231769b3 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -1,11 +1,11 @@ # This strings autochanged from release_lib.sh: -set(VERSION_REVISION 54401 CACHE STRING "") +set(VERSION_REVISION 54402 CACHE STRING "") set(VERSION_MAJOR 18 CACHE STRING "") -set(VERSION_MINOR 6 CACHE STRING "") +set(VERSION_MINOR 7 CACHE STRING "") set(VERSION_PATCH 0 CACHE STRING "") -set(VERSION_GITHASH d72ba2701b90541f916d5e030e42617b09d5c6d7 CACHE STRING "") -set(VERSION_DESCRIBE v18.6.0-testing CACHE STRING "") -set(VERSION_STRING 18.6.0 CACHE STRING "") +set(VERSION_GITHASH 13f0dff3ed2ed9f87c84098eba1f38b0f04c6da4 CACHE STRING "") +set(VERSION_DESCRIBE v18.7.0-testing CACHE STRING "") +set(VERSION_STRING 18.7.0 CACHE STRING "") # end of autochange set(VERSION_EXTRA "" CACHE STRING "") diff --git a/debian/changelog b/debian/changelog index 2f24d69cd47..9f4737dd00b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (18.6.0) unstable; urgency=low +clickhouse (18.7.0) unstable; urgency=low * Modified source code - -- Wed, 01 Aug 2018 00:50:05 +0300 + -- Wed, 01 Aug 2018 16:20:46 +0300 From da3e2a6de2df3ab61548fe01057d421f137f4db4 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Wed, 1 Aug 2018 12:21:12 +0300 Subject: [PATCH 04/20] Added changes from english version of documentation to the russian one. --- docs/ru/data_types/datetime.md | 3 +- docs/ru/data_types/int_uint.md | 6 +- docs/ru/interfaces/jdbc.md | 3 +- docs/ru/interfaces/third-party_gui.md | 9 ++- docs/ru/operations/settings/index.md | 11 ++-- docs/ru/operations/system_tables.md | 50 +++++++++------- .../table_engines/aggregatingmergetree.md | 4 +- docs/ru/operations/table_engines/mysql.md | 14 ++++- .../agg_functions/parametric_functions.md | 58 +++++++++++++++++++ docs/ru/query_language/alter.md | 5 +- .../query_language/table_functions/numbers.md | 9 ++- 11 files changed, 131 insertions(+), 41 deletions(-) diff --git a/docs/ru/data_types/datetime.md b/docs/ru/data_types/datetime.md index 8b0ca24d9dd..dfc54e43add 100644 --- a/docs/ru/data_types/datetime.md +++ b/docs/ru/data_types/datetime.md @@ -1,5 +1,6 @@ -# DateTime + +# DateTime Дата-с-временем. Хранится в 4 байтах, в виде (беззнакового) unix timestamp. Позволяет хранить значения в том же интервале, что и для типа Date. Минимальное значение выводится как 0000-00-00 00:00:00. Время хранится с точностью до одной секунды (без учёта секунд координации). diff --git a/docs/ru/data_types/int_uint.md b/docs/ru/data_types/int_uint.md index d79ce7326bc..49b7bbbbcf8 100644 --- a/docs/ru/data_types/int_uint.md +++ b/docs/ru/data_types/int_uint.md @@ -1,3 +1,5 @@ + + # UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64 Целые числа фиксированной длины, без знака или со знаком. @@ -9,8 +11,8 @@ - Int32 - [ -2147483648 : 2147483647 ] - Int64 - [ -9223372036854775808 : 9223372036854775807 ] - - + + ## Диапазоны Uint - UInt8 - [ 0 : 255 ] diff --git a/docs/ru/interfaces/jdbc.md b/docs/ru/interfaces/jdbc.md index 4cdf4a5769f..90051ad464f 100644 --- a/docs/ru/interfaces/jdbc.md +++ b/docs/ru/interfaces/jdbc.md @@ -1,3 +1,4 @@ # JDBC-драйвер -Для ClickHouse существует официальный JDBC драйвер. Смотрите [здесь](https://github.com/yandex/clickhouse-jdbc) . +- [Официальный драйвер](https://github.com/yandex/clickhouse-jdbc). +- Драйвер от сторонней огранизации [ClickHouse-Native-JDBC](https://github.com/housepower/ClickHouse-Native-JDBC). diff --git a/docs/ru/interfaces/third-party_gui.md b/docs/ru/interfaces/third-party_gui.md index 676b58a7ba0..a06a6f3bc2e 100644 --- a/docs/ru/interfaces/third-party_gui.md +++ b/docs/ru/interfaces/third-party_gui.md @@ -20,13 +20,18 @@ Основные возможности: -- Создание запросов. +- Построение запросов с подсветкой синтаксиса. Просмотр ответа в табличном или JSON представлении. +- Экспортирование результатов запроса в формате CSV или JSON. +- Список процессов с описанием. Режим записи. Возможность остановки (`KILL`) процесса. +- Граф базы данных. Показывает все таблицы и их столбцы с дополнительной информацией. +- Быстрый просмотр размера стоблбца. +- Конфигурирование сервера. Планируется разработка следующих возможностей: - Управление базами. - Управление пользователями. -- Управление кластером. - Анализ данных в режиме реального времени. - Мониторинг кластера. +- Управление кластером. - Мониторинг реплицированных и Kafka таблиц. diff --git a/docs/ru/operations/settings/index.md b/docs/ru/operations/settings/index.md index f6da2dcccae..3de41c00b3f 100644 --- a/docs/ru/operations/settings/index.md +++ b/docs/ru/operations/settings/index.md @@ -5,18 +5,19 @@ Все настройки, описанные ниже, могут быть заданы несколькими способами. Настройки задаются послойно, т.е. каждый следующий слой перезаписывает предыдущие настройки. -Способы задания настроек, упорядоченные по их приоритету: +Способы задания настроек, упорядоченные по приоритету: -- Настройки в конфигурационных файлах сервера. +- Настройки в конфигурационном файле сервера `users.xml`. - Задаются через профили пользователей. + Устанавливаются в элементе ``. -- Для сессии. +- Настройки для сессии. Из консольного клиента ClickHouse в интерактивном режиме отправьте запрос `SET setting=value`. Аналогично можно использовать ClickHouse-сессии в HTTP-протоколе, для этого необходимо указывать HTTP-праметр `session_id`. -- Для запроса. +- Настройки для запроса. + - При запуске консольного клиента ClickHouse в неинтерактивном режиме установите параметр запуска `--setting=value`. - При использовании HTTP API передавайте cgi-параметры (`URL?setting_1=value&setting_2=value...`). diff --git a/docs/ru/operations/system_tables.md b/docs/ru/operations/system_tables.md index 3d8f3032d10..84fcb14092b 100644 --- a/docs/ru/operations/system_tables.md +++ b/docs/ru/operations/system_tables.md @@ -7,13 +7,14 @@ Системные таблицы расположены в базе данных system. -# system.asynchronous_metrics +## system.asynchronous_metrics Содержат метрики, используемые для профилирования и мониторинга. Обычно отражают количество событий, происходящих в данный момент в системе, или ресурсов, суммарно потребляемых системой. Пример: количество запросов типа SELECT, исполняемых в текущий момент; количество потребляемой памяти. `system.asynchronous_metrics` и `system.metrics` отличаются набором и способом вычисления метрик. -# system.clusters + +## system.clusters Содержит информацию о доступных в конфигурационном файле кластерах и серверах, которые в них входят. Столбцы: @@ -28,7 +29,7 @@ host_address String - IP-адрес хоста, полученный из DNS port UInt16 - порт, на который обращаться для соединения с сервером user String - имя пользователя, которого использовать для соединения с сервером ``` -# system.columns +## system.columns Содержит информацию о столбцах всех таблиц. С помощью этой таблицы можно получить информацию аналогично запросу `DESCRIBE TABLE`, но для многих таблиц сразу. @@ -41,12 +42,12 @@ type String - тип столбца default_type String - тип (DEFAULT, MATERIALIZED, ALIAS) выражения для значения по умолчанию, или пустая строка, если оно не описано default_expression String - выражение для значения по умолчанию, или пустая строка, если оно не описано ``` -# system.databases +## system.databases Таблица содержит один столбец name типа String - имя базы данных. Для каждой базы данных, о которой знает сервер, будет присутствовать соответствующая запись в таблице. Эта системная таблица используется для реализации запроса `SHOW DATABASES`. -# system.dictionaries +## system.dictionaries Содержит информацию о внешних словарях. @@ -70,12 +71,12 @@ default_expression String - выражение для значения по ум Заметим, что количество оперативной памяти, которое использует словарь, не является пропорциональным количеству элементов, хранящихся в словаре. Так, для flat и cached словарей, все ячейки памяти выделяются заранее, независимо от реальной заполненности словаря. -# system.events +## system.events Содержит информацию о количестве произошедших в системе событий, для профилирования и мониторинга. Пример: количество обработанных запросов типа SELECT. Столбцы: event String - имя события, value UInt64 - количество. -# system.functions +## system.functions Содержит информацию об обычных и агрегатных функциях. @@ -83,7 +84,7 @@ default_expression String - выражение для значения по ум - `name` (`String`) – Имя функции. - `is_aggregate` (`UInt8`) – Признак, является ли функция агрегатной. -# system.merges +## system.merges Содержит информацию о производящихся прямо сейчас слияниях для таблиц семейства MergeTree. @@ -103,22 +104,23 @@ default_expression String - выражение для значения по ум - `rows_written UInt64` — Количество записанных строк. -# system.metrics -# system.numbers +## system.metrics +## system.numbers Таблица содержит один столбец с именем number типа UInt64, содержащим почти все натуральные числа, начиная с нуля. Эту таблицу можно использовать для тестов, а также если вам нужно сделать перебор. Чтения из этой таблицы не распараллеливаются. -# system.numbers_mt +## system.numbers_mt То же самое, что и system.numbers, но чтение распараллеливается. Числа могут возвращаться в произвольном порядке. Используется для тестов. -# system.one +## system.one Таблица содержит одну строку с одним столбцом dummy типа UInt8, содержащим значение 0. Эта таблица используется, если в SELECT запросе не указана секция FROM. То есть, это - аналог таблицы DUAL, которую можно найти в других СУБД. -# system.parts + +## system.parts Содержит информацию о кусках таблиц семейства [MergeTree](../operations/table_engines/mergetree.md#table_engines-mergetree). @@ -126,7 +128,13 @@ default_expression String - выражение для значения по ум Столбцы: -- partition (String) - Имя партиции. Формат YYYYMM. Что такое партиция можно узнать из описания запроса [ALTER](../query_language/alter.md#query_language_queries_alter). +- partition (String) - Имя партиции. Что такое партиция можно узнать из описания запроса [ALTER](../query_language/alter.md#query_language_queries_alter). + + Форматы: + + - `YYYYMM` для автоматической схемы партиционирования по месяцам. + - `any_string` при партиционировании вручную. + - name (String) - Имя куска. - active (UInt8) - Признак активности. Если кусок активен, то он используется таблице, в противном случает он будет удален. Неактивные куски остаются после слияний. - marks (UInt64) - Количество засечек. Чтобы получить примерное количество строк в куске, умножьте ``marks`` на гранулированность индекса (обычно 8192). @@ -146,7 +154,7 @@ default_expression String - выражение для значения по ум - database (String) - Имя базы данных. - table (String) - Имя таблицы. - engine (String) - Имя движка таблицы, без параметров. -# system.processes +## system.processes Эта системная таблица используется для реализации запроса `SHOW PROCESSLIST`. Столбцы: @@ -170,7 +178,7 @@ query String - текст запроса. В случае INSERT - query_id String - идентификатор запроса, если был задан. ``` -# system.replicas +## system.replicas Содержит информацию и статус для реплицируемых таблиц, расположенных на локальном сервере. Эту таблицу можно использовать для мониторинга. Таблица содержит по строчке для каждой Replicated\*-таблицы. @@ -294,7 +302,7 @@ WHERE ``` Если этот запрос ничего не возвращает - значит всё хорошо. -# system.settings +## system.settings Содержит информацию о настройках, используемых в данный момент. То есть, используемых для выполнения запроса, с помощью которого вы читаете из таблицы system.settings. @@ -323,15 +331,17 @@ WHERE changed │ max_memory_usage │ 10000000000 │ 1 │ └────────────────────────┴─────────────┴─────────┘ ``` -# system.tables +## system.tables Таблица содержит столбцы database, name, engine типа String. Также таблица содержит три виртуальных столбца: metadata_modification_time типа DateTime, create_table_query и engine_full типа String. Для каждой таблицы, о которой знает сервер, будет присутствовать соответствующая запись в таблице system.tables. Эта системная таблица используется для реализации запросов SHOW TABLES. -# system.zookeeper -Позволяет читать данные из ZooKeeper кластера, описанного в конфигурации. + +## system.zookeeper + +Таблицы не существует, если ZooKeeper не сконфигурирован. Позволяет читать данные из ZooKeeper кластера, описанного в конфигурации. В запросе обязательно в секции WHERE должно присутствовать условие на равенство path - путь в ZooKeeper, для детей которого вы хотите получить данные. Запрос `SELECT * FROM system.zookeeper WHERE path = '/clickhouse'` выведет данные по всем детям узла `/clickhouse`. diff --git a/docs/ru/operations/table_engines/aggregatingmergetree.md b/docs/ru/operations/table_engines/aggregatingmergetree.md index f14bb679cd0..cff9116988d 100644 --- a/docs/ru/operations/table_engines/aggregatingmergetree.md +++ b/docs/ru/operations/table_engines/aggregatingmergetree.md @@ -28,8 +28,8 @@ CREATE TABLE t Значение типа `AggregateFunction` нельзя вывести в Pretty-форматах. В других форматах, значения такого типа выводятся в виде implementation-specific бинарных данных. То есть, значения типа `AggregateFunction` не предназначены для вывода, сохранения в дамп. -Единственную полезную вещь, которую можно сделать со значениями типа `AggregateFunction` - это объединить состояния и получить результат, по сути - доагрегировать до конца. Для этого используются агрегатные функции с суффиксом Merge. -Пример: `uniqMerge(UserIDState), где UserIDState имеет тип AggregateFunction`. +Единственную полезную вещь, которую можно сделать со значениями типа `AggregateFunction` — это объединить состояния и получить результат, по сути — доагрегировать до конца. Для этого используются агрегатные функции с суффиксом Merge. +Пример: `uniqMerge(UserIDState)`, где `UserIDState` имеет тип `AggregateFunction`. То есть, агрегатная функция с суффиксом Merge берёт множество состояний, объединяет их, и возвращает готовый результат. Для примера, эти два запроса возвращают один и тот же результат: diff --git a/docs/ru/operations/table_engines/mysql.md b/docs/ru/operations/table_engines/mysql.md index b62320633ea..e08edc4e180 100644 --- a/docs/ru/operations/table_engines/mysql.md +++ b/docs/ru/operations/table_engines/mysql.md @@ -7,11 +7,21 @@ Формат вызова: ``` -MySQL('host:port', 'database', 'table', 'user', 'password'); +MySQL('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']); ``` +**Параметры вызова** + +- `host:port` — Адрес сервера MySQL. +- `database` — Имя базы данных на сервере MySQL. +- `table` — Имя таблицы. +- `user` — Пользователь MySQL. +- `password` — Пароль пользователя. +- `replace_query` — Флаг, устанавливающий замену запроса `INSERT INTO` на `REPLACE INTO`. Если `replace_query=1`, то запрос заменяется. +- `'on_duplicate_clause'` — Добавляет выражение `ON DUPLICATE KEY UPDATE 'on_duplicate_clause'` в запрос `INSERT`. Например, `impression = VALUES(impression) + impression`. Чтобы указать `'on_duplicate_clause'` необходимо передать `0` в параметр `replace_query`. Если одновременно передать `replace_query = 1` и `'on_duplicate_clause'`, то ClickHouse сгенерирует исключение. + На данный момент простые условия `WHERE`, такие как `=, !=, >, >=, <, <=` будут выполняться на стороне сервера MySQL. Остальные условия и ограничение выборки `LIMIT` будут выполнены в ClickHouse только после выполнения запроса к MySQL. -Движок `MySQL` не поддерживает тип данных [Nullable](../../data_types/nullable.md#data_type-nullable), поэтому при чтении данных из таблиц MySQL `NULL` преобразуются в значения по умолчанию для заданного типа столбца, обычно это `0` или пустая строка. +Движок `MySQL` не поддерживает тип данных [Nullable](../../data_types/nullable.md#data_type-nullable), поэтому при чтении данных из таблиц MySQL `NULL` преобразуются в значения по умолчанию для заданного типа столбца, обычно это 0 или пустая строка. diff --git a/docs/ru/query_language/agg_functions/parametric_functions.md b/docs/ru/query_language/agg_functions/parametric_functions.md index 4a1c1ee7d38..9754f6444c4 100644 --- a/docs/ru/query_language/agg_functions/parametric_functions.md +++ b/docs/ru/query_language/agg_functions/parametric_functions.md @@ -50,6 +50,64 @@ minIf(EventTime, URL LIKE '%company%') < maxIf(EventTime, URL LIKE '%cart%'). Аналогично функции sequenceMatch, но возвращает не факт наличия цепочки событий, а UInt64 - количество найденных цепочек. Цепочки ищутся без перекрытия. То есть, следующая цепочка может начаться только после окончания предыдущей. +## windowFunnel(window)(timestamp, cond1, cond2, cond3, ...) + +Отыскивает цепочки событий в скользящем окне по времени и вычисляет максимальное количество произошедших событий из цепочки. + + +``` +windowFunnel(window)(timestamp, cond1, cond2, cond3, ...) +``` + +**Параметры** + +- `window` — ширина скользящего окна по времени в секундах. +- `timestamp` — имя столбца, содержащего отметки времени. Тип данных [DateTime](../../data_types/datetime.md#data_type-datetime) или [UInt32](../../data_types/int_uint.md#data_type-int). +- `cond1`, `cond2`... — условия или данные, описывающие цепочку событий. Тип данных — `UInt8`. Значения могут быть 0 или 1. + +**Алгоритм** + +- Функция отыскивает данные, на которых срабатывает первое условие из цепочки, и присваивает счетчику событий значение 1. С этого же момента начинается отсчет времени скользящего окна. +- Если в пределах окна последовательно попадаются события из цепочки, то счетчик увеличивается. Если последовательность событий нарушается, то счетчик не растёт. +- На каждое первое событие из цепочки отсчет начинается заново. +- Если в данных оказалось несколько цепочек разной степени завершенности, то функция выдаст только размер самой длинной цепочки. + +**Возвращаемое значение** + +- Целое число. Максимальное количество последовательно сработавших условий из цепочки в пределах скользящего окна по времени. Исследуются все цепочки в выборке. + +**Пример** + +Определим, успевает ли пользователь за час выбрать телефон в интернет-магазине и купить его. + +Зададим следующую цепочку событий: + +1. Пользователь вошел в личный кабинет магазина (`eventID=1001`). +2. Пользователь ищет телефон (`eventID = 1003, product = 'phone'`). +3. Пользователь сделал заказ (`eventID = 1009`). + +Чтобы узнать, как далеко пользователь `user_id` смог пройти по цепочке за час в январе 2017-го года, составим запрос: + +``` +SELECT + level, + count() AS c +FROM +( + SELECT + user_id, + windowFunnel(3600)(timestamp, eventID = 1001, eventID = 1003 AND product = 'phone', eventID = 1009) AS level + FROM trend_event + WHERE (event_date >= '2017-01-01') AND (event_date <= '2017-01-31') + GROUP BY user_id +) +GROUP BY level +ORDER BY level +``` + +В результате мы можем получить 0, 1, 2 или 3 в зависимости от действий пользователя. + + ## uniqUpTo(N)(x) Вычисляет количество различных значений аргумента, если оно меньше или равно N. diff --git a/docs/ru/query_language/alter.md b/docs/ru/query_language/alter.md index b26a3ba9e32..454114231f6 100644 --- a/docs/ru/query_language/alter.md +++ b/docs/ru/query_language/alter.md @@ -225,7 +225,7 @@ ALTER TABLE [db.]table FETCH PARTITION 'name' FROM 'path-in-zookeeper' Мутации - разновидность запроса ALTER, позволяющая изменять или удалять данные в таблице. В отличие от стандартных запросов `DELETE` и `UPDATE`, рассчитанных на точечное изменение данных, область применения мутаций - достаточно тяжёлые изменения, затрагивающие много строк в таблице. -Функциональность находится в состоянии beta и доступна начиная с версии 1.1.54388. Реализована поддержка *MergeTree таблиц (с репликацией и без). +Функциональность находится в состоянии beta и доступна начиная с версии 1.1.54388. Реализована поддержка \*MergeTree таблиц (с репликацией и без). Конвертировать существующие таблицы для работы с мутациями не нужно. Но после применения первой мутации формат данных таблицы становится несовместимым с предыдущими версиями и откатиться на предыдущую версию уже не получится. @@ -239,7 +239,7 @@ ALTER TABLE [db.]table DELETE WHERE expr В одном запросе можно указать несколько команд через запятую. -Для *MergeTree-таблиц мутации выполняются, перезаписывая данные по кускам (parts). При этом атомарности нет - куски заменяются на помутированные по мере выполнения и запрос `SELECT`, заданный во время выполнения мутации, увидит данные как из измененных кусков, так и из кусков, которые еще не были изменены. +Для \*MergeTree-таблиц мутации выполняются, перезаписывая данные по кускам (parts). При этом атомарности нет — куски заменяются на помутированные по мере выполнения и запрос `SELECT`, заданный во время выполнения мутации, увидит данные как из измененных кусков, так и из кусков, которые еще не были изменены. Мутации линейно упорядочены между собой и накладываются на каждый кусок в порядке добавления. Мутации также упорядочены со вставками - гарантируется, что данные, вставленные в таблицу до начала выполнения запроса мутации, будут изменены, а данные, вставленные после окончания запроса мутации, изменены не будут. При этом мутации никак не блокируют вставки. @@ -262,4 +262,3 @@ ALTER TABLE [db.]table DELETE WHERE expr **parts_to_do** - Количество кусков таблицы, которые ещё предстоит изменить. **is_done** - Завершена ли мутация. Замечание: даже если `parts_to_do = 0`, для реплицированной таблицы возможна ситуация, когда мутация ещё не завершена из-за долго выполняющейся вставки, которая добавляет данные, которые нужно будет мутировать. - diff --git a/docs/ru/query_language/table_functions/numbers.md b/docs/ru/query_language/table_functions/numbers.md index 62da54b8d2b..8ff5ae3ac97 100644 --- a/docs/ru/query_language/table_functions/numbers.md +++ b/docs/ru/query_language/table_functions/numbers.md @@ -1,12 +1,15 @@ # numbers -`numbers(N)` - возвращает таблицу с единственным столбцом number (тип UInt64), содержащим натуральные числа от 0 до N-1. +`numbers(N)` - возвращает таблицу с единственным столбцом `number` (UInt64), содержащим натуральные числа от `0` до `N-1`. +`numbers(N, M)` - возвращает таблицу с единственным столбцом `number` (UInt64), содержащим натуральные числа от `N` to `(N + M - 1)`. -Так же как и таблица `system.numbers` может использоваться для тестов и генерации последовательных значений. +Так же как и таблица `system.numbers` может использоваться для тестов и генерации последовательных значений. Функция `numbers(N, M)` работает более эффективно, чем выборка из `system.numbers`. + +Следующие запросы эквивалентны: -Следующие 2 запроса эквивалентны: ```sql SELECT * FROM numbers(10); +SELECT * FROM numbers(0,10); SELECT * FROM system.numbers LIMIT 10; ``` Примеры: From 9f23d7c497af1a0df9708f76d45ac5a4c796df5e Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 1 Aug 2018 19:45:59 +0300 Subject: [PATCH 05/20] Update parametric_functions.md --- docs/ru/query_language/agg_functions/parametric_functions.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/ru/query_language/agg_functions/parametric_functions.md b/docs/ru/query_language/agg_functions/parametric_functions.md index 9754f6444c4..b86b75baf6c 100644 --- a/docs/ru/query_language/agg_functions/parametric_functions.md +++ b/docs/ru/query_language/agg_functions/parametric_functions.md @@ -69,7 +69,6 @@ windowFunnel(window)(timestamp, cond1, cond2, cond3, ...) - Функция отыскивает данные, на которых срабатывает первое условие из цепочки, и присваивает счетчику событий значение 1. С этого же момента начинается отсчет времени скользящего окна. - Если в пределах окна последовательно попадаются события из цепочки, то счетчик увеличивается. Если последовательность событий нарушается, то счетчик не растёт. -- На каждое первое событие из цепочки отсчет начинается заново. - Если в данных оказалось несколько цепочек разной степени завершенности, то функция выдаст только размер самой длинной цепочки. **Возвращаемое значение** From 11940f823e3317c1e2010aff6695bb2831670fdc Mon Sep 17 00:00:00 2001 From: Amy Krishnevsky Date: Wed, 1 Aug 2018 11:52:59 -0500 Subject: [PATCH 06/20] translated last two releases https://st.yandex-team.ru/TANKER-452304 --- CHANGELOG.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed71baf8046..36e53821d7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,41 @@ +## ClickHouse release 18.5.1, 2018-07-31 + +### New features: + +* Added the hash function `murmurHash2_32` [#2756](https://github.com/yandex/ClickHouse/pull/2756). + +### Improvements: + +* Now you can use the `from_env` attribute to set values in config files from environment variables [#2741](https://github.com/yandex/ClickHouse/pull/2741). +* Added case-insensitive versions of the `coalesce`, `ifNull`, and `nullIf functions` [#2752](https://github.com/yandex/ClickHouse/pull/2752). + +### Bug fixes: + +* Fixed a possible bug when starting a replica [#2759](https://github.com/yandex/ClickHouse/pull/2759). + +## ClickHouse release 18.4.0, 2018-07-28 + +### New features: + +* Added system tables: `formats`, `data_type_families`, `aggregate_function_combined`, `table_functions`, `table_engines`, `collections` [#2721](https://github.com/yandex/ClickHouse/pull/2721). +* Added the ability to use a table function instead of a table as an argument of a `remote` or `cluster table function` [#2708](https://github.com/yandex/ClickHouse/pull/2708). +* Support for `HTTP Basic` authentication in the replication protocol [#2727](https://github.com/yandex/ClickHouse/pull/2727). +* The `has` function now allows searching for a numeric value in an array of `Enum` values [Maxim Khrisanfov](https://github.com/yandex/ClickHouse/pull/2699). +* Support for adding arbitrary message separators when reading from `Kafka` [Amos Bird](https://github.com/yandex/ClickHouse/pull/2701). + +### Improvements: + +* The `ALTER TABLE t DELETE WHERE` query does not overwrite data chunks that were not affected by the WHERE condition [#2694](https://github.com/yandex/ClickHouse/pull/2694). +* The `use_minimalistic_checksums_in_zookeeper` option for `ReplicatedMergeTree` tables is enabled by default. This setting was added in version 1.1.54378, 2018-04-16. Versions that are older than 1.1.54378 can no longer be installed. +* Support for running `KILL` and `OPTIMIZE` queries that specify `ON CLUSTER` [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2689). + +### Bug fixes: + +* Fixed the error `Column ... is not under an aggregate function and not in GROUP BY` for aggregation with an IN expression. This bug appeared in version 18.1.0. ([bbdd780b](https://github.com/yandex/ClickHouse/commit/bbdd780be0be06a0f336775941cdd536878dd2c2)) +* Fixed a bug in the `windowFunnel aggregate function` [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2735). +* Fixed a bug in the `anyHeavy` aggregate function ([a2101df2](https://github.com/yandex/ClickHouse/commit/a2101df25a6a0fba99aa71f8793d762af2b801ee)) +* Fixed server crash when using the `countArray()` feature. + ## ClickHouse release 18.1.0, 2018-07-23 ### New features: From d46434eda14068c516e33e099beaff7d4c685e53 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 1 Aug 2018 20:05:53 +0300 Subject: [PATCH 07/20] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36e53821d7a..510edd3c307 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ ### New features: -* Added system tables: `formats`, `data_type_families`, `aggregate_function_combined`, `table_functions`, `table_engines`, `collections` [#2721](https://github.com/yandex/ClickHouse/pull/2721). +* Added system tables: `formats`, `data_type_families`, `aggregate_function_combined`, `table_functions`, `table_engines`, `collations` [#2721](https://github.com/yandex/ClickHouse/pull/2721). * Added the ability to use a table function instead of a table as an argument of a `remote` or `cluster table function` [#2708](https://github.com/yandex/ClickHouse/pull/2708). * Support for `HTTP Basic` authentication in the replication protocol [#2727](https://github.com/yandex/ClickHouse/pull/2727). * The `has` function now allows searching for a numeric value in an array of `Enum` values [Maxim Khrisanfov](https://github.com/yandex/ClickHouse/pull/2699). From 535e19764f4bc307698dfbb1a939830ec35f319a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 1 Aug 2018 20:06:08 +0300 Subject: [PATCH 08/20] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 510edd3c307..ddbb12ec9e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ ### New features: * Added system tables: `formats`, `data_type_families`, `aggregate_function_combined`, `table_functions`, `table_engines`, `collations` [#2721](https://github.com/yandex/ClickHouse/pull/2721). -* Added the ability to use a table function instead of a table as an argument of a `remote` or `cluster table function` [#2708](https://github.com/yandex/ClickHouse/pull/2708). +* Added the ability to use a table function instead of a table as an argument of a `remote` or `cluster` table function [#2708](https://github.com/yandex/ClickHouse/pull/2708). * Support for `HTTP Basic` authentication in the replication protocol [#2727](https://github.com/yandex/ClickHouse/pull/2727). * The `has` function now allows searching for a numeric value in an array of `Enum` values [Maxim Khrisanfov](https://github.com/yandex/ClickHouse/pull/2699). * Support for adding arbitrary message separators when reading from `Kafka` [Amos Bird](https://github.com/yandex/ClickHouse/pull/2701). From 22c7fd2030d672fd3aacb40c7364e7fcdb7d9d93 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 1 Aug 2018 20:07:08 +0300 Subject: [PATCH 09/20] Update CHANGELOG.md --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddbb12ec9e6..3bb55ea8754 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,20 +19,20 @@ * Added system tables: `formats`, `data_type_families`, `aggregate_function_combined`, `table_functions`, `table_engines`, `collations` [#2721](https://github.com/yandex/ClickHouse/pull/2721). * Added the ability to use a table function instead of a table as an argument of a `remote` or `cluster` table function [#2708](https://github.com/yandex/ClickHouse/pull/2708). -* Support for `HTTP Basic` authentication in the replication protocol [#2727](https://github.com/yandex/ClickHouse/pull/2727). +* Support for `HTTP Basic` authentication in the replication protocol [#2727](https://github.com/yandex/ClickHouse/pull/2727). * The `has` function now allows searching for a numeric value in an array of `Enum` values [Maxim Khrisanfov](https://github.com/yandex/ClickHouse/pull/2699). * Support for adding arbitrary message separators when reading from `Kafka` [Amos Bird](https://github.com/yandex/ClickHouse/pull/2701). ### Improvements: -* The `ALTER TABLE t DELETE WHERE` query does not overwrite data chunks that were not affected by the WHERE condition [#2694](https://github.com/yandex/ClickHouse/pull/2694). +* The `ALTER TABLE t DELETE WHERE` query does not rewrite data chunks that were not affected by the WHERE condition [#2694](https://github.com/yandex/ClickHouse/pull/2694). * The `use_minimalistic_checksums_in_zookeeper` option for `ReplicatedMergeTree` tables is enabled by default. This setting was added in version 1.1.54378, 2018-04-16. Versions that are older than 1.1.54378 can no longer be installed. * Support for running `KILL` and `OPTIMIZE` queries that specify `ON CLUSTER` [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2689). ### Bug fixes: * Fixed the error `Column ... is not under an aggregate function and not in GROUP BY` for aggregation with an IN expression. This bug appeared in version 18.1.0. ([bbdd780b](https://github.com/yandex/ClickHouse/commit/bbdd780be0be06a0f336775941cdd536878dd2c2)) -* Fixed a bug in the `windowFunnel aggregate function` [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2735). +* Fixed a bug in the `windowFunnel` aggregate function [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2735). * Fixed a bug in the `anyHeavy` aggregate function ([a2101df2](https://github.com/yandex/ClickHouse/commit/a2101df25a6a0fba99aa71f8793d762af2b801ee)) * Fixed server crash when using the `countArray()` feature. From 1799ca5a4e8af2862beb513f266d4b8cc873146c Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 1 Aug 2018 20:07:21 +0300 Subject: [PATCH 10/20] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bb55ea8754..b5f3a41410a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,7 @@ * Fixed the error `Column ... is not under an aggregate function and not in GROUP BY` for aggregation with an IN expression. This bug appeared in version 18.1.0. ([bbdd780b](https://github.com/yandex/ClickHouse/commit/bbdd780be0be06a0f336775941cdd536878dd2c2)) * Fixed a bug in the `windowFunnel` aggregate function [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2735). * Fixed a bug in the `anyHeavy` aggregate function ([a2101df2](https://github.com/yandex/ClickHouse/commit/a2101df25a6a0fba99aa71f8793d762af2b801ee)) -* Fixed server crash when using the `countArray()` feature. +* Fixed server crash when using the `countArray()` aggregate function. ## ClickHouse release 18.1.0, 2018-07-23 From eb697dca93a0ddea05c7a580c5c94e041d169789 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 1 Aug 2018 21:13:35 +0300 Subject: [PATCH 11/20] Addition to prev. revision [#CLICKHOUSE-3861] --- dbms/src/TableFunctions/TableFunctionODBC.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/TableFunctions/TableFunctionODBC.cpp b/dbms/src/TableFunctions/TableFunctionODBC.cpp index 7d274156075..a45283c65f2 100644 --- a/dbms/src/TableFunctions/TableFunctionODBC.cpp +++ b/dbms/src/TableFunctions/TableFunctionODBC.cpp @@ -9,7 +9,7 @@ #include #include #include - +#include #include #include #include @@ -76,7 +76,7 @@ StoragePtr TableFunctionODBC::executeImpl(const ASTPtr & ast_function, const Con for (int i = 0; i < 2; ++i) args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(args[i], context); - std::string connection_string = static_cast(*args[0]).value.safeGet(); + std::string connection_string = validateODBCConnectionString(static_cast(*args[0]).value.safeGet()); std::string table_name = static_cast(*args[1]).value.safeGet(); Poco::Data::ODBC::SessionImpl session(connection_string, DBMS_DEFAULT_CONNECT_TIMEOUT_SEC); From 60187ab716b0a8143a65880672b4d11802430a03 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 1 Aug 2018 21:21:03 +0300 Subject: [PATCH 12/20] Miscellaneous [#CLICKHOUSE-3861] --- dbms/src/Dictionaries/validateODBCConnectionString.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dbms/src/Dictionaries/validateODBCConnectionString.cpp b/dbms/src/Dictionaries/validateODBCConnectionString.cpp index c394661af87..c8c7851d5f8 100644 --- a/dbms/src/Dictionaries/validateODBCConnectionString.cpp +++ b/dbms/src/Dictionaries/validateODBCConnectionString.cpp @@ -39,6 +39,9 @@ std::string validateODBCConnectionString(const std::string & connection_string) if (connection_string.empty()) throw Exception("ODBC connection string cannot be empty", ErrorCodes::BAD_ODBC_CONNECTION_STRING); + if (connection_string.size() >= MAX_CONNECTION_STRING_SIZE) + throw Exception("ODBC connection string is too long", ErrorCodes::BAD_ODBC_CONNECTION_STRING); + const char * pos = connection_string.data(); const char * end = pos + connection_string.size(); From d799ba683dd992d45113106ab9d57d7fdea05d99 Mon Sep 17 00:00:00 2001 From: Amy Krishnevsky Date: Wed, 1 Aug 2018 13:10:58 -0500 Subject: [PATCH 13/20] fixed typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5f3a41410a..4f26f565e8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ ### New features: -* Added system tables: `formats`, `data_type_families`, `aggregate_function_combined`, `table_functions`, `table_engines`, `collations` [#2721](https://github.com/yandex/ClickHouse/pull/2721). +* Added system tables: `formats`, `data_type_families`, `aggregate_function_combinators`, `table_functions`, `table_engines`, `collations` [#2721](https://github.com/yandex/ClickHouse/pull/2721). * Added the ability to use a table function instead of a table as an argument of a `remote` or `cluster` table function [#2708](https://github.com/yandex/ClickHouse/pull/2708). * Support for `HTTP Basic` authentication in the replication protocol [#2727](https://github.com/yandex/ClickHouse/pull/2727). * The `has` function now allows searching for a numeric value in an array of `Enum` values [Maxim Khrisanfov](https://github.com/yandex/ClickHouse/pull/2699). From 0906c61a106d580c8722877f347616ddc825ddfe Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 2 Aug 2018 01:24:01 +0300 Subject: [PATCH 14/20] Auto version update to [18.7.1] [54402] --- dbms/cmake/version.cmake | 8 ++++---- debian/changelog | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index fd1231769b3..c5ac46ab75a 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -2,10 +2,10 @@ set(VERSION_REVISION 54402 CACHE STRING "") set(VERSION_MAJOR 18 CACHE STRING "") set(VERSION_MINOR 7 CACHE STRING "") -set(VERSION_PATCH 0 CACHE STRING "") -set(VERSION_GITHASH 13f0dff3ed2ed9f87c84098eba1f38b0f04c6da4 CACHE STRING "") -set(VERSION_DESCRIBE v18.7.0-testing CACHE STRING "") -set(VERSION_STRING 18.7.0 CACHE STRING "") +set(VERSION_PATCH 1 CACHE STRING "") +set(VERSION_GITHASH d799ba683dd992d45113106ab9d57d7fdea05d99 CACHE STRING "") +set(VERSION_DESCRIBE v18.7.1-testing CACHE STRING "") +set(VERSION_STRING 18.7.1 CACHE STRING "") # end of autochange set(VERSION_EXTRA "" CACHE STRING "") diff --git a/debian/changelog b/debian/changelog index 9f4737dd00b..d5ac568a9b6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (18.7.0) unstable; urgency=low +clickhouse (18.7.1) unstable; urgency=low * Modified source code - -- Wed, 01 Aug 2018 16:20:46 +0300 + -- Thu, 02 Aug 2018 01:24:01 +0300 From e271c782b360517a12e6d4a094239af4decfb860 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 2 Aug 2018 01:28:25 +0300 Subject: [PATCH 15/20] Added changelog for 18.6.0 [#CLICKHOUSE-2] --- CHANGELOG.draft.md | 5 ----- CHANGELOG_RU.md | 12 ++++++++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.draft.md b/CHANGELOG.draft.md index 3ec4386ac76..93c681b0336 100644 --- a/CHANGELOG.draft.md +++ b/CHANGELOG.draft.md @@ -1,6 +1 @@ ## RU - -### Новые возможности: -* Добавлена поддержка ON выражений для JOIN ON синтаксиса: -`JOIN ON Expr([table.]column, ...) = Expr([table.]column, ...) [AND Expr([table.]column, ...) = Expr([table.]column, ...) ...]` -Выражение должно представлять из себя цепочку равенств, объединенных оператором AND. Каждая часть равенства может являться произвольным выражением над столбцами одной из таблиц. Поддержана возможность использования fully qualified имен столбцов (`table.name`, `database.table.name`, `table_alias.name`, `subquery_alias.name`) для правой таблицы. [#2742](https://github.com/yandex/ClickHouse/pull/2742) diff --git a/CHANGELOG_RU.md b/CHANGELOG_RU.md index e4780f92e4b..494f76379c2 100644 --- a/CHANGELOG_RU.md +++ b/CHANGELOG_RU.md @@ -1,3 +1,15 @@ +## ClickHouse release 18.6.0, 2018-08-02 + +### Новые возможности: +* Добавлена поддержка ON выражений для JOIN ON синтаксиса: +`JOIN ON Expr([table.]column, ...) = Expr([table.]column, ...) [AND Expr([table.]column, ...) = Expr([table.]column, ...) ...]` +Выражение должно представлять из себя цепочку равенств, объединенных оператором AND. Каждая часть равенства может являться произвольным выражением над столбцами одной из таблиц. Поддержана возможность использования fully qualified имен столбцов (`table.name`, `database.table.name`, `table_alias.name`, `subquery_alias.name`) для правой таблицы. [#2742](https://github.com/yandex/ClickHouse/pull/2742) +* Добавлена возможность включить HTTPS для репликации. [#2760](https://github.com/yandex/ClickHouse/pull/2760) + +### Улучшения: +* Сервер передаёт на клиент также patch-компонент своей версии. Данные о patch компоненте версии добавлены в `system.processes` и `query_log`. [#2646](https://github.com/yandex/ClickHouse/pull/2646) + + ## ClickHouse release 18.5.1, 2018-07-31 ### Новые возможности: From 9cd5228df263329c550f3ab48fb748dfd5a888e8 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 2 Aug 2018 03:20:20 +0300 Subject: [PATCH 16/20] Using jemalloc instead of tcmalloc. (#2773) * Added jemalloc as submodule [#CLICKHOUSE-2] * Imported jemalloc-cmake [#CLICKHOUSE-2] * Added README for jemalloc-cmake [#CLICKHOUSE-2] * Trying to add jemalloc [#CLICKHOUSE-2] * Trying to add jemalloc [#CLICKHOUSE-2] * Trying to add jemalloc [#CLICKHOUSE-2] * Add support for jemalloc: development [#CLICKHOUSE-2] * Add support for jemalloc: development [#CLICKHOUSE-2] * Add support for jemalloc: development [#CLICKHOUSE-2] * Add support for jemalloc: development [#CLICKHOUSE-2] * Add support for jemalloc: development [#CLICKHOUSE-2] * Add support for jemalloc: development [#CLICKHOUSE-2] * Add support for jemalloc: development [#CLICKHOUSE-2] * Add support for jemalloc: development [#CLICKHOUSE-2] * Add support for jemalloc: development [#CLICKHOUSE-2] * Add support for jemalloc: development [#CLICKHOUSE-2] * Add support for jemalloc: development [#CLICKHOUSE-2] * Add support for jemalloc: development [#CLICKHOUSE-2] * Add support for jemalloc: development [#CLICKHOUSE-2] * Add support for jemalloc: development [#CLICKHOUSE-2] --- .gitmodules | 3 + contrib/CMakeLists.txt | 4 + contrib/jemalloc | 1 + contrib/jemalloc-cmake/CMakeLists.txt | 52 +++ contrib/jemalloc-cmake/README | 1 + .../include/jemalloc/jemalloc.h | 16 + .../include/jemalloc/jemalloc_rename.h | 29 ++ .../include_linux_x86_64/README | 7 + .../internal/jemalloc_internal_defs.h | 372 ++++++++++++++++++ .../jemalloc/internal/jemalloc_preamble.h | 194 +++++++++ .../jemalloc/jemalloc_defs.h | 43 ++ .../jemalloc/jemalloc_macros.h | 122 ++++++ .../jemalloc/jemalloc_protos.h | 66 ++++ .../jemalloc/jemalloc_typedefs.h | 77 ++++ dbms/src/Interpreters/AsynchronousMetrics.cpp | 34 ++ libs/libcommon/CMakeLists.txt | 12 +- libs/libcommon/cmake/find_gperftools.cmake | 7 +- libs/libcommon/cmake/find_jemalloc.cmake | 24 +- utils/travis/normal.sh | 2 +- utils/travis/pbuilder.sh | 2 +- 20 files changed, 1044 insertions(+), 24 deletions(-) create mode 160000 contrib/jemalloc create mode 100644 contrib/jemalloc-cmake/CMakeLists.txt create mode 100644 contrib/jemalloc-cmake/README create mode 100644 contrib/jemalloc-cmake/include/jemalloc/jemalloc.h create mode 100644 contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h create mode 100644 contrib/jemalloc-cmake/include_linux_x86_64/README create mode 100644 contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h create mode 100644 contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_preamble.h create mode 100644 contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_defs.h create mode 100644 contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_macros.h create mode 100644 contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_protos.h create mode 100644 contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_typedefs.h diff --git a/.gitmodules b/.gitmodules index 1f392b73c83..8c70ad95789 100644 --- a/.gitmodules +++ b/.gitmodules @@ -37,3 +37,6 @@ [submodule "contrib/llvm"] path = contrib/llvm url = https://github.com/ClickHouse-Extras/llvm +[submodule "contrib/jemalloc"] + path = contrib/jemalloc + url = https://github.com/jemalloc/jemalloc.git diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 960a52e2ee4..07db4b9a7d3 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -85,6 +85,10 @@ if (ENABLE_TCMALLOC AND USE_INTERNAL_GPERFTOOLS_LIBRARY) add_subdirectory (libtcmalloc) endif () +if (ENABLE_JEMALLOC AND USE_INTERNAL_JEMALLOC_LIBRARY) + add_subdirectory (jemalloc-cmake) +endif () + if (USE_INTERNAL_CPUID_LIBRARY) add_subdirectory (libcpuid) endif () diff --git a/contrib/jemalloc b/contrib/jemalloc new file mode 160000 index 00000000000..41b7372eade --- /dev/null +++ b/contrib/jemalloc @@ -0,0 +1 @@ +Subproject commit 41b7372eadee941b9164751b8d4963f915d3ceae diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt new file mode 100644 index 00000000000..65fda9cd0ce --- /dev/null +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -0,0 +1,52 @@ +set(JEMALLOC_SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/jemalloc) + +set(SRCS +${JEMALLOC_SOURCE_DIR}/src/arena.c +${JEMALLOC_SOURCE_DIR}/src/background_thread.c +${JEMALLOC_SOURCE_DIR}/src/base.c +${JEMALLOC_SOURCE_DIR}/src/bin.c +${JEMALLOC_SOURCE_DIR}/src/bitmap.c +${JEMALLOC_SOURCE_DIR}/src/ckh.c +${JEMALLOC_SOURCE_DIR}/src/ctl.c +${JEMALLOC_SOURCE_DIR}/src/div.c +${JEMALLOC_SOURCE_DIR}/src/extent.c +${JEMALLOC_SOURCE_DIR}/src/extent_dss.c +${JEMALLOC_SOURCE_DIR}/src/extent_mmap.c +${JEMALLOC_SOURCE_DIR}/src/hash.c +${JEMALLOC_SOURCE_DIR}/src/hook.c +${JEMALLOC_SOURCE_DIR}/src/jemalloc.c +${JEMALLOC_SOURCE_DIR}/src/jemalloc_cpp.cpp +${JEMALLOC_SOURCE_DIR}/src/large.c +${JEMALLOC_SOURCE_DIR}/src/log.c +${JEMALLOC_SOURCE_DIR}/src/malloc_io.c +${JEMALLOC_SOURCE_DIR}/src/mutex.c +${JEMALLOC_SOURCE_DIR}/src/mutex_pool.c +${JEMALLOC_SOURCE_DIR}/src/nstime.c +${JEMALLOC_SOURCE_DIR}/src/pages.c +${JEMALLOC_SOURCE_DIR}/src/prng.c +${JEMALLOC_SOURCE_DIR}/src/prof.c +${JEMALLOC_SOURCE_DIR}/src/rtree.c +${JEMALLOC_SOURCE_DIR}/src/sc.c +${JEMALLOC_SOURCE_DIR}/src/stats.c +${JEMALLOC_SOURCE_DIR}/src/sz.c +${JEMALLOC_SOURCE_DIR}/src/tcache.c +${JEMALLOC_SOURCE_DIR}/src/test_hooks.c +${JEMALLOC_SOURCE_DIR}/src/ticker.c +${JEMALLOC_SOURCE_DIR}/src/tsd.c +${JEMALLOC_SOURCE_DIR}/src/witness.c +) + +if(CMAKE_SYSTEM_NAME MATCHES "Darwin") + list(APPEND SRCS src/zone.c) +endif() + +add_library(jemalloc STATIC ${SRCS}) + +target_include_directories(jemalloc PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/include_linux_x86_64) # jemalloc.h + +target_include_directories(jemalloc PRIVATE + ${JEMALLOC_SOURCE_DIR}/include) + +target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE) diff --git a/contrib/jemalloc-cmake/README b/contrib/jemalloc-cmake/README new file mode 100644 index 00000000000..0af9c4f0e45 --- /dev/null +++ b/contrib/jemalloc-cmake/README @@ -0,0 +1 @@ +It allows to integrate JEMalloc into CMake project. diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc.h new file mode 100644 index 00000000000..d06243c5239 --- /dev/null +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc.h @@ -0,0 +1,16 @@ +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include + +#ifdef __cplusplus +} +#endif + diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h new file mode 100644 index 00000000000..a2ea2dd3533 --- /dev/null +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h @@ -0,0 +1,29 @@ +/* + * Name mangling for public symbols is controlled by --with-mangling and + * --with-jemalloc-prefix. With default settings the je_ prefix is stripped by + * these macro definitions. + */ +#ifndef JEMALLOC_NO_RENAME +# define je_aligned_alloc aligned_alloc +# define je_calloc calloc +# define je_dallocx dallocx +# define je_free free +# define je_mallctl mallctl +# define je_mallctlbymib mallctlbymib +# define je_mallctlnametomib mallctlnametomib +# define je_malloc malloc +# define je_malloc_conf malloc_conf +# define je_malloc_message malloc_message +# define je_malloc_stats_print malloc_stats_print +# define je_malloc_usable_size malloc_usable_size +# define je_mallocx mallocx +# define je_nallocx nallocx +# define je_posix_memalign posix_memalign +# define je_rallocx rallocx +# define je_realloc realloc +# define je_sallocx sallocx +# define je_sdallocx sdallocx +# define je_xallocx xallocx +# define je_memalign memalign +# define je_valloc valloc +#endif diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/README b/contrib/jemalloc-cmake/include_linux_x86_64/README new file mode 100644 index 00000000000..bf7663bda8d --- /dev/null +++ b/contrib/jemalloc-cmake/include_linux_x86_64/README @@ -0,0 +1,7 @@ +Here are pre-generated files from jemalloc on Linux x86_64. +You can obtain these files by running ./autogen.sh inside jemalloc source directory. + +Added #define GNU_SOURCE +Added JEMALLOC_OVERRIDE___POSIX_MEMALIGN because why not. +Removed JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF because it's non standard. +Removed JEMALLOC_PURGE_MADVISE_FREE because it's available only from Linux 4.5. diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h new file mode 100644 index 00000000000..ab949914ecb --- /dev/null +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h @@ -0,0 +1,372 @@ +/* include/jemalloc/internal/jemalloc_internal_defs.h. Generated from jemalloc_internal_defs.h.in by configure. */ +#ifndef JEMALLOC_INTERNAL_DEFS_H_ +#define JEMALLOC_INTERNAL_DEFS_H_ + +#ifndef _GNU_SOURCE + #define _GNU_SOURCE +#endif + +/* + * If JEMALLOC_PREFIX is defined via --with-jemalloc-prefix, it will cause all + * public APIs to be prefixed. This makes it possible, with some care, to use + * multiple allocators simultaneously. + */ +/* #undef JEMALLOC_PREFIX */ +/* #undef JEMALLOC_CPREFIX */ + +/* + * Define overrides for non-standard allocator-related functions if they are + * present on the system. + */ +#define JEMALLOC_OVERRIDE___LIBC_CALLOC +#define JEMALLOC_OVERRIDE___LIBC_FREE +#define JEMALLOC_OVERRIDE___LIBC_MALLOC +#define JEMALLOC_OVERRIDE___LIBC_MEMALIGN +#define JEMALLOC_OVERRIDE___LIBC_REALLOC +#define JEMALLOC_OVERRIDE___LIBC_VALLOC +#define JEMALLOC_OVERRIDE___POSIX_MEMALIGN + +/* + * JEMALLOC_PRIVATE_NAMESPACE is used as a prefix for all library-private APIs. + * For shared libraries, symbol visibility mechanisms prevent these symbols + * from being exported, but for static libraries, naming collisions are a real + * possibility. + */ +#define JEMALLOC_PRIVATE_NAMESPACE je_ + +/* + * Hyper-threaded CPUs may need a special instruction inside spin loops in + * order to yield to another virtual CPU. + */ +#define CPU_SPINWAIT __asm__ volatile("pause") +/* 1 if CPU_SPINWAIT is defined, 0 otherwise. */ +#define HAVE_CPU_SPINWAIT 1 + +/* + * Number of significant bits in virtual addresses. This may be less than the + * total number of bits in a pointer, e.g. on x64, for which the uppermost 16 + * bits are the same as bit 47. + */ +#define LG_VADDR 48 + +/* Defined if C11 atomics are available. */ +#define JEMALLOC_C11_ATOMICS 1 + +/* Defined if GCC __atomic atomics are available. */ +#define JEMALLOC_GCC_ATOMIC_ATOMICS 1 + +/* Defined if GCC __sync atomics are available. */ +#define JEMALLOC_GCC_SYNC_ATOMICS 1 + +/* + * Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and + * __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite + * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the + * functions are defined in libgcc instead of being inlines). + */ +/* #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4 */ + +/* + * Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and + * __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite + * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the + * functions are defined in libgcc instead of being inlines). + */ +/* #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8 */ + +/* + * Defined if __builtin_clz() and __builtin_clzl() are available. + */ +#define JEMALLOC_HAVE_BUILTIN_CLZ + +/* + * Defined if os_unfair_lock_*() functions are available, as provided by Darwin. + */ +/* #undef JEMALLOC_OS_UNFAIR_LOCK */ + +/* + * Defined if OSSpin*() functions are available, as provided by Darwin, and + * documented in the spinlock(3) manual page. + */ +/* #undef JEMALLOC_OSSPIN */ + +/* Defined if syscall(2) is usable. */ +#define JEMALLOC_USE_SYSCALL + +/* + * Defined if secure_getenv(3) is available. + */ +#define JEMALLOC_HAVE_SECURE_GETENV + +/* + * Defined if issetugid(2) is available. + */ +/* #undef JEMALLOC_HAVE_ISSETUGID */ + +/* Defined if pthread_atfork(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_ATFORK + +/* Defined if pthread_setname_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_SETNAME_NP + +/* + * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE 1 + +/* + * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_MONOTONIC 1 + +/* + * Defined if mach_absolute_time() is available. + */ +/* #undef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME */ + +/* + * Defined if _malloc_thread_cleanup() exists. At least in the case of + * FreeBSD, pthread_key_create() allocates, which if used during malloc + * bootstrapping will cause recursion into the pthreads library. Therefore, if + * _malloc_thread_cleanup() exists, use it as the basis for thread cleanup in + * malloc_tsd. + */ +/* #undef JEMALLOC_MALLOC_THREAD_CLEANUP */ + +/* + * Defined if threaded initialization is known to be safe on this platform. + * Among other things, it must be possible to initialize a mutex without + * triggering allocation in order for threaded allocation to be safe. + */ +#define JEMALLOC_THREADED_INIT + +/* + * Defined if the pthreads implementation defines + * _pthread_mutex_init_calloc_cb(), in which case the function is used in order + * to avoid recursive allocation during mutex initialization. + */ +/* #undef JEMALLOC_MUTEX_INIT_CB */ + +/* Non-empty if the tls_model attribute is supported. */ +#define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) + +/* + * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables + * inline functions. + */ +/* #undef JEMALLOC_DEBUG */ + +/* JEMALLOC_STATS enables statistics calculation. */ +#define JEMALLOC_STATS + +/* JEMALLOC_PROF enables allocation profiling. */ +/* #undef JEMALLOC_PROF */ + +/* Use libunwind for profile backtracing if defined. */ +/* #undef JEMALLOC_PROF_LIBUNWIND */ + +/* Use libgcc for profile backtracing if defined. */ +/* #undef JEMALLOC_PROF_LIBGCC */ + +/* Use gcc intrinsics for profile backtracing if defined. */ +/* #undef JEMALLOC_PROF_GCC */ + +/* + * JEMALLOC_DSS enables use of sbrk(2) to allocate extents from the data storage + * segment (DSS). + */ +#define JEMALLOC_DSS + +/* Support memory filling (junk/zero). */ +#define JEMALLOC_FILL + +/* Support utrace(2)-based tracing. */ +/* #undef JEMALLOC_UTRACE */ + +/* Support optional abort() on OOM. */ +/* #undef JEMALLOC_XMALLOC */ + +/* Support lazy locking (avoid locking unless a second thread is launched). */ +/* #undef JEMALLOC_LAZY_LOCK */ + +/* + * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size + * classes). + */ +/* #undef LG_QUANTUM */ + +/* One page is 2^LG_PAGE bytes. */ +#define LG_PAGE 12 + +/* + * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the + * system does not explicitly support huge pages; system calls that require + * explicit huge page support are separately configured. + */ +#define LG_HUGEPAGE 21 + +/* + * If defined, adjacent virtual memory mappings with identical attributes + * automatically coalesce, and they fragment when changes are made to subranges. + * This is the normal order of things for mmap()/munmap(), but on Windows + * VirtualAlloc()/VirtualFree() operations must be precisely matched, i.e. + * mappings do *not* coalesce/fragment. + */ +#define JEMALLOC_MAPS_COALESCE + +/* + * If defined, retain memory for later reuse by default rather than using e.g. + * munmap() to unmap freed extents. This is enabled on 64-bit Linux because + * common sequences of mmap()/munmap() calls will cause virtual memory map + * holes. + */ +#define JEMALLOC_RETAIN + +/* TLS is used to map arenas and magazine caches to threads. */ +#define JEMALLOC_TLS + +/* + * Used to mark unreachable code to quiet "end of non-void" compiler warnings. + * Don't use this directly; instead use unreachable() from util.h + */ +#define JEMALLOC_INTERNAL_UNREACHABLE __builtin_unreachable + +/* + * ffs*() functions to use for bitmapping. Don't use these directly; instead, + * use ffs_*() from util.h. + */ +#define JEMALLOC_INTERNAL_FFSLL __builtin_ffsll +#define JEMALLOC_INTERNAL_FFSL __builtin_ffsl +#define JEMALLOC_INTERNAL_FFS __builtin_ffs + +/* + * If defined, explicitly attempt to more uniformly distribute large allocation + * pointer alignments across all cache indices. + */ +#define JEMALLOC_CACHE_OBLIVIOUS + +/* + * If defined, enable logging facilities. We make this a configure option to + * avoid taking extra branches everywhere. + */ +/* #undef JEMALLOC_LOG */ + +/* + * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings. + */ +/* #undef JEMALLOC_ZONE */ + +/* + * Methods for determining whether the OS overcommits. + * JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY: Linux's + * /proc/sys/vm.overcommit_memory file. + * JEMALLOC_SYSCTL_VM_OVERCOMMIT: FreeBSD's vm.overcommit sysctl. + */ +/* #undef JEMALLOC_SYSCTL_VM_OVERCOMMIT */ +#define JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY + +/* Defined if madvise(2) is available. */ +#define JEMALLOC_HAVE_MADVISE + +/* + * Defined if transparent huge pages are supported via the MADV_[NO]HUGEPAGE + * arguments to madvise(2). + */ +#define JEMALLOC_HAVE_MADVISE_HUGE + +/* + * Methods for purging unused pages differ between operating systems. + * + * madvise(..., MADV_FREE) : This marks pages as being unused, such that they + * will be discarded rather than swapped out. + * madvise(..., MADV_DONTNEED) : If JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS is + * defined, this immediately discards pages, + * such that new pages will be demand-zeroed if + * the address region is later touched; + * otherwise this behaves similarly to + * MADV_FREE, though typically with higher + * system overhead. + */ +//#define JEMALLOC_PURGE_MADVISE_FREE +#define JEMALLOC_PURGE_MADVISE_DONTNEED +#define JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS + +/* Defined if madvise(2) is available but MADV_FREE is not (x86 Linux only). */ +/* #undef JEMALLOC_DEFINE_MADVISE_FREE */ + +/* + * Defined if MADV_DO[NT]DUMP is supported as an argument to madvise. + */ +#define JEMALLOC_MADVISE_DONTDUMP + +/* + * Defined if transparent huge pages (THPs) are supported via the + * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. + */ +/* #undef JEMALLOC_THP */ + +/* Define if operating system has alloca.h header. */ +#define JEMALLOC_HAS_ALLOCA_H 1 + +/* C99 restrict keyword supported. */ +#define JEMALLOC_HAS_RESTRICT 1 + +/* For use by hash code. */ +/* #undef JEMALLOC_BIG_ENDIAN */ + +/* sizeof(int) == 2^LG_SIZEOF_INT. */ +#define LG_SIZEOF_INT 2 + +/* sizeof(long) == 2^LG_SIZEOF_LONG. */ +#define LG_SIZEOF_LONG 3 + +/* sizeof(long long) == 2^LG_SIZEOF_LONG_LONG. */ +#define LG_SIZEOF_LONG_LONG 3 + +/* sizeof(intmax_t) == 2^LG_SIZEOF_INTMAX_T. */ +#define LG_SIZEOF_INTMAX_T 3 + +/* glibc malloc hooks (__malloc_hook, __realloc_hook, __free_hook). */ +#define JEMALLOC_GLIBC_MALLOC_HOOK + +/* glibc memalign hook. */ +#define JEMALLOC_GLIBC_MEMALIGN_HOOK + +/* pthread support */ +#define JEMALLOC_HAVE_PTHREAD + +/* dlsym() support */ +#define JEMALLOC_HAVE_DLSYM + +/* Adaptive mutex support in pthreads. */ +#define JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP + +/* GNU specific sched_getcpu support */ +#define JEMALLOC_HAVE_SCHED_GETCPU + +/* GNU specific sched_setaffinity support */ +#define JEMALLOC_HAVE_SCHED_SETAFFINITY + +/* + * If defined, all the features necessary for background threads are present. + */ +#define JEMALLOC_BACKGROUND_THREAD 1 + +/* + * If defined, jemalloc symbols are not exported (doesn't work when + * JEMALLOC_PREFIX is not defined). + */ +/* #undef JEMALLOC_EXPORT */ + +/* config.malloc_conf options string. */ +#define JEMALLOC_CONFIG_MALLOC_CONF "" + +/* If defined, jemalloc takes the malloc/free/etc. symbol names. */ +#define JEMALLOC_IS_MALLOC 1 + +/* + * Defined if strerror_r returns char * if _GNU_SOURCE is defined. + */ +#define JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE + +#endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_preamble.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_preamble.h new file mode 100644 index 00000000000..c150785fb4a --- /dev/null +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_preamble.h @@ -0,0 +1,194 @@ +#ifndef JEMALLOC_PREAMBLE_H +#define JEMALLOC_PREAMBLE_H + +#include "jemalloc_internal_defs.h" +#include "jemalloc/internal/jemalloc_internal_decls.h" + +#ifdef JEMALLOC_UTRACE +#include +#endif + +#define JEMALLOC_NO_DEMANGLE +#ifdef JEMALLOC_JET +# undef JEMALLOC_IS_MALLOC +# define JEMALLOC_N(n) jet_##n +# include "jemalloc/internal/public_namespace.h" +# define JEMALLOC_NO_RENAME +# include "jemalloc/jemalloc.h" +# undef JEMALLOC_NO_RENAME +#else +# define JEMALLOC_N(n) je_##n +# include "jemalloc/jemalloc.h" +#endif + +#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN)) +#include +#endif + +#ifdef JEMALLOC_ZONE +#include +#include +#include +#endif + +#include "jemalloc/internal/jemalloc_internal_macros.h" + +/* + * Note that the ordering matters here; the hook itself is name-mangled. We + * want the inclusion of hooks to happen early, so that we hook as much as + * possible. + */ +#ifndef JEMALLOC_NO_PRIVATE_NAMESPACE +# ifndef JEMALLOC_JET +# include "jemalloc/internal/private_namespace.h" +# else +# include "jemalloc/internal/private_namespace_jet.h" +# endif +#endif +#include "jemalloc/internal/test_hooks.h" + +#ifdef JEMALLOC_DEFINE_MADVISE_FREE +# define JEMALLOC_MADV_FREE 8 +#endif + +static const bool config_debug = +#ifdef JEMALLOC_DEBUG + true +#else + false +#endif + ; +static const bool have_dss = +#ifdef JEMALLOC_DSS + true +#else + false +#endif + ; +static const bool have_madvise_huge = +#ifdef JEMALLOC_HAVE_MADVISE_HUGE + true +#else + false +#endif + ; +static const bool config_fill = +#ifdef JEMALLOC_FILL + true +#else + false +#endif + ; +static const bool config_lazy_lock = +#ifdef JEMALLOC_LAZY_LOCK + true +#else + false +#endif + ; +static const char * const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF; +static const bool config_prof = +#ifdef JEMALLOC_PROF + true +#else + false +#endif + ; +static const bool config_prof_libgcc = +#ifdef JEMALLOC_PROF_LIBGCC + true +#else + false +#endif + ; +static const bool config_prof_libunwind = +#ifdef JEMALLOC_PROF_LIBUNWIND + true +#else + false +#endif + ; +static const bool maps_coalesce = +#ifdef JEMALLOC_MAPS_COALESCE + true +#else + false +#endif + ; +static const bool config_stats = +#ifdef JEMALLOC_STATS + true +#else + false +#endif + ; +static const bool config_tls = +#ifdef JEMALLOC_TLS + true +#else + false +#endif + ; +static const bool config_utrace = +#ifdef JEMALLOC_UTRACE + true +#else + false +#endif + ; +static const bool config_xmalloc = +#ifdef JEMALLOC_XMALLOC + true +#else + false +#endif + ; +static const bool config_cache_oblivious = +#ifdef JEMALLOC_CACHE_OBLIVIOUS + true +#else + false +#endif + ; +/* + * Undocumented, for jemalloc development use only at the moment. See the note + * in jemalloc/internal/log.h. + */ +static const bool config_log = +#ifdef JEMALLOC_LOG + true +#else + false +#endif + ; +#ifdef JEMALLOC_HAVE_SCHED_GETCPU +/* Currently percpu_arena depends on sched_getcpu. */ +#define JEMALLOC_PERCPU_ARENA +#endif +static const bool have_percpu_arena = +#ifdef JEMALLOC_PERCPU_ARENA + true +#else + false +#endif + ; +/* + * Undocumented, and not recommended; the application should take full + * responsibility for tracking provenance. + */ +static const bool force_ivsalloc = +#ifdef JEMALLOC_FORCE_IVSALLOC + true +#else + false +#endif + ; +static const bool have_background_thread = +#ifdef JEMALLOC_BACKGROUND_THREAD + true +#else + false +#endif + ; + +#endif /* JEMALLOC_PREAMBLE_H */ diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_defs.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_defs.h new file mode 100644 index 00000000000..d1389237a77 --- /dev/null +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_defs.h @@ -0,0 +1,43 @@ +/* include/jemalloc/jemalloc_defs.h. Generated from jemalloc_defs.h.in by configure. */ +/* Defined if __attribute__((...)) syntax is supported. */ +#define JEMALLOC_HAVE_ATTR + +/* Defined if alloc_size attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_ALLOC_SIZE + +/* Defined if format(printf, ...) attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_FORMAT_PRINTF + +/* + * Define overrides for non-standard allocator-related functions if they are + * present on the system. + */ +#define JEMALLOC_OVERRIDE_MEMALIGN +#define JEMALLOC_OVERRIDE_VALLOC + +/* + * At least Linux omits the "const" in: + * + * size_t malloc_usable_size(const void *ptr); + * + * Match the operating system's prototype. + */ +#define JEMALLOC_USABLE_SIZE_CONST + +/* + * If defined, specify throw() for the public function prototypes when compiling + * with C++. The only justification for this is to match the prototypes that + * glibc defines. + */ +#define JEMALLOC_USE_CXX_THROW + +#ifdef _MSC_VER +# ifdef _WIN64 +# define LG_SIZEOF_PTR_WIN 3 +# else +# define LG_SIZEOF_PTR_WIN 2 +# endif +#endif + +/* sizeof(void *) == 2^LG_SIZEOF_PTR. */ +#define LG_SIZEOF_PTR 3 diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_macros.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_macros.h new file mode 100644 index 00000000000..7432f1cda53 --- /dev/null +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_macros.h @@ -0,0 +1,122 @@ +#include +#include +#include +#include +#include + +#define JEMALLOC_VERSION "5.1.0-56-g41b7372eadee941b9164751b8d4963f915d3ceae" +#define JEMALLOC_VERSION_MAJOR 5 +#define JEMALLOC_VERSION_MINOR 1 +#define JEMALLOC_VERSION_BUGFIX 0 +#define JEMALLOC_VERSION_NREV 56 +#define JEMALLOC_VERSION_GID "41b7372eadee941b9164751b8d4963f915d3ceae" + +#define MALLOCX_LG_ALIGN(la) ((int)(la)) +#if LG_SIZEOF_PTR == 2 +# define MALLOCX_ALIGN(a) ((int)(ffs((int)(a))-1)) +#else +# define MALLOCX_ALIGN(a) \ + ((int)(((size_t)(a) < (size_t)INT_MAX) ? ffs((int)(a))-1 : \ + ffs((int)(((size_t)(a))>>32))+31)) +#endif +#define MALLOCX_ZERO ((int)0x40) +/* + * Bias tcache index bits so that 0 encodes "automatic tcache management", and 1 + * encodes MALLOCX_TCACHE_NONE. + */ +#define MALLOCX_TCACHE(tc) ((int)(((tc)+2) << 8)) +#define MALLOCX_TCACHE_NONE MALLOCX_TCACHE(-1) +/* + * Bias arena index bits so that 0 encodes "use an automatically chosen arena". + */ +#define MALLOCX_ARENA(a) ((((int)(a))+1) << 20) + +/* + * Use as arena index in "arena..{purge,decay,dss}" and + * "stats.arenas..*" mallctl interfaces to select all arenas. This + * definition is intentionally specified in raw decimal format to support + * cpp-based string concatenation, e.g. + * + * #define STRINGIFY_HELPER(x) #x + * #define STRINGIFY(x) STRINGIFY_HELPER(x) + * + * mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", NULL, NULL, NULL, + * 0); + */ +#define MALLCTL_ARENAS_ALL 4096 +/* + * Use as arena index in "stats.arenas..*" mallctl interfaces to select + * destroyed arenas. + */ +#define MALLCTL_ARENAS_DESTROYED 4097 + +#if defined(__cplusplus) && defined(JEMALLOC_USE_CXX_THROW) +# define JEMALLOC_CXX_THROW throw() +#else +# define JEMALLOC_CXX_THROW +#endif + +#if defined(_MSC_VER) +# define JEMALLOC_ATTR(s) +# define JEMALLOC_ALIGNED(s) __declspec(align(s)) +# define JEMALLOC_ALLOC_SIZE(s) +# define JEMALLOC_ALLOC_SIZE2(s1, s2) +# ifndef JEMALLOC_EXPORT +# ifdef DLLEXPORT +# define JEMALLOC_EXPORT __declspec(dllexport) +# else +# define JEMALLOC_EXPORT __declspec(dllimport) +# endif +# endif +# define JEMALLOC_FORMAT_PRINTF(s, i) +# define JEMALLOC_NOINLINE __declspec(noinline) +# ifdef __cplusplus +# define JEMALLOC_NOTHROW __declspec(nothrow) +# else +# define JEMALLOC_NOTHROW +# endif +# define JEMALLOC_SECTION(s) __declspec(allocate(s)) +# define JEMALLOC_RESTRICT_RETURN __declspec(restrict) +# if _MSC_VER >= 1900 && !defined(__EDG__) +# define JEMALLOC_ALLOCATOR __declspec(allocator) +# else +# define JEMALLOC_ALLOCATOR +# endif +#elif defined(JEMALLOC_HAVE_ATTR) +# define JEMALLOC_ATTR(s) __attribute__((s)) +# define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s)) +# ifdef JEMALLOC_HAVE_ATTR_ALLOC_SIZE +# define JEMALLOC_ALLOC_SIZE(s) JEMALLOC_ATTR(alloc_size(s)) +# define JEMALLOC_ALLOC_SIZE2(s1, s2) JEMALLOC_ATTR(alloc_size(s1, s2)) +# else +# define JEMALLOC_ALLOC_SIZE(s) +# define JEMALLOC_ALLOC_SIZE2(s1, s2) +# endif +# ifndef JEMALLOC_EXPORT +# define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default")) +# endif +# ifdef JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF +# define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(gnu_printf, s, i)) +# elif defined(JEMALLOC_HAVE_ATTR_FORMAT_PRINTF) +# define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(printf, s, i)) +# else +# define JEMALLOC_FORMAT_PRINTF(s, i) +# endif +# define JEMALLOC_NOINLINE JEMALLOC_ATTR(noinline) +# define JEMALLOC_NOTHROW JEMALLOC_ATTR(nothrow) +# define JEMALLOC_SECTION(s) JEMALLOC_ATTR(section(s)) +# define JEMALLOC_RESTRICT_RETURN +# define JEMALLOC_ALLOCATOR +#else +# define JEMALLOC_ATTR(s) +# define JEMALLOC_ALIGNED(s) +# define JEMALLOC_ALLOC_SIZE(s) +# define JEMALLOC_ALLOC_SIZE2(s1, s2) +# define JEMALLOC_EXPORT +# define JEMALLOC_FORMAT_PRINTF(s, i) +# define JEMALLOC_NOINLINE +# define JEMALLOC_NOTHROW +# define JEMALLOC_SECTION(s) +# define JEMALLOC_RESTRICT_RETURN +# define JEMALLOC_ALLOCATOR +#endif diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_protos.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_protos.h new file mode 100644 index 00000000000..ff025e30fa7 --- /dev/null +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_protos.h @@ -0,0 +1,66 @@ +/* + * The je_ prefix on the following public symbol declarations is an artifact + * of namespace management, and should be omitted in application code unless + * JEMALLOC_NO_DEMANGLE is defined (see jemalloc_mangle.h). + */ +extern JEMALLOC_EXPORT const char *je_malloc_conf; +extern JEMALLOC_EXPORT void (*je_malloc_message)(void *cbopaque, + const char *s); + +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_malloc(size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_calloc(size_t num, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2); +JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_posix_memalign(void **memptr, + size_t alignment, size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(nonnull(1)); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_aligned_alloc(size_t alignment, + size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) + JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_realloc(void *ptr, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_free(void *ptr) + JEMALLOC_CXX_THROW; + +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_mallocx(size_t size, int flags) + JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_rallocx(void *ptr, size_t size, + int flags) JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_xallocx(void *ptr, size_t size, + size_t extra, int flags); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_sallocx(const void *ptr, + int flags) JEMALLOC_ATTR(pure); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_dallocx(void *ptr, int flags); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_sdallocx(void *ptr, size_t size, + int flags); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_nallocx(size_t size, int flags) + JEMALLOC_ATTR(pure); + +JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctl(const char *name, + void *oldp, size_t *oldlenp, void *newp, size_t newlen); +JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctlnametomib(const char *name, + size_t *mibp, size_t *miblenp); +JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctlbymib(const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_malloc_stats_print( + void (*write_cb)(void *, const char *), void *je_cbopaque, + const char *opts); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_malloc_usable_size( + JEMALLOC_USABLE_SIZE_CONST void *ptr) JEMALLOC_CXX_THROW; + +#ifdef JEMALLOC_OVERRIDE_MEMALIGN +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_memalign(size_t alignment, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc); +#endif + +#ifdef JEMALLOC_OVERRIDE_VALLOC +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_valloc(size_t size) JEMALLOC_CXX_THROW + JEMALLOC_ATTR(malloc); +#endif diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_typedefs.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_typedefs.h new file mode 100644 index 00000000000..1a58874306e --- /dev/null +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_typedefs.h @@ -0,0 +1,77 @@ +typedef struct extent_hooks_s extent_hooks_t; + +/* + * void * + * extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, + * size_t alignment, bool *zero, bool *commit, unsigned arena_ind); + */ +typedef void *(extent_alloc_t)(extent_hooks_t *, void *, size_t, size_t, bool *, + bool *, unsigned); + +/* + * bool + * extent_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size, + * bool committed, unsigned arena_ind); + */ +typedef bool (extent_dalloc_t)(extent_hooks_t *, void *, size_t, bool, + unsigned); + +/* + * void + * extent_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, + * bool committed, unsigned arena_ind); + */ +typedef void (extent_destroy_t)(extent_hooks_t *, void *, size_t, bool, + unsigned); + +/* + * bool + * extent_commit(extent_hooks_t *extent_hooks, void *addr, size_t size, + * size_t offset, size_t length, unsigned arena_ind); + */ +typedef bool (extent_commit_t)(extent_hooks_t *, void *, size_t, size_t, size_t, + unsigned); + +/* + * bool + * extent_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size, + * size_t offset, size_t length, unsigned arena_ind); + */ +typedef bool (extent_decommit_t)(extent_hooks_t *, void *, size_t, size_t, + size_t, unsigned); + +/* + * bool + * extent_purge(extent_hooks_t *extent_hooks, void *addr, size_t size, + * size_t offset, size_t length, unsigned arena_ind); + */ +typedef bool (extent_purge_t)(extent_hooks_t *, void *, size_t, size_t, size_t, + unsigned); + +/* + * bool + * extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size, + * size_t size_a, size_t size_b, bool committed, unsigned arena_ind); + */ +typedef bool (extent_split_t)(extent_hooks_t *, void *, size_t, size_t, size_t, + bool, unsigned); + +/* + * bool + * extent_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, + * void *addr_b, size_t size_b, bool committed, unsigned arena_ind); + */ +typedef bool (extent_merge_t)(extent_hooks_t *, void *, size_t, void *, size_t, + bool, unsigned); + +struct extent_hooks_s { + extent_alloc_t *alloc; + extent_dalloc_t *dalloc; + extent_destroy_t *destroy; + extent_commit_t *commit; + extent_decommit_t *decommit; + extent_purge_t *purge_lazy; + extent_purge_t *purge_forced; + extent_split_t *split; + extent_merge_t *merge; +}; diff --git a/dbms/src/Interpreters/AsynchronousMetrics.cpp b/dbms/src/Interpreters/AsynchronousMetrics.cpp index 611711b317e..9fa890d6850 100644 --- a/dbms/src/Interpreters/AsynchronousMetrics.cpp +++ b/dbms/src/Interpreters/AsynchronousMetrics.cpp @@ -27,6 +27,10 @@ } malloc_extension_initializer; #endif +#if USE_JEMALLOC + #include +#endif + namespace DB { @@ -229,6 +233,36 @@ void AsynchronousMetrics::update() } #endif +#if USE_JEMALLOC + { + #define FOR_EACH_METRIC(M) \ + M("allocated", size_t) \ + M("active", size_t) \ + M("metadata", size_t) \ + M("metadata_thp", size_t) \ + M("resident", size_t) \ + M("mapped", size_t) \ + M("retained", size_t) \ + M("background_thread.num_threads", size_t) \ + M("background_thread.num_runs", uint64_t) \ + M("background_thread.run_interval", uint64_t) \ + + #define GET_METRIC(NAME, TYPE) \ + do \ + { \ + TYPE value{}; \ + size_t size = sizeof(value); \ + mallctl("stats." NAME, &value, &size, nullptr, 0); \ + set("jemalloc." NAME, value); \ + } while (0); + + FOR_EACH_METRIC(GET_METRIC); + + #undef GET_METRIC + #undef FOR_EACH_METRIC + } +#endif + /// Add more metrics as you wish. } diff --git a/libs/libcommon/CMakeLists.txt b/libs/libcommon/CMakeLists.txt index 8bbd454287f..7ad6067410b 100644 --- a/libs/libcommon/CMakeLists.txt +++ b/libs/libcommon/CMakeLists.txt @@ -57,26 +57,26 @@ add_library (common ${SPLIT_SHARED} ${CONFIG_COMMON} ) -# When testing for memory leaks, dont link tcmalloc or jemalloc. +# When testing for memory leaks with Valgrind, dont link tcmalloc or jemalloc. if (USE_JEMALLOC) - message (STATUS "Link jemalloc : ${JEMALLOC_LIBRARIES}") + message (STATUS "Link jemalloc: ${JEMALLOC_LIBRARIES}") set (MALLOC_LIBRARIES ${JEMALLOC_LIBRARIES}) -elseif (USE_TCMALLOC) +elseif (USE_TCMALLOC) if (DEBUG_TCMALLOC AND NOT GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG) - message (WARNING "Requested DEBUG_TCMALLOC but debug lib not found. try install libgoogle-perftools-dev") + message (FATAL_ERROR "Requested DEBUG_TCMALLOC but debug library is not found. You should install Google Perftools. Example: sudo apt-get install libgoogle-perftools-dev") endif () if (DEBUG_TCMALLOC AND GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG) message (STATUS "Link libtcmalloc_minimal_debug for testing: ${GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG}") set (MALLOC_LIBRARIES ${GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG}) else () - message (STATUS "Link libtcmalloc_minimal : ${GPERFTOOLS_TCMALLOC_MINIMAL}") + message (STATUS "Link libtcmalloc_minimal: ${GPERFTOOLS_TCMALLOC_MINIMAL}") set (MALLOC_LIBRARIES ${GPERFTOOLS_TCMALLOC_MINIMAL}) endif () else () - message (STATUS "Disabling libtcmalloc for valgrind better analysis") + message (WARNING "Non default allocator is disabled. This is not recommended for production Linux builds.") endif () if (GLIBC_COMPATIBILITY) diff --git a/libs/libcommon/cmake/find_gperftools.cmake b/libs/libcommon/cmake/find_gperftools.cmake index a311f2ba74e..a3eba003800 100644 --- a/libs/libcommon/cmake/find_gperftools.cmake +++ b/libs/libcommon/cmake/find_gperftools.cmake @@ -4,12 +4,7 @@ else () option (USE_INTERNAL_GPERFTOOLS_LIBRARY "Set to FALSE to use system gperftools (tcmalloc) library instead of bundled" ${NOT_UNBUNDLED}) endif () -if (ARCH_FREEBSD OR MSVC) - option (ENABLE_TCMALLOC "Set to TRUE to enable tcmalloc" OFF) -else () - option (ENABLE_TCMALLOC "Set to TRUE to enable tcmalloc" ON) -endif () - +option (ENABLE_TCMALLOC "Set to TRUE to enable tcmalloc" OFF) option (DEBUG_TCMALLOC "Set to TRUE to use debug version of libtcmalloc" OFF) if (ENABLE_TCMALLOC) diff --git a/libs/libcommon/cmake/find_jemalloc.cmake b/libs/libcommon/cmake/find_jemalloc.cmake index f8fb20fe9b6..33c1162b6bf 100644 --- a/libs/libcommon/cmake/find_jemalloc.cmake +++ b/libs/libcommon/cmake/find_jemalloc.cmake @@ -1,14 +1,18 @@ -option (ENABLE_JEMALLOC "Set to TRUE to use jemalloc instead of tcmalloc" OFF) +option (ENABLE_JEMALLOC "Set to TRUE to use jemalloc" ON) +option (USE_INTERNAL_JEMALLOC_LIBRARY "Set to FALSE to use system jemalloc library instead of bundled" ${NOT_UNBUNDLED}) if (ENABLE_JEMALLOC) - find_package (JeMalloc) - - if (JEMALLOC_INCLUDE_DIR AND JEMALLOC_LIBRARIES) - set (USE_JEMALLOC 1) - if (USE_TCMALLOC) - message (WARNING "Disabling tcmalloc") - set (USE_TCMALLOC 0) - endif () + if (USE_INTERNAL_JEMALLOC_LIBRARY) + set (JEMALLOC_LIBRARIES "jemalloc") + else () + find_package (JeMalloc) endif () - message (STATUS "Using jemalloc=${USE_JEMALLOC}: ${JEMALLOC_INCLUDE_DIR} : ${JEMALLOC_LIBRARIES}") + + if (JEMALLOC_LIBRARIES) + set (USE_JEMALLOC 1) + else () + message (FATAL_ERROR "ENABLE_JEMALLOC is set to true, but library was not found") + endif () + + message (STATUS "Using jemalloc=${USE_JEMALLOC}: ${JEMALLOC_LIBRARIES}") endif () diff --git a/utils/travis/normal.sh b/utils/travis/normal.sh index b361744a3ec..45190799a6b 100755 --- a/utils/travis/normal.sh +++ b/utils/travis/normal.sh @@ -32,7 +32,7 @@ cmake $CUR_DIR/../.. -DCMAKE_CXX_COMPILER=`which $DEB_CXX $CXX` -DCMAKE_C_COMPIL `# Use all possible contrib libs from system` \ -DUNBUNDLED=1 \ `# Disable all features` \ - -DENABLE_CAPNP=0 -DENABLE_RDKAFKA=0 -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_TCMALLOC=0 -DENABLE_UNWIND=0 -DENABLE_MYSQL=0 -DUSE_INTERNAL_LLVM_LIBRARY=0 $CMAKE_FLAGS \ + -DENABLE_CAPNP=0 -DENABLE_RDKAFKA=0 -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_TCMALLOC=0 -DENABLE_JEMALLOC=0 -DENABLE_UNWIND=0 -DENABLE_MYSQL=0 -DUSE_INTERNAL_LLVM_LIBRARY=0 $CMAKE_FLAGS \ && ninja clickhouse-bundle \ `# Skip tests:` \ `# 00281 requires internal compiler` \ diff --git a/utils/travis/pbuilder.sh b/utils/travis/pbuilder.sh index 796dcf3e8d9..17cd6629528 100755 --- a/utils/travis/pbuilder.sh +++ b/utils/travis/pbuilder.sh @@ -27,7 +27,7 @@ env TEST_RUN=${TEST_RUN=1} \ CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Debug -DUNBUNDLED=1 -DENABLE_UNWIND=0 -DENABLE_MYSQL=0 -DENABLE_CAPNP=0 -DENABLE_RDKAFKA=0 -DUSE_INTERNAL_LLVM_LIBRARY=0 -DCMAKE_C_FLAGS_ADD='-O0 -g0' -DCMAKE_CXX_FLAGS_ADD='-O0 -g0' $CMAKE_FLAGS" \ `# Use all possible contrib libs from system` \ `# psmisc - killall` \ - EXTRAPACKAGES="psmisc clang-5.0 lld-5.0 liblld-5.0-dev libclang-5.0-dev liblld-5.0 libc++abi-dev libc++-dev libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libboost-thread-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libsparsehash-dev librdkafka-dev libpoco-dev libsparsehash-dev libgoogle-perftools-dev libzstd-dev libre2-dev $EXTRAPACKAGES" \ + EXTRAPACKAGES="psmisc clang-5.0 lld-5.0 liblld-5.0-dev libclang-5.0-dev liblld-5.0 libc++abi-dev libc++-dev libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libboost-thread-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libsparsehash-dev librdkafka-dev libpoco-dev libsparsehash-dev libgoogle-perftools-dev libzstd-dev libre2-dev libjemalloc-dev $EXTRAPACKAGES" \ `# Travis trusty cant unpack bionic: E: debootstrap failed, TODO: check again, can be fixed` \ DIST=${DIST=artful} \ $CUR_DIR/../../release $RELEASE_OPT From 25cc323d1d7c3d5d116e5db1e584b25bd9c91d8d Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 2 Aug 2018 03:49:31 +0300 Subject: [PATCH 17/20] Using mariadb-connector-c from submodule. (#2785) * Using internal mariadb-client library [#CLICKHOUSE-2] * Using internal mariadb-client library [#CLICKHOUSE-2] * Using internal mariadb-client library [#CLICKHOUSE-2] * Using internal mariadb-client library [#CLICKHOUSE-2] * Using internal mariadb-client library [#CLICKHOUSE-2] --- .gitmodules | 3 + ci/install-libraries.sh | 1 - ci/install-os-packages.sh | 10 - contrib/CMakeLists.txt | 6 + contrib/mariadb-connector-c | 1 + .../mariadb-connector-c-cmake/CMakeLists.txt | 64 +++ .../common/include/mysql/mysql.h | 1 + .../common/include/mysql/mysqld_error.h | 1 + .../linux_x86_64/include/config.h | 269 ++++++++++ .../linux_x86_64/include/ma_config.h | 269 ++++++++++ .../linux_x86_64/include/mariadb_version.h | 36 ++ .../libmariadb/ma_client_plugin.c | 499 ++++++++++++++++++ docs/en/development/build.md | 2 +- libs/libmysqlxx/CMakeLists.txt | 38 +- libs/libmysqlxx/cmake/find_mysqlclient.cmake | 73 +-- 15 files changed, 1214 insertions(+), 59 deletions(-) create mode 160000 contrib/mariadb-connector-c create mode 100644 contrib/mariadb-connector-c-cmake/CMakeLists.txt create mode 100644 contrib/mariadb-connector-c-cmake/common/include/mysql/mysql.h create mode 100644 contrib/mariadb-connector-c-cmake/common/include/mysql/mysqld_error.h create mode 100644 contrib/mariadb-connector-c-cmake/linux_x86_64/include/config.h create mode 100644 contrib/mariadb-connector-c-cmake/linux_x86_64/include/ma_config.h create mode 100644 contrib/mariadb-connector-c-cmake/linux_x86_64/include/mariadb_version.h create mode 100644 contrib/mariadb-connector-c-cmake/linux_x86_64/libmariadb/ma_client_plugin.c diff --git a/.gitmodules b/.gitmodules index 8c70ad95789..6c71fec419a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -37,6 +37,9 @@ [submodule "contrib/llvm"] path = contrib/llvm url = https://github.com/ClickHouse-Extras/llvm +[submodule "contrib/mariadb-connector-c"] + path = contrib/mariadb-connector-c + url = https://github.com/MariaDB/mariadb-connector-c.git [submodule "contrib/jemalloc"] path = contrib/jemalloc url = https://github.com/jemalloc/jemalloc.git diff --git a/ci/install-libraries.sh b/ci/install-libraries.sh index 4868221b342..0fd77d51915 100755 --- a/ci/install-libraries.sh +++ b/ci/install-libraries.sh @@ -6,7 +6,6 @@ source default-config ./install-os-packages.sh libssl-dev ./install-os-packages.sh libicu-dev ./install-os-packages.sh libreadline-dev -./install-os-packages.sh libmariadbclient-dev ./install-os-packages.sh libunixodbc-dev if [[ "$ENABLE_EMBEDDED_COMPILER" == 1 && "$USE_LLVM_LIBRARIES_FROM_SYSTEM" == 1 ]]; then diff --git a/ci/install-os-packages.sh b/ci/install-os-packages.sh index 4aae6268aa1..e5055c660eb 100755 --- a/ci/install-os-packages.sh +++ b/ci/install-os-packages.sh @@ -55,9 +55,6 @@ case $PACKAGE_MANAGER in libunixodbc-dev) $SUDO apt-get install -y unixodbc-dev ;; - libmariadbclient-dev) - $SUDO apt-get install -y libmariadbclient-dev - ;; llvm-libs*) $SUDO apt-get install -y ${WHAT/llvm-libs/liblld}-dev ${WHAT/llvm-libs/libclang}-dev ;; @@ -109,10 +106,6 @@ case $PACKAGE_MANAGER in libunixodbc-dev) $SUDO yum install -y unixODBC-devel libtool-ltdl-devel ;; - libmariadbclient-dev) - echo "There is no package with static mysqlclient library"; echo 1; - #$SUDO yum install -y mariadb-connector-c-devel - ;; *) echo "Unknown package"; exit 1; ;; @@ -158,9 +151,6 @@ case $PACKAGE_MANAGER in libunixodbc-dev) $SUDO pkg install -y unixODBC libltdl ;; - libmariadbclient-dev) - $SUDO pkg install -y mariadb102-client - ;; *) echo "Unknown package"; exit 1; ;; diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 07db4b9a7d3..1ee83056110 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -103,6 +103,12 @@ if (USE_INTERNAL_SSL_LIBRARY) target_include_directories(${OPENSSL_SSL_LIBRARY} PUBLIC ${OPENSSL_INCLUDE_DIR}) endif () +if (ENABLE_MYSQL AND USE_INTERNAL_MYSQL_LIBRARY) + add_subdirectory (mariadb-connector-c-cmake) + target_include_directories(mysqlclient PRIVATE BEFORE ${ZLIB_INCLUDE_DIR}) + target_include_directories(mysqlclient PRIVATE BEFORE ${OPENSSL_INCLUDE_DIR}) +endif () + if (USE_INTERNAL_RDKAFKA_LIBRARY) set (RDKAFKA_BUILD_EXAMPLES OFF CACHE INTERNAL "") set (RDKAFKA_BUILD_TESTS OFF CACHE INTERNAL "") diff --git a/contrib/mariadb-connector-c b/contrib/mariadb-connector-c new file mode 160000 index 00000000000..a0fd36cc5a5 --- /dev/null +++ b/contrib/mariadb-connector-c @@ -0,0 +1 @@ +Subproject commit a0fd36cc5a5313414a5a2ebe9322577a29b4782a diff --git a/contrib/mariadb-connector-c-cmake/CMakeLists.txt b/contrib/mariadb-connector-c-cmake/CMakeLists.txt new file mode 100644 index 00000000000..b4210b92123 --- /dev/null +++ b/contrib/mariadb-connector-c-cmake/CMakeLists.txt @@ -0,0 +1,64 @@ +set(MARIADB_CLIENT_SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/mariadb-connector-c) +set(MARIADB_CLIENT_BINARY_DIR ${CMAKE_BINARY_DIR}/contrib/mariadb-connector-c) + +set(SRCS +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/bmove_upp.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/get_password.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_alloc.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_array.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_charset.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_compress.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_context.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_default.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_dtoa.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_errmsg.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_hash.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_init.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_io.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_list.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_ll2str.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_loaddata.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_net.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_password.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_pvio.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/mariadb_async.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/mariadb_charset.c +#${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/mariadb_dyncol.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/mariadb_lib.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/mariadb_stmt.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_sha1.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_stmt_codec.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_string.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_time.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_tls.c +#${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/secure/gnutls.c +#${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/secure/ma_schannel.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/secure/openssl.c +#${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/secure/schannel.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/auth_gssapi_client.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/dialog.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/gssapi_client.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/gssapi_errmsg.c +${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/mariadb_cleartext.c +${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/my_auth.c +${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/old_password.c +${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/sha256_pw.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/sspi_client.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/sspi_errmsg.c +${MARIADB_CLIENT_SOURCE_DIR}/plugins/connection/aurora.c +${MARIADB_CLIENT_SOURCE_DIR}/plugins/connection/replication.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/io/remote_io.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/pvio/pvio_npipe.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/pvio/pvio_shmem.c +${MARIADB_CLIENT_SOURCE_DIR}/plugins/pvio/pvio_socket.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/trace/trace_example.c +${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/libmariadb/ma_client_plugin.c +) + +add_library(mysqlclient STATIC ${SRCS}) + +target_include_directories(mysqlclient PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/include) +target_include_directories(mysqlclient PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/common/include) +target_include_directories(mysqlclient PUBLIC ${MARIADB_CLIENT_SOURCE_DIR}/include) + +target_compile_definitions(mysqlclient PRIVATE -D THREAD -D HAVE_OPENSSL -D HAVE_TLS) diff --git a/contrib/mariadb-connector-c-cmake/common/include/mysql/mysql.h b/contrib/mariadb-connector-c-cmake/common/include/mysql/mysql.h new file mode 100644 index 00000000000..741c7ba03c9 --- /dev/null +++ b/contrib/mariadb-connector-c-cmake/common/include/mysql/mysql.h @@ -0,0 +1 @@ +#include diff --git a/contrib/mariadb-connector-c-cmake/common/include/mysql/mysqld_error.h b/contrib/mariadb-connector-c-cmake/common/include/mysql/mysqld_error.h new file mode 100644 index 00000000000..95d26eef163 --- /dev/null +++ b/contrib/mariadb-connector-c-cmake/common/include/mysql/mysqld_error.h @@ -0,0 +1 @@ +#include diff --git a/contrib/mariadb-connector-c-cmake/linux_x86_64/include/config.h b/contrib/mariadb-connector-c-cmake/linux_x86_64/include/config.h new file mode 100644 index 00000000000..90c42c97df6 --- /dev/null +++ b/contrib/mariadb-connector-c-cmake/linux_x86_64/include/config.h @@ -0,0 +1,269 @@ + +/* + * Include file constants (processed in LibmysqlIncludeFiles.txt 1 + */ +#define HAVE_ALLOCA_H 1 +/* #undef HAVE_BIGENDIAN */ +#define HAVE_SETLOCALE 1 +#define HAVE_NL_LANGINFO 1 +#define HAVE_ARPA_INET_H 1 +#define HAVE_CRYPT_H 1 +#define HAVE_DIRENT_H 1 +#define HAVE_DLFCN_H 1 +#define HAVE_EXECINFO_H 1 +#define HAVE_FCNTL_H 1 +#define HAVE_FENV_H 1 +#define HAVE_FLOAT_H 1 +/* #undef HAVE_FPU_CONTROL_H */ +#define HAVE_GRP_H 1 +/* #undef HAVE_IEEEFP_H */ +#define HAVE_LIMITS_H 1 +#define HAVE_MALLOC_H 1 +#define HAVE_MEMORY_H 1 +#define HAVE_NETINET_IN_H 1 +#define HAVE_PATHS_H 1 +#define HAVE_PWD_H 1 +#define HAVE_SCHED_H 1 +/* #undef HAVE_SELECT_H */ +#define HAVE_STDDEF_H 1 +#define HAVE_STDINT_H 1 +#define HAVE_STDLIB_H 1 +#define HAVE_STRING_H 1 +#define HAVE_STRINGS_H 1 +/* #undef HAVE_SYNCH_H */ +/* #undef HAVE_SYS_FPU_H */ +#define HAVE_SYS_IOCTL_H 1 +#define HAVE_SYS_IPC_H 1 +#define HAVE_SYS_MMAN_H 1 +#define HAVE_SYS_PRCTL_H 1 +#define HAVE_SYS_SELECT_H 1 +#define HAVE_SYS_SHM_H 1 +#define HAVE_SYS_SOCKET_H 1 +#define HAVE_SYS_STAT_H 1 +/* #undef HAVE_SYS_STREAM_H */ +#define HAVE_SYS_TIMEB_H 1 +#define HAVE_SYS_TYPES_H 1 +#define HAVE_SYS_UN_H 1 +/* #undef HAVE_SYSENT_H */ +#define HAVE_TERMIO_H 1 +#define HAVE_TERMIOS_H 1 +#define HAVE_UNISTD_H 1 +#define HAVE_UTIME_H 1 +#define HAVE_UCONTEXT_H 1 + +/* + * function definitions - processed in LibmysqlFunctions.txt + */ +#define HAVE_ACCESS 1 +/* #undef HAVE_AIOWAIT */ +#define HAVE_ALARM 1 +/* #undef HAVE_ALLOCA */ +#define HAVE_BCMP 1 +/* #undef HAVE_BFILL */ +/* #undef HAVE_BMOVE */ +#define HAVE_BZERO 1 +#define HAVE_CLOCK_GETTIME 1 +/* #undef HAVE_COMPRESS */ +/* #undef HAVE_CRYPT */ +#define HAVE_DLERROR 1 +#define HAVE_DLOPEN 1 +#define HAVE_FCHMOD 1 +#define HAVE_FCNTL 1 +/* #undef HAVE_FCONVERT */ +#define HAVE_FDATASYNC 1 +#define HAVE_FESETROUND 1 +#define HAVE_FINITE 1 +#define HAVE_FSEEKO 1 +#define HAVE_FSYNC 1 +#define HAVE_GETADDRINFO 1 +#define HAVE_GETCWD 1 +#define HAVE_GETHOSTBYADDR_R 1 +#define HAVE_GETHOSTBYNAME_R 1 +/* #undef HAVE_GETHRTIME */ +#define HAVE_GETNAMEINFO 1 +#define HAVE_GETPAGESIZE 1 +#define HAVE_GETPASS 1 +/* #undef HAVE_GETPASSPHRASE */ +#define HAVE_GETPWNAM 1 +#define HAVE_GETPWUID 1 +#define HAVE_GETRLIMIT 1 +#define HAVE_GETRUSAGE 1 +#define HAVE_GETWD 1 +#define HAVE_GMTIME_R 1 +#define HAVE_INITGROUPS 1 +#define HAVE_LDIV 1 +#define HAVE_LOCALTIME_R 1 +#define HAVE_LOG2 1 +#define HAVE_LONGJMP 1 +#define HAVE_LSTAT 1 +#define HAVE_MADVISE 1 +#define HAVE_MALLINFO 1 +#define HAVE_MEMALIGN 1 +#define HAVE_MEMCPY 1 +#define HAVE_MEMMOVE 1 +#define HAVE_MKSTEMP 1 +#define HAVE_MLOCK 1 +#define HAVE_MLOCKALL 1 +#define HAVE_MMAP 1 +#define HAVE_MMAP64 1 +#define HAVE_PERROR 1 +#define HAVE_POLL 1 +#define HAVE_PREAD 1 +/* #undef HAVE_PTHREAD_ATTR_CREATE */ +#define HAVE_PTHREAD_ATTR_GETSTACKSIZE 1 +/* #undef HAVE_PTHREAD_ATTR_SETPRIO */ +#define HAVE_PTHREAD_ATTR_SETSCHEDPARAM 1 +#define HAVE_PTHREAD_ATTR_SETSCOPE 1 +#define HAVE_PTHREAD_ATTR_SETSTACKSIZE 1 +/* #undef HAVE_PTHREAD_CONDATTR_CREATE */ +/* #undef HAVE_PTHREAD_INIT */ +#define HAVE_PTHREAD_KEY_DELETE 1 +#define HAVE_PTHREAD_KILL 1 +#define HAVE_PTHREAD_RWLOCK_RDLOCK 1 +/* #undef HAVE_PTHREAD_SETPRIO_NP */ +#define HAVE_PTHREAD_SETSCHEDPARAM 1 +#define HAVE_PTHREAD_SIGMASK 1 +/* #undef HAVE_PTHREAD_THREADMASK */ +/* #undef HAVE_PTHREAD_YIELD_NP */ +#define HAVE_READDIR_R 1 +#define HAVE_READLINK 1 +#define HAVE_REALPATH 1 +#define HAVE_RENAME 1 +#define HAVE_SCHED_YIELD 1 +#define HAVE_SELECT 1 +/* #undef HAVE_SETFD */ +/* #undef HAVE_SETFILEPOINTER */ +#define HAVE_SIGNAL 1 +#define HAVE_SIGACTION 1 +/* #undef HAVE_SIGTHREADMASK */ +#define HAVE_SIGWAIT 1 +#define HAVE_SLEEP 1 +#define HAVE_SNPRINTF 1 +/* #undef HAVE_SQLITE */ +#define HAVE_STPCPY 1 +#define HAVE_STRERROR 1 +/* #undef HAVE_STRLCPY */ +#define HAVE_STRNLEN 1 +#define HAVE_STRPBRK 1 +#define HAVE_STRSEP 1 +#define HAVE_STRSTR 1 +#define HAVE_STRTOK_R 1 +#define HAVE_STRTOL 1 +#define HAVE_STRTOLL 1 +#define HAVE_STRTOUL 1 +#define HAVE_STRTOULL 1 +/* #undef HAVE_TELL */ +/* #undef HAVE_THR_SETCONCURRENCY */ +/* #undef HAVE_THR_YIELD */ +#define HAVE_VASPRINTF 1 +#define HAVE_VSNPRINTF 1 + +/* + * types and sizes + */ +/* Types we may use */ +#define SIZEOF_CHAR 1 +#if defined(SIZEOF_CHAR) +# define HAVE_CHAR 1 +#endif + +#define SIZEOF_CHARP 8 +#if defined(SIZEOF_CHARP) +# define HAVE_CHARP 1 +#endif + +#define SIZEOF_SHORT 2 +#if defined(SIZEOF_SHORT) +# define HAVE_SHORT 1 +#endif + +#define SIZEOF_INT 4 +#if defined(SIZEOF_INT) +# define HAVE_INT 1 +#endif + +#define SIZEOF_LONG 8 +#if defined(SIZEOF_LONG) +# define HAVE_LONG 1 +#endif + +#define SIZEOF_LONG_LONG 8 +#if defined(SIZEOF_LONG_LONG) +# define HAVE_LONG_LONG 1 +#endif + + +#define SIZEOF_SIGSET_T 128 +#if defined(SIZEOF_SIGSET_T) +# define HAVE_SIGSET_T 1 +#endif + +#define SIZEOF_SIZE_T 8 +#if defined(SIZEOF_SIZE_T) +# define HAVE_SIZE_T 1 +#endif + +/* #undef SIZEOF_UCHAR */ +#if defined(SIZEOF_UCHAR) +# define HAVE_UCHAR 1 +#endif + +#define SIZEOF_UINT 4 +#if defined(SIZEOF_UINT) +# define HAVE_UINT 1 +#endif + +#define SIZEOF_ULONG 8 +#if defined(SIZEOF_ULONG) +# define HAVE_ULONG 1 +#endif + +/* #undef SIZEOF_INT8 */ +#if defined(SIZEOF_INT8) +# define HAVE_INT8 1 +#endif +/* #undef SIZEOF_UINT8 */ +#if defined(SIZEOF_UINT8) +# define HAVE_UINT8 1 +#endif + +/* #undef SIZEOF_INT16 */ +#if defined(SIZEOF_INT16) +# define HAVE_INT16 1 +#endif +/* #undef SIZEOF_UINT16 */ +#if defined(SIZEOF_UINT16) +# define HAVE_UINT16 1 +#endif + +/* #undef SIZEOF_INT32 */ +#if defined(SIZEOF_INT32) +# define HAVE_INT32 1 +#endif +/* #undef SIZEOF_UINT32 */ +#if defined(SIZEOF_UINT32) +# define HAVE_UINT32 1 +#endif +/* #undef SIZEOF_U_INT32_T */ +#if defined(SIZEOF_U_INT32_T) +# define HAVE_U_INT32_T 1 +#endif + +/* #undef SIZEOF_INT64 */ +#if defined(SIZEOF_INT64) +# define HAVE_INT64 1 +#endif +/* #undef SIZEOF_UINT64 */ +#if defined(SIZEOF_UINT64) +# define HAVE_UINT64 1 +#endif + +/* #undef SIZEOF_SOCKLEN_T */ +#if defined(SIZEOF_SOCKLEN_T) +# define HAVE_SOCKLEN_T 1 +#endif + +#define SOCKET_SIZE_TYPE socklen_t + +#define MARIADB_DEFAULT_CHARSET "latin1" + diff --git a/contrib/mariadb-connector-c-cmake/linux_x86_64/include/ma_config.h b/contrib/mariadb-connector-c-cmake/linux_x86_64/include/ma_config.h new file mode 100644 index 00000000000..90c42c97df6 --- /dev/null +++ b/contrib/mariadb-connector-c-cmake/linux_x86_64/include/ma_config.h @@ -0,0 +1,269 @@ + +/* + * Include file constants (processed in LibmysqlIncludeFiles.txt 1 + */ +#define HAVE_ALLOCA_H 1 +/* #undef HAVE_BIGENDIAN */ +#define HAVE_SETLOCALE 1 +#define HAVE_NL_LANGINFO 1 +#define HAVE_ARPA_INET_H 1 +#define HAVE_CRYPT_H 1 +#define HAVE_DIRENT_H 1 +#define HAVE_DLFCN_H 1 +#define HAVE_EXECINFO_H 1 +#define HAVE_FCNTL_H 1 +#define HAVE_FENV_H 1 +#define HAVE_FLOAT_H 1 +/* #undef HAVE_FPU_CONTROL_H */ +#define HAVE_GRP_H 1 +/* #undef HAVE_IEEEFP_H */ +#define HAVE_LIMITS_H 1 +#define HAVE_MALLOC_H 1 +#define HAVE_MEMORY_H 1 +#define HAVE_NETINET_IN_H 1 +#define HAVE_PATHS_H 1 +#define HAVE_PWD_H 1 +#define HAVE_SCHED_H 1 +/* #undef HAVE_SELECT_H */ +#define HAVE_STDDEF_H 1 +#define HAVE_STDINT_H 1 +#define HAVE_STDLIB_H 1 +#define HAVE_STRING_H 1 +#define HAVE_STRINGS_H 1 +/* #undef HAVE_SYNCH_H */ +/* #undef HAVE_SYS_FPU_H */ +#define HAVE_SYS_IOCTL_H 1 +#define HAVE_SYS_IPC_H 1 +#define HAVE_SYS_MMAN_H 1 +#define HAVE_SYS_PRCTL_H 1 +#define HAVE_SYS_SELECT_H 1 +#define HAVE_SYS_SHM_H 1 +#define HAVE_SYS_SOCKET_H 1 +#define HAVE_SYS_STAT_H 1 +/* #undef HAVE_SYS_STREAM_H */ +#define HAVE_SYS_TIMEB_H 1 +#define HAVE_SYS_TYPES_H 1 +#define HAVE_SYS_UN_H 1 +/* #undef HAVE_SYSENT_H */ +#define HAVE_TERMIO_H 1 +#define HAVE_TERMIOS_H 1 +#define HAVE_UNISTD_H 1 +#define HAVE_UTIME_H 1 +#define HAVE_UCONTEXT_H 1 + +/* + * function definitions - processed in LibmysqlFunctions.txt + */ +#define HAVE_ACCESS 1 +/* #undef HAVE_AIOWAIT */ +#define HAVE_ALARM 1 +/* #undef HAVE_ALLOCA */ +#define HAVE_BCMP 1 +/* #undef HAVE_BFILL */ +/* #undef HAVE_BMOVE */ +#define HAVE_BZERO 1 +#define HAVE_CLOCK_GETTIME 1 +/* #undef HAVE_COMPRESS */ +/* #undef HAVE_CRYPT */ +#define HAVE_DLERROR 1 +#define HAVE_DLOPEN 1 +#define HAVE_FCHMOD 1 +#define HAVE_FCNTL 1 +/* #undef HAVE_FCONVERT */ +#define HAVE_FDATASYNC 1 +#define HAVE_FESETROUND 1 +#define HAVE_FINITE 1 +#define HAVE_FSEEKO 1 +#define HAVE_FSYNC 1 +#define HAVE_GETADDRINFO 1 +#define HAVE_GETCWD 1 +#define HAVE_GETHOSTBYADDR_R 1 +#define HAVE_GETHOSTBYNAME_R 1 +/* #undef HAVE_GETHRTIME */ +#define HAVE_GETNAMEINFO 1 +#define HAVE_GETPAGESIZE 1 +#define HAVE_GETPASS 1 +/* #undef HAVE_GETPASSPHRASE */ +#define HAVE_GETPWNAM 1 +#define HAVE_GETPWUID 1 +#define HAVE_GETRLIMIT 1 +#define HAVE_GETRUSAGE 1 +#define HAVE_GETWD 1 +#define HAVE_GMTIME_R 1 +#define HAVE_INITGROUPS 1 +#define HAVE_LDIV 1 +#define HAVE_LOCALTIME_R 1 +#define HAVE_LOG2 1 +#define HAVE_LONGJMP 1 +#define HAVE_LSTAT 1 +#define HAVE_MADVISE 1 +#define HAVE_MALLINFO 1 +#define HAVE_MEMALIGN 1 +#define HAVE_MEMCPY 1 +#define HAVE_MEMMOVE 1 +#define HAVE_MKSTEMP 1 +#define HAVE_MLOCK 1 +#define HAVE_MLOCKALL 1 +#define HAVE_MMAP 1 +#define HAVE_MMAP64 1 +#define HAVE_PERROR 1 +#define HAVE_POLL 1 +#define HAVE_PREAD 1 +/* #undef HAVE_PTHREAD_ATTR_CREATE */ +#define HAVE_PTHREAD_ATTR_GETSTACKSIZE 1 +/* #undef HAVE_PTHREAD_ATTR_SETPRIO */ +#define HAVE_PTHREAD_ATTR_SETSCHEDPARAM 1 +#define HAVE_PTHREAD_ATTR_SETSCOPE 1 +#define HAVE_PTHREAD_ATTR_SETSTACKSIZE 1 +/* #undef HAVE_PTHREAD_CONDATTR_CREATE */ +/* #undef HAVE_PTHREAD_INIT */ +#define HAVE_PTHREAD_KEY_DELETE 1 +#define HAVE_PTHREAD_KILL 1 +#define HAVE_PTHREAD_RWLOCK_RDLOCK 1 +/* #undef HAVE_PTHREAD_SETPRIO_NP */ +#define HAVE_PTHREAD_SETSCHEDPARAM 1 +#define HAVE_PTHREAD_SIGMASK 1 +/* #undef HAVE_PTHREAD_THREADMASK */ +/* #undef HAVE_PTHREAD_YIELD_NP */ +#define HAVE_READDIR_R 1 +#define HAVE_READLINK 1 +#define HAVE_REALPATH 1 +#define HAVE_RENAME 1 +#define HAVE_SCHED_YIELD 1 +#define HAVE_SELECT 1 +/* #undef HAVE_SETFD */ +/* #undef HAVE_SETFILEPOINTER */ +#define HAVE_SIGNAL 1 +#define HAVE_SIGACTION 1 +/* #undef HAVE_SIGTHREADMASK */ +#define HAVE_SIGWAIT 1 +#define HAVE_SLEEP 1 +#define HAVE_SNPRINTF 1 +/* #undef HAVE_SQLITE */ +#define HAVE_STPCPY 1 +#define HAVE_STRERROR 1 +/* #undef HAVE_STRLCPY */ +#define HAVE_STRNLEN 1 +#define HAVE_STRPBRK 1 +#define HAVE_STRSEP 1 +#define HAVE_STRSTR 1 +#define HAVE_STRTOK_R 1 +#define HAVE_STRTOL 1 +#define HAVE_STRTOLL 1 +#define HAVE_STRTOUL 1 +#define HAVE_STRTOULL 1 +/* #undef HAVE_TELL */ +/* #undef HAVE_THR_SETCONCURRENCY */ +/* #undef HAVE_THR_YIELD */ +#define HAVE_VASPRINTF 1 +#define HAVE_VSNPRINTF 1 + +/* + * types and sizes + */ +/* Types we may use */ +#define SIZEOF_CHAR 1 +#if defined(SIZEOF_CHAR) +# define HAVE_CHAR 1 +#endif + +#define SIZEOF_CHARP 8 +#if defined(SIZEOF_CHARP) +# define HAVE_CHARP 1 +#endif + +#define SIZEOF_SHORT 2 +#if defined(SIZEOF_SHORT) +# define HAVE_SHORT 1 +#endif + +#define SIZEOF_INT 4 +#if defined(SIZEOF_INT) +# define HAVE_INT 1 +#endif + +#define SIZEOF_LONG 8 +#if defined(SIZEOF_LONG) +# define HAVE_LONG 1 +#endif + +#define SIZEOF_LONG_LONG 8 +#if defined(SIZEOF_LONG_LONG) +# define HAVE_LONG_LONG 1 +#endif + + +#define SIZEOF_SIGSET_T 128 +#if defined(SIZEOF_SIGSET_T) +# define HAVE_SIGSET_T 1 +#endif + +#define SIZEOF_SIZE_T 8 +#if defined(SIZEOF_SIZE_T) +# define HAVE_SIZE_T 1 +#endif + +/* #undef SIZEOF_UCHAR */ +#if defined(SIZEOF_UCHAR) +# define HAVE_UCHAR 1 +#endif + +#define SIZEOF_UINT 4 +#if defined(SIZEOF_UINT) +# define HAVE_UINT 1 +#endif + +#define SIZEOF_ULONG 8 +#if defined(SIZEOF_ULONG) +# define HAVE_ULONG 1 +#endif + +/* #undef SIZEOF_INT8 */ +#if defined(SIZEOF_INT8) +# define HAVE_INT8 1 +#endif +/* #undef SIZEOF_UINT8 */ +#if defined(SIZEOF_UINT8) +# define HAVE_UINT8 1 +#endif + +/* #undef SIZEOF_INT16 */ +#if defined(SIZEOF_INT16) +# define HAVE_INT16 1 +#endif +/* #undef SIZEOF_UINT16 */ +#if defined(SIZEOF_UINT16) +# define HAVE_UINT16 1 +#endif + +/* #undef SIZEOF_INT32 */ +#if defined(SIZEOF_INT32) +# define HAVE_INT32 1 +#endif +/* #undef SIZEOF_UINT32 */ +#if defined(SIZEOF_UINT32) +# define HAVE_UINT32 1 +#endif +/* #undef SIZEOF_U_INT32_T */ +#if defined(SIZEOF_U_INT32_T) +# define HAVE_U_INT32_T 1 +#endif + +/* #undef SIZEOF_INT64 */ +#if defined(SIZEOF_INT64) +# define HAVE_INT64 1 +#endif +/* #undef SIZEOF_UINT64 */ +#if defined(SIZEOF_UINT64) +# define HAVE_UINT64 1 +#endif + +/* #undef SIZEOF_SOCKLEN_T */ +#if defined(SIZEOF_SOCKLEN_T) +# define HAVE_SOCKLEN_T 1 +#endif + +#define SOCKET_SIZE_TYPE socklen_t + +#define MARIADB_DEFAULT_CHARSET "latin1" + diff --git a/contrib/mariadb-connector-c-cmake/linux_x86_64/include/mariadb_version.h b/contrib/mariadb-connector-c-cmake/linux_x86_64/include/mariadb_version.h new file mode 100644 index 00000000000..21c14178454 --- /dev/null +++ b/contrib/mariadb-connector-c-cmake/linux_x86_64/include/mariadb_version.h @@ -0,0 +1,36 @@ +/* Copyright Abandoned 1996, 1999, 2001 MySQL AB + This file is public domain and comes with NO WARRANTY of any kind */ + +/* Version numbers for protocol & mysqld */ + +#ifndef _mariadb_version_h_ +#define _mariadb_version_h_ + +#ifdef _CUSTOMCONFIG_ +#include +#else +#define PROTOCOL_VERSION 10 +#define MARIADB_CLIENT_VERSION_STR "10.3.6" +#define MARIADB_BASE_VERSION "mariadb-10.3" +#define MARIADB_VERSION_ID 100306 +#define MYSQL_VERSION_ID 100306 +#define MARIADB_PORT 3306 +#define MARIADB_UNIX_ADDR "/tmp/mysql.sock" +#define MYSQL_CONFIG_NAME "my" + +#define MARIADB_PACKAGE_VERSION "3.0.6" +#define MARIADB_PACKAGE_VERSION_ID 30006 +#define MARIADB_SYSTEM_TYPE "Linux" +#define MARIADB_MACHINE_TYPE "x86_64" +#define MARIADB_PLUGINDIR "lib/mariadb/plugin" + +/* mysqld compile time options */ +#ifndef MYSQL_CHARSET +#define MYSQL_CHARSET "" +#endif +#endif + +/* Source information */ +#define CC_SOURCE_REVISION "a0fd36cc5a5313414a5a2ebe9322577a29b4782a" + +#endif /* _mariadb_version_h_ */ diff --git a/contrib/mariadb-connector-c-cmake/linux_x86_64/libmariadb/ma_client_plugin.c b/contrib/mariadb-connector-c-cmake/linux_x86_64/libmariadb/ma_client_plugin.c new file mode 100644 index 00000000000..b7fdcdbcb85 --- /dev/null +++ b/contrib/mariadb-connector-c-cmake/linux_x86_64/libmariadb/ma_client_plugin.c @@ -0,0 +1,499 @@ +/* Copyright (C) 2010 - 2012 Sergei Golubchik and Monty Program Ab + 2015-2016 MariaDB Corporation AB + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public + License as published by the Free Software Foundation; either + version 2 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public + License along with this library; if not see + or write to the Free Software Foundation, Inc., + 51 Franklin St., Fifth Floor, Boston, MA 02110, USA */ + +/** + @file + + Support code for the client side (libmariadb) plugins + + Client plugins are somewhat different from server plugins, they are simpler. + + They do not need to be installed or in any way explicitly loaded on the + client, they are loaded automatically on demand. + One client plugin per shared object, soname *must* match the plugin name. + + There is no reference counting and no unloading either. +*/ + +#if _MSC_VER +/* Silence warnings about variable 'unused' being used. */ +#define FORCE_INIT_OF_VARS 1 +#endif + +#include +#include +#include +#include +#include + +#include "errmsg.h" +#include + +struct st_client_plugin_int { + struct st_client_plugin_int *next; + void *dlhandle; + struct st_mysql_client_plugin *plugin; +}; + +static my_bool initialized= 0; +static MA_MEM_ROOT mem_root; + +static uint valid_plugins[][2]= { + {MYSQL_CLIENT_AUTHENTICATION_PLUGIN, MYSQL_CLIENT_AUTHENTICATION_PLUGIN_INTERFACE_VERSION}, + {MARIADB_CLIENT_PVIO_PLUGIN, MARIADB_CLIENT_PVIO_PLUGIN_INTERFACE_VERSION}, + {MARIADB_CLIENT_TRACE_PLUGIN, MARIADB_CLIENT_TRACE_PLUGIN_INTERFACE_VERSION}, + {MARIADB_CLIENT_CONNECTION_PLUGIN, MARIADB_CLIENT_CONNECTION_PLUGIN_INTERFACE_VERSION}, + {0, 0} +}; + +/* + Loaded plugins are stored in a linked list. + The list is append-only, the elements are added to the head (like in a stack). + The elements are added under a mutex, but the list can be read and traversed + without any mutex because once an element is added to the list, it stays + there. The main purpose of a mutex is to prevent two threads from + loading the same plugin twice in parallel. +*/ + + +struct st_client_plugin_int *plugin_list[MYSQL_CLIENT_MAX_PLUGINS + MARIADB_CLIENT_MAX_PLUGINS]; +#ifdef THREAD +static pthread_mutex_t LOCK_load_client_plugin; +#endif + + extern struct st_mysql_client_plugin mysql_native_password_client_plugin; + extern struct st_mysql_client_plugin mysql_old_password_client_plugin; + extern struct st_mysql_client_plugin pvio_socket_client_plugin; + + +struct st_mysql_client_plugin *mysql_client_builtins[]= +{ + (struct st_mysql_client_plugin *)&mysql_native_password_client_plugin, + (struct st_mysql_client_plugin *)&mysql_old_password_client_plugin, + (struct st_mysql_client_plugin *)&pvio_socket_client_plugin, + + 0 +}; + + +static int is_not_initialized(MYSQL *mysql, const char *name) +{ + if (initialized) + return 0; + + my_set_error(mysql, CR_AUTH_PLUGIN_CANNOT_LOAD, + SQLSTATE_UNKNOWN, ER(CR_AUTH_PLUGIN_CANNOT_LOAD), + name, "not initialized"); + return 1; +} + +static int get_plugin_nr(uint type) +{ + uint i= 0; + for(; valid_plugins[i][1]; i++) + if (valid_plugins[i][0] == type) + return i; + return -1; +} + +static const char *check_plugin_version(struct st_mysql_client_plugin *plugin, unsigned int version) +{ + if (plugin->interface_version < version || + (plugin->interface_version >> 8) > (version >> 8)) + return "Incompatible client plugin interface"; + return 0; +} + +/** + finds a plugin in the list + + @param name plugin name to search for + @param type plugin type + + @note this does NOT necessarily need a mutex, take care! + + @retval a pointer to a found plugin or 0 +*/ +static struct st_mysql_client_plugin *find_plugin(const char *name, int type) +{ + struct st_client_plugin_int *p; + int plugin_nr= get_plugin_nr(type); + + DBUG_ASSERT(initialized); + if (plugin_nr == -1) + return 0; + + if (!name) + return plugin_list[plugin_nr]->plugin; + + for (p= plugin_list[plugin_nr]; p; p= p->next) + { + if (strcmp(p->plugin->name, name) == 0) + return p->plugin; + } + return NULL; +} + + +/** + verifies the plugin and adds it to the list + + @param mysql MYSQL structure (for error reporting) + @param plugin plugin to install + @param dlhandle a handle to the shared object (returned by dlopen) + or 0 if the plugin was not dynamically loaded + @param argc number of arguments in the 'va_list args' + @param args arguments passed to the plugin initialization function + + @retval a pointer to an installed plugin or 0 +*/ + +static struct st_mysql_client_plugin * +add_plugin(MYSQL *mysql, struct st_mysql_client_plugin *plugin, void *dlhandle, + int argc, va_list args) +{ + const char *errmsg; + struct st_client_plugin_int plugin_int, *p; + char errbuf[1024]; + int plugin_nr; + + DBUG_ASSERT(initialized); + + plugin_int.plugin= plugin; + plugin_int.dlhandle= dlhandle; + + if ((plugin_nr= get_plugin_nr(plugin->type)) == -1) + { + errmsg= "Unknown client plugin type"; + goto err1; + } + if ((errmsg= check_plugin_version(plugin, valid_plugins[plugin_nr][1]))) + goto err1; + + /* Call the plugin initialization function, if any */ + if (plugin->init && plugin->init(errbuf, sizeof(errbuf), argc, args)) + { + errmsg= errbuf; + goto err1; + } + + p= (struct st_client_plugin_int *) + ma_memdup_root(&mem_root, (char *)&plugin_int, sizeof(plugin_int)); + + if (!p) + { + errmsg= "Out of memory"; + goto err2; + } + +#ifdef THREAD + safe_mutex_assert_owner(&LOCK_load_client_plugin); +#endif + + p->next= plugin_list[plugin_nr]; + plugin_list[plugin_nr]= p; + + return plugin; + +err2: + if (plugin->deinit) + plugin->deinit(); +err1: + my_set_error(mysql, CR_AUTH_PLUGIN_CANNOT_LOAD, SQLSTATE_UNKNOWN, + ER(CR_AUTH_PLUGIN_CANNOT_LOAD), plugin->name, errmsg); + if (dlhandle) + (void)dlclose(dlhandle); + return NULL; +} + + +/** + Loads plugins which are specified in the environment variable + LIBMYSQL_PLUGINS. + + Multiple plugins must be separated by semicolon. This function doesn't + return or log an error. + + The function is be called by mysql_client_plugin_init + + @todo + Support extended syntax, passing parameters to plugins, for example + LIBMYSQL_PLUGINS="plugin1(param1,param2);plugin2;..." + or + LIBMYSQL_PLUGINS="plugin1=int:param1,str:param2;plugin2;..." +*/ + +static void load_env_plugins(MYSQL *mysql) +{ + char *plugs, *free_env, *s= getenv("LIBMYSQL_PLUGINS"); + + if (ma_check_env_str(s)) + return; + + free_env= strdup(s); + plugs= s= free_env; + + do { + if ((s= strchr(plugs, ';'))) + *s= '\0'; + mysql_load_plugin(mysql, plugs, -1, 0); + plugs= s + 1; + } while (s); + + free(free_env); +} + +/********** extern functions to be used by libmariadb *********************/ + +/** + Initializes the client plugin layer. + + This function must be called before any other client plugin function. + + @retval 0 successful + @retval != 0 error occurred +*/ + +int mysql_client_plugin_init() +{ + MYSQL mysql; + struct st_mysql_client_plugin **builtin; + va_list unused; + LINT_INIT_STRUCT(unused); + + if (initialized) + return 0; + + memset(&mysql, 0, sizeof(mysql)); /* dummy mysql for set_mysql_extended_error */ + + pthread_mutex_init(&LOCK_load_client_plugin, MY_MUTEX_INIT_SLOW); + ma_init_alloc_root(&mem_root, 128, 128); + + memset(&plugin_list, 0, sizeof(plugin_list)); + + initialized= 1; + + pthread_mutex_lock(&LOCK_load_client_plugin); + for (builtin= mysql_client_builtins; *builtin; builtin++) + add_plugin(&mysql, *builtin, 0, 0, unused); + + pthread_mutex_unlock(&LOCK_load_client_plugin); + + load_env_plugins(&mysql); + + return 0; +} + + +/** + Deinitializes the client plugin layer. + + Unloades all client plugins and frees any associated resources. +*/ + +void mysql_client_plugin_deinit() +{ + int i; + struct st_client_plugin_int *p; + + if (!initialized) + return; + + for (i=0; i < MYSQL_CLIENT_MAX_PLUGINS; i++) + for (p= plugin_list[i]; p; p= p->next) + { + if (p->plugin->deinit) + p->plugin->deinit(); + if (p->dlhandle) + (void)dlclose(p->dlhandle); + } + + memset(&plugin_list, 0, sizeof(plugin_list)); + initialized= 0; + ma_free_root(&mem_root, MYF(0)); + pthread_mutex_destroy(&LOCK_load_client_plugin); +} + +/************* public facing functions, for client consumption *********/ + +/* see for a full description */ +struct st_mysql_client_plugin * STDCALL +mysql_client_register_plugin(MYSQL *mysql, + struct st_mysql_client_plugin *plugin) +{ + va_list unused; + LINT_INIT_STRUCT(unused); + + if (is_not_initialized(mysql, plugin->name)) + return NULL; + + pthread_mutex_lock(&LOCK_load_client_plugin); + + /* make sure the plugin wasn't loaded meanwhile */ + if (find_plugin(plugin->name, plugin->type)) + { + my_set_error(mysql, CR_AUTH_PLUGIN_CANNOT_LOAD, + SQLSTATE_UNKNOWN, ER(CR_AUTH_PLUGIN_CANNOT_LOAD), + plugin->name, "it is already loaded"); + plugin= NULL; + } + else + plugin= add_plugin(mysql, plugin, 0, 0, unused); + + pthread_mutex_unlock(&LOCK_load_client_plugin); + return plugin; +} + + +/* see for a full description */ +struct st_mysql_client_plugin * STDCALL +mysql_load_plugin_v(MYSQL *mysql, const char *name, int type, + int argc, va_list args) +{ + const char *errmsg; +#ifdef _WIN32 + char errbuf[1024]; +#endif + char dlpath[FN_REFLEN+1]; + void *sym, *dlhandle = NULL; + struct st_mysql_client_plugin *plugin; + char *env_plugin_dir= getenv("MARIADB_PLUGIN_DIR"); + + CLEAR_CLIENT_ERROR(mysql); + if (is_not_initialized(mysql, name)) + return NULL; + + pthread_mutex_lock(&LOCK_load_client_plugin); + + /* make sure the plugin wasn't loaded meanwhile */ + if (type >= 0 && find_plugin(name, type)) + { + errmsg= "it is already loaded"; + goto err; + } + + /* Compile dll path */ + snprintf(dlpath, sizeof(dlpath) - 1, "%s/%s%s", + mysql->options.extension && mysql->options.extension->plugin_dir ? + mysql->options.extension->plugin_dir : (env_plugin_dir) ? env_plugin_dir : + MARIADB_PLUGINDIR, name, SO_EXT); + + /* Open new dll handle */ + if (!(dlhandle= dlopen((const char *)dlpath, RTLD_NOW))) + { +#ifdef _WIN32 + char winmsg[255]; + size_t len; + winmsg[0] = 0; + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, + NULL, + GetLastError(), + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + winmsg, 255, NULL); + len= strlen(winmsg); + while (len > 0 && (winmsg[len - 1] == '\n' || winmsg[len - 1] == '\r')) + len--; + if (len) + winmsg[len] = 0; + snprintf(errbuf, sizeof(errbuf), "%s Library path is '%s'", winmsg, dlpath); + errmsg= errbuf; +#else + errmsg= dlerror(); +#endif + goto err; + } + + + if (!(sym= dlsym(dlhandle, plugin_declarations_sym))) + { + errmsg= "not a plugin"; + (void)dlclose(dlhandle); + goto err; + } + + plugin= (struct st_mysql_client_plugin*)sym; + + if (type >=0 && type != plugin->type) + { + errmsg= "type mismatch"; + goto err; + } + + if (strcmp(name, plugin->name)) + { + errmsg= "name mismatch"; + goto err; + } + + if (type < 0 && find_plugin(name, plugin->type)) + { + errmsg= "it is already loaded"; + goto err; + } + + plugin= add_plugin(mysql, plugin, dlhandle, argc, args); + + pthread_mutex_unlock(&LOCK_load_client_plugin); + + return plugin; + +err: + if (dlhandle) + dlclose(dlhandle); + pthread_mutex_unlock(&LOCK_load_client_plugin); + my_set_error(mysql, CR_AUTH_PLUGIN_CANNOT_LOAD, SQLSTATE_UNKNOWN, + ER(CR_AUTH_PLUGIN_CANNOT_LOAD), name, errmsg); + return NULL; +} + + +/* see for a full description */ +struct st_mysql_client_plugin * STDCALL +mysql_load_plugin(MYSQL *mysql, const char *name, int type, int argc, ...) +{ + struct st_mysql_client_plugin *p; + va_list args; + va_start(args, argc); + p= mysql_load_plugin_v(mysql, name, type, argc, args); + va_end(args); + return p; +} + +/* see for a full description */ +struct st_mysql_client_plugin * STDCALL +mysql_client_find_plugin(MYSQL *mysql, const char *name, int type) +{ + struct st_mysql_client_plugin *p; + int plugin_nr= get_plugin_nr(type); + + if (is_not_initialized(mysql, name)) + return NULL; + + if (plugin_nr == -1) + { + my_set_error(mysql, CR_AUTH_PLUGIN_CANNOT_LOAD, SQLSTATE_UNKNOWN, + ER(CR_AUTH_PLUGIN_CANNOT_LOAD), name, "invalid type"); + } + + if ((p= find_plugin(name, type))) + return p; + + /* not found, load it */ + return mysql_load_plugin(mysql, name, type, 0); +} + diff --git a/docs/en/development/build.md b/docs/en/development/build.md index 98e25c5ab4a..6bf67e58c9a 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -46,7 +46,7 @@ export CXX=g++-7 ## Install required libraries from packages ```bash -sudo apt-get install libicu-dev libreadline-dev libmysqlclient-dev libssl-dev unixodbc-dev ninja-build +sudo apt-get install libicu-dev libreadline-dev libssl-dev unixodbc-dev ninja-build ``` ## Checkout ClickHouse sources diff --git a/libs/libmysqlxx/CMakeLists.txt b/libs/libmysqlxx/CMakeLists.txt index 95c1c44c315..76f11b20ea8 100644 --- a/libs/libmysqlxx/CMakeLists.txt +++ b/libs/libmysqlxx/CMakeLists.txt @@ -29,25 +29,29 @@ add_library (mysqlxx ${SPLIT_SHARED} target_include_directories (mysqlxx PUBLIC include) -set(PLATFORM_LIBRARIES ${CMAKE_DL_LIBS}) +if (USE_INTERNAL_MYSQL_LIBRARY) -if (USE_MYSQL) - target_include_directories (mysqlxx SYSTEM PRIVATE ${MYSQL_INCLUDE_DIR}) -endif () +else () + set(PLATFORM_LIBRARIES ${CMAKE_DL_LIBS}) -if (APPLE) - find_library (ICONV_LIBRARY iconv) - set (MYSQLCLIENT_LIBRARIES ${MYSQLCLIENT_LIBRARIES} ${STATIC_MYSQLCLIENT_LIB} ${ICONV_LIBRARY}) -elseif (USE_STATIC_LIBRARIES AND STATIC_MYSQLCLIENT_LIB) - set (MYSQLCLIENT_LIB ${CMAKE_CURRENT_BINARY_DIR}/libmysqlclient.a) - add_custom_command ( - OUTPUT ${MYSQLCLIENT_LIB} - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/patch.sh ${STATIC_MYSQLCLIENT_LIB} ${MYSQLCLIENT_LIB} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "Patching mysqlclient library.") - add_custom_target (our_mysql_client DEPENDS ${MYSQLCLIENT_LIB}) - add_dependencies (mysqlxx our_mysql_client) - set (MYSQLCLIENT_LIBRARIES ${MYSQLCLIENT_LIB}) + if (USE_MYSQL) + target_include_directories (mysqlxx SYSTEM PRIVATE ${MYSQL_INCLUDE_DIR}) + endif () + + if (APPLE) + find_library (ICONV_LIBRARY iconv) + set (MYSQLCLIENT_LIBRARIES ${MYSQLCLIENT_LIBRARIES} ${STATIC_MYSQLCLIENT_LIB} ${ICONV_LIBRARY}) + elseif (USE_STATIC_LIBRARIES AND STATIC_MYSQLCLIENT_LIB) + set (MYSQLCLIENT_LIB ${CMAKE_CURRENT_BINARY_DIR}/libmysqlclient.a) + add_custom_command ( + OUTPUT ${MYSQLCLIENT_LIB} + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/patch.sh ${STATIC_MYSQLCLIENT_LIB} ${MYSQLCLIENT_LIB} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Patching mysqlclient library.") + add_custom_target (our_mysql_client DEPENDS ${MYSQLCLIENT_LIB}) + add_dependencies (mysqlxx our_mysql_client) + set (MYSQLCLIENT_LIBRARIES ${MYSQLCLIENT_LIB}) + endif () endif () target_link_libraries (mysqlxx common ${MYSQLCLIENT_LIBRARIES} ${OPENSSL_LIBRARIES} ${ZLIB_LIBRARIES} ${PLATFORM_LIBRARIES}) diff --git a/libs/libmysqlxx/cmake/find_mysqlclient.cmake b/libs/libmysqlxx/cmake/find_mysqlclient.cmake index d019703e876..2a98b76d3a8 100644 --- a/libs/libmysqlxx/cmake/find_mysqlclient.cmake +++ b/libs/libmysqlxx/cmake/find_mysqlclient.cmake @@ -1,38 +1,51 @@ option (ENABLE_MYSQL "Enable MySQL" ON) +option (USE_INTERNAL_MYSQL_LIBRARY "Set to FALSE to use system mysqlclient library instead of bundled" ${NOT_UNBUNDLED}) + +if (USE_INTERNAL_MYSQL_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/mariadb-connector-c/README") + message (WARNING "submodule contrib/mariadb-connector-c is missing. to fix try run: \n git submodule update --init --recursive") + set (USE_INTERNAL_MYSQL_LIBRARY 0) +endif () + if (ENABLE_MYSQL) - set (MYSQL_LIB_PATHS - "/usr/local/opt/mysql/lib" - "/usr/local/lib" - "/usr/local/lib64" - "/usr/local/lib/mariadb" # macos brew mariadb-connector-c - "/usr/mysql/lib" - "/usr/mysql/lib64" - "/usr/lib" - "/usr/lib64" - "/lib" - "/lib64") - - set (MYSQL_INCLUDE_PATHS - "/usr/local/opt/mysql/include" - "/usr/mysql/include" - "/usr/local/include" - "/usr/include") - - find_path (MYSQL_INCLUDE_DIR NAMES mysql/mysql.h mariadb/mysql.h PATHS ${MYSQL_INCLUDE_PATHS} PATH_SUFFIXES mysql) - - if (USE_STATIC_LIBRARIES) - find_library (STATIC_MYSQLCLIENT_LIB NAMES mariadbclient mysqlclient PATHS ${MYSQL_LIB_PATHS} PATH_SUFFIXES mysql) - else () - find_library (MYSQLCLIENT_LIBRARIES NAMES mariadb mariadbclient mysqlclient PATHS ${MYSQL_LIB_PATHS} PATH_SUFFIXES mysql) - endif () - - if (MYSQL_INCLUDE_DIR AND (STATIC_MYSQLCLIENT_LIB OR MYSQLCLIENT_LIBRARIES)) + if (USE_INTERNAL_MYSQL_LIBRARY) + set (MYSQLCLIENT_LIBRARIES mysqlclient) set (USE_MYSQL 1) set (MYSQLXX_LIBRARY mysqlxx) - if (APPLE) - # /usr/local/include/mysql/mysql_com.h:1011:10: fatal error: mysql/udf_registration_types.h: No such file or directory - set(MYSQL_INCLUDE_DIR ${MYSQL_INCLUDE_DIR} ${MYSQL_INCLUDE_DIR}/mysql) + else () + set (MYSQL_LIB_PATHS + "/usr/local/opt/mysql/lib" + "/usr/local/lib" + "/usr/local/lib64" + "/usr/local/lib/mariadb" # macos brew mariadb-connector-c + "/usr/mysql/lib" + "/usr/mysql/lib64" + "/usr/lib" + "/usr/lib64" + "/lib" + "/lib64") + + set (MYSQL_INCLUDE_PATHS + "/usr/local/opt/mysql/include" + "/usr/mysql/include" + "/usr/local/include" + "/usr/include") + + find_path (MYSQL_INCLUDE_DIR NAMES mysql/mysql.h mariadb/mysql.h PATHS ${MYSQL_INCLUDE_PATHS} PATH_SUFFIXES mysql) + + if (USE_STATIC_LIBRARIES) + find_library (STATIC_MYSQLCLIENT_LIB NAMES mariadbclient mysqlclient PATHS ${MYSQL_LIB_PATHS} PATH_SUFFIXES mysql) + else () + find_library (MYSQLCLIENT_LIBRARIES NAMES mariadb mariadbclient mysqlclient PATHS ${MYSQL_LIB_PATHS} PATH_SUFFIXES mysql) + endif () + + if (MYSQL_INCLUDE_DIR AND (STATIC_MYSQLCLIENT_LIB OR MYSQLCLIENT_LIBRARIES)) + set (USE_MYSQL 1) + set (MYSQLXX_LIBRARY mysqlxx) + if (APPLE) + # /usr/local/include/mysql/mysql_com.h:1011:10: fatal error: mysql/udf_registration_types.h: No such file or directory + set(MYSQL_INCLUDE_DIR ${MYSQL_INCLUDE_DIR} ${MYSQL_INCLUDE_DIR}/mysql) + endif () endif () endif () endif () From 76e06fe0ebeebd98ab4c6a6cc18300a7fe4fffe9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 2 Aug 2018 08:30:52 +0300 Subject: [PATCH 18/20] Don't escape values of parameters of ODBC connection string if it is not necessarily, because drivers don't recognize them [#CLICKHOUSE-3861] --- .../validateODBCConnectionString.cpp | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/dbms/src/Dictionaries/validateODBCConnectionString.cpp b/dbms/src/Dictionaries/validateODBCConnectionString.cpp index c8c7851d5f8..2f199e1b88c 100644 --- a/dbms/src/Dictionaries/validateODBCConnectionString.cpp +++ b/dbms/src/Dictionaries/validateODBCConnectionString.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -179,7 +180,12 @@ std::string validateODBCConnectionString(const std::string & connection_string) std::string reconstructed_connection_string; - auto write_value = [&](const std::string & value) + auto write_plain_value = [&](const std::string & value) + { + reconstructed_connection_string += value; + }; + + auto write_escaped_value = [&](const std::string & value) { reconstructed_connection_string += '{'; @@ -205,6 +211,14 @@ std::string validateODBCConnectionString(const std::string & connection_string) reconstructed_connection_string += '}'; }; + auto write_value = [&](const std::string & value) + { + if (std::all_of(value.begin(), value.end(), isWordCharASCII)) + write_plain_value(value); + else + write_escaped_value(value); + }; + auto write_element = [&](const std::string & name, const std::string & value) { reconstructed_connection_string.append(name); From e6be3df322f24ff3aa9ae9a97b9b01b2c88ab7b0 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 2 Aug 2018 11:18:15 +0300 Subject: [PATCH 19/20] Added unixodbc as a submodule. (#2789) * Using unixodbc as submodule [#CLICKHOUSE-3] * Using unixodbc as submodule [#CLICKHOUSE-3] * Using unixodbc as submodule [#CLICKHOUSE-3] * Using unixodbc as submodule [#CLICKHOUSE-3] * Using unixodbc as submodule [#CLICKHOUSE-3] * Using unixodbc as submodule [#CLICKHOUSE-3] * Using unixodbc as submodule [#CLICKHOUSE-3] --- .gitmodules | 3 + CMakeLists.txt | 7 +- ci/install-libraries.sh | 1 - ci/install-os-packages.sh | 9 - cmake/find_ltdl.cmake | 8 +- cmake/find_odbc.cmake | 110 ++-- contrib/CMakeLists.txt | 4 + contrib/unixodbc | 1 + contrib/unixodbc-cmake/CMakeLists.txt | 287 ++++++++++ contrib/unixodbc-cmake/linux_x86_64/config.h | 496 ++++++++++++++++++ .../linux_x86_64/libltdl/config.h | 181 +++++++ .../linux_x86_64/libltdl/libltdlcS.c | 53 ++ .../linux_x86_64/unixodbc_conf.h | 60 +++ docs/en/development/build.md | 2 +- docs/en/development/build_osx.md | 2 +- 15 files changed, 1156 insertions(+), 68 deletions(-) create mode 160000 contrib/unixodbc create mode 100644 contrib/unixodbc-cmake/CMakeLists.txt create mode 100644 contrib/unixodbc-cmake/linux_x86_64/config.h create mode 100644 contrib/unixodbc-cmake/linux_x86_64/libltdl/config.h create mode 100644 contrib/unixodbc-cmake/linux_x86_64/libltdl/libltdlcS.c create mode 100644 contrib/unixodbc-cmake/linux_x86_64/unixodbc_conf.h diff --git a/.gitmodules b/.gitmodules index 6c71fec419a..742e4616276 100644 --- a/.gitmodules +++ b/.gitmodules @@ -43,3 +43,6 @@ [submodule "contrib/jemalloc"] path = contrib/jemalloc url = https://github.com/jemalloc/jemalloc.git +[submodule "contrib/unixodbc"] + path = contrib/unixodbc + url = https://github.com/ClickHouse-Extras/UnixODBC.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 4467184517c..68e293a5d7c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -258,12 +258,7 @@ include (cmake/find_zlib.cmake) include (cmake/find_zstd.cmake) include (cmake/find_ltdl.cmake) # for odbc include (cmake/find_termcap.cmake) -if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/poco/cmake/FindODBC.cmake) - include (${CMAKE_CURRENT_SOURCE_DIR}/contrib/poco/cmake/FindODBC.cmake) # for poco -else () - include (cmake/find_odbc.cmake) -endif () -message (STATUS "Using odbc: ${ODBC_INCLUDE_DIRECTORIES} : ${ODBC_LIBRARIES}") +include (cmake/find_odbc.cmake) include (cmake/find_poco.cmake) include (cmake/find_lz4.cmake) include (cmake/find_sparsehash.cmake) diff --git a/ci/install-libraries.sh b/ci/install-libraries.sh index 0fd77d51915..ee2e5bc4cb0 100755 --- a/ci/install-libraries.sh +++ b/ci/install-libraries.sh @@ -6,7 +6,6 @@ source default-config ./install-os-packages.sh libssl-dev ./install-os-packages.sh libicu-dev ./install-os-packages.sh libreadline-dev -./install-os-packages.sh libunixodbc-dev if [[ "$ENABLE_EMBEDDED_COMPILER" == 1 && "$USE_LLVM_LIBRARIES_FROM_SYSTEM" == 1 ]]; then ./install-os-packages.sh llvm-libs-5.0 diff --git a/ci/install-os-packages.sh b/ci/install-os-packages.sh index e5055c660eb..66644ba4616 100755 --- a/ci/install-os-packages.sh +++ b/ci/install-os-packages.sh @@ -52,9 +52,6 @@ case $PACKAGE_MANAGER in libreadline-dev) $SUDO apt-get install -y libreadline-dev ;; - libunixodbc-dev) - $SUDO apt-get install -y unixodbc-dev - ;; llvm-libs*) $SUDO apt-get install -y ${WHAT/llvm-libs/liblld}-dev ${WHAT/llvm-libs/libclang}-dev ;; @@ -103,9 +100,6 @@ case $PACKAGE_MANAGER in libreadline-dev) $SUDO yum install -y readline-devel ;; - libunixodbc-dev) - $SUDO yum install -y unixODBC-devel libtool-ltdl-devel - ;; *) echo "Unknown package"; exit 1; ;; @@ -148,9 +142,6 @@ case $PACKAGE_MANAGER in libreadline-dev) $SUDO pkg install -y readline ;; - libunixodbc-dev) - $SUDO pkg install -y unixODBC libltdl - ;; *) echo "Unknown package"; exit 1; ;; diff --git a/cmake/find_ltdl.cmake b/cmake/find_ltdl.cmake index 935de0d4124..18003618dbd 100644 --- a/cmake/find_ltdl.cmake +++ b/cmake/find_ltdl.cmake @@ -1,3 +1,5 @@ -set (LTDL_PATHS "/usr/local/opt/libtool/lib") -find_library (LTDL_LIBRARY ltdl PATHS ${LTDL_PATHS}) -message (STATUS "Using ltdl: ${LTDL_LIBRARY}") +if (ENABLE_ODBC AND NOT USE_INTERNAL_ODBC_LIBRARY) + set (LTDL_PATHS "/usr/local/opt/libtool/lib") + find_library (LTDL_LIBRARY ltdl PATHS ${LTDL_PATHS}) + message (STATUS "Using ltdl: ${LTDL_LIBRARY}") +endif () diff --git a/cmake/find_odbc.cmake b/cmake/find_odbc.cmake index 338108910bf..3d481d34f0d 100644 --- a/cmake/find_odbc.cmake +++ b/cmake/find_odbc.cmake @@ -13,54 +13,70 @@ # This module defines # ODBC_INCLUDE_DIRECTORIES, where to find sql.h # ODBC_LIBRARIES, the libraries to link against to use ODBC -# ODBC_FOUND. If false, you cannot build anything that requires MySQL. +# ODBC_FOUND. If false, you cannot build anything that requires ODBC. -find_path(ODBC_INCLUDE_DIRECTORIES - NAMES sql.h - HINTS - /usr/include - /usr/include/odbc - /usr/include/iodbc - /usr/local/include - /usr/local/include/odbc - /usr/local/include/iodbc - /usr/local/odbc/include - /usr/local/iodbc/include - "C:/Program Files/ODBC/include" - "C:/Program Files/Microsoft SDKs/Windows/v7.0/include" - "C:/Program Files/Microsoft SDKs/Windows/v6.0a/include" - "C:/ODBC/include" - DOC "Specify the directory containing sql.h." -) +option (ENABLE_ODBC "Enable ODBC" ON) +option (USE_INTERNAL_ODBC_LIBRARY "Set to FALSE to use system odbc library instead of bundled" ${NOT_UNBUNDLED}) +if (USE_INTERNAL_ODBC_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/unixodbc/README") + message (WARNING "submodule contrib/unixodbc is missing. to fix try run: \n git submodule update --init --recursive") + set (USE_INTERNAL_ODBC_LIBRARY 0) +endif () -find_library(ODBC_LIBRARIES - NAMES iodbc odbc iodbcinst odbcinst odbc32 - HINTS - /usr/lib - /usr/lib/odbc - /usr/lib/iodbc - /usr/local/lib - /usr/local/lib/odbc - /usr/local/lib/iodbc - /usr/local/odbc/lib - /usr/local/iodbc/lib - "C:/Program Files/ODBC/lib" - "C:/ODBC/lib/debug" - "C:/Program Files (x86)/Microsoft SDKs/Windows/v7.0A/Lib" - DOC "Specify the ODBC driver manager library here." -) +if (ENABLE_ODBC) + if (USE_INTERNAL_ODBC_LIBRARY) + set (ODBC_LIBRARIES unixodbc) + set (ODBC_FOUND 1) + set (USE_ODBC 1) + else () + find_path(ODBC_INCLUDE_DIRECTORIES + NAMES sql.h + HINTS + /usr/include + /usr/include/iodbc + /usr/include/odbc + /usr/local/include + /usr/local/include/iodbc + /usr/local/include/odbc + /usr/local/iodbc/include + /usr/local/odbc/include + "C:/Program Files/ODBC/include" + "C:/Program Files/Microsoft SDKs/Windows/v7.0/include" + "C:/Program Files/Microsoft SDKs/Windows/v6.0a/include" + "C:/ODBC/include" + DOC "Specify the directory containing sql.h." + ) -# MinGW find usually fails -if(MINGW) - set(ODBC_INCLUDE_DIRECTORIES ".") - set(ODBC_LIBRARIES odbc32) -endif() - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(ODBC - DEFAULT_MSG - ODBC_INCLUDE_DIRECTORIES - ODBC_LIBRARIES - ) + find_library(ODBC_LIBRARIES + NAMES iodbc odbc iodbcinst odbcinst odbc32 + HINTS + /usr/lib + /usr/lib/iodbc + /usr/lib/odbc + /usr/local/lib + /usr/local/lib/iodbc + /usr/local/lib/odbc + /usr/local/iodbc/lib + /usr/local/odbc/lib + "C:/Program Files/ODBC/lib" + "C:/ODBC/lib/debug" + "C:/Program Files (x86)/Microsoft SDKs/Windows/v7.0A/Lib" + DOC "Specify the ODBC driver manager library here." + ) -mark_as_advanced(ODBC_FOUND ODBC_LIBRARIES ODBC_INCLUDE_DIRECTORIES) + # MinGW find usually fails + if(MINGW) + set(ODBC_INCLUDE_DIRECTORIES ".") + set(ODBC_LIBRARIES odbc32) + endif() + + include(FindPackageHandleStandardArgs) + find_package_handle_standard_args(ODBC + DEFAULT_MSG + ODBC_INCLUDE_DIRECTORIES + ODBC_LIBRARIES) + + mark_as_advanced(ODBC_FOUND ODBC_LIBRARIES ODBC_INCLUDE_DIRECTORIES) + endif () +endif () + +message (STATUS "Using odbc: ${ODBC_INCLUDE_DIRECTORIES} : ${ODBC_LIBRARIES}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 1ee83056110..62fe516bba0 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -132,6 +132,10 @@ if (USE_INTERNAL_RDKAFKA_LIBRARY) target_include_directories(rdkafka PRIVATE BEFORE ${ZLIB_INCLUDE_DIR}) endif () +if (ENABLE_ODBC AND USE_INTERNAL_ODBC_LIBRARY) + add_subdirectory (unixodbc-cmake) +endif () + if (USE_INTERNAL_CAPNP_LIBRARY) set (BUILD_TESTING 0 CACHE INTERNAL "") set (_save ${CMAKE_CXX_EXTENSIONS}) diff --git a/contrib/unixodbc b/contrib/unixodbc new file mode 160000 index 00000000000..b0ad30f7f62 --- /dev/null +++ b/contrib/unixodbc @@ -0,0 +1 @@ +Subproject commit b0ad30f7f6289c12b76f04bfb9d466374bb32168 diff --git a/contrib/unixodbc-cmake/CMakeLists.txt b/contrib/unixodbc-cmake/CMakeLists.txt new file mode 100644 index 00000000000..fa147eb3861 --- /dev/null +++ b/contrib/unixodbc-cmake/CMakeLists.txt @@ -0,0 +1,287 @@ +set(ODBC_SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/unixodbc) +set(ODBC_BINARY_DIR ${CMAKE_BINARY_DIR}/contrib/unixodbc) + + +set(SRCS +${ODBC_SOURCE_DIR}/libltdl/lt__alloc.c +${ODBC_SOURCE_DIR}/libltdl/lt__strl.c +${ODBC_SOURCE_DIR}/libltdl/ltdl.c +${ODBC_SOURCE_DIR}/libltdl/lt_dlloader.c +${ODBC_SOURCE_DIR}/libltdl/slist.c +${ODBC_SOURCE_DIR}/libltdl/lt_error.c +${ODBC_SOURCE_DIR}/libltdl/loaders/dlopen.c +${ODBC_SOURCE_DIR}/libltdl/loaders/preopen.c +#${ODBC_SOURCE_DIR}/libltdl/lt__dirent.c +#${ODBC_SOURCE_DIR}/libltdl/lt__argz.c +#${ODBC_SOURCE_DIR}/libltdl/loaders/dld_link.c +#${ODBC_SOURCE_DIR}/libltdl/loaders/load_add_on.c +#${ODBC_SOURCE_DIR}/libltdl/loaders/shl_load.c +#${ODBC_SOURCE_DIR}/libltdl/loaders/loadlibrary.c +#${ODBC_SOURCE_DIR}/libltdl/loaders/dyld.c + +# This file is generated by 'libtool' inside libltdl directory and then removed. +${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/libltdl/libltdlcS.c +) + +add_library(ltdl STATIC ${SRCS}) + +target_include_directories(ltdl PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/libltdl) +target_include_directories(ltdl PUBLIC ${ODBC_SOURCE_DIR}/libltdl) +target_include_directories(ltdl PUBLIC ${ODBC_SOURCE_DIR}/libltdl/libltdl) + +target_compile_definitions(ltdl PRIVATE -DHAVE_CONFIG_H -DLTDL -DLTDLOPEN=libltdlc) + +target_compile_options(ltdl PRIVATE -Wno-constant-logical-operand -Wno-unknown-warning-option -O2) + + +set(SRCS +${ODBC_SOURCE_DIR}/DriverManager/__attribute.c +${ODBC_SOURCE_DIR}/DriverManager/__connection.c +${ODBC_SOURCE_DIR}/DriverManager/__handles.c +${ODBC_SOURCE_DIR}/DriverManager/__info.c +${ODBC_SOURCE_DIR}/DriverManager/__stats.c +${ODBC_SOURCE_DIR}/DriverManager/SQLAllocConnect.c +${ODBC_SOURCE_DIR}/DriverManager/SQLAllocEnv.c +${ODBC_SOURCE_DIR}/DriverManager/SQLAllocHandle.c +${ODBC_SOURCE_DIR}/DriverManager/SQLAllocHandleStd.c +${ODBC_SOURCE_DIR}/DriverManager/SQLAllocStmt.c +${ODBC_SOURCE_DIR}/DriverManager/SQLBindCol.c +${ODBC_SOURCE_DIR}/DriverManager/SQLBindParam.c +${ODBC_SOURCE_DIR}/DriverManager/SQLBindParameter.c +${ODBC_SOURCE_DIR}/DriverManager/SQLBrowseConnect.c +${ODBC_SOURCE_DIR}/DriverManager/SQLBrowseConnectW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLBulkOperations.c +${ODBC_SOURCE_DIR}/DriverManager/SQLCancel.c +${ODBC_SOURCE_DIR}/DriverManager/SQLCancelHandle.c +${ODBC_SOURCE_DIR}/DriverManager/SQLCloseCursor.c +${ODBC_SOURCE_DIR}/DriverManager/SQLColAttribute.c +${ODBC_SOURCE_DIR}/DriverManager/SQLColAttributes.c +${ODBC_SOURCE_DIR}/DriverManager/SQLColAttributesW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLColAttributeW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLColumnPrivileges.c +${ODBC_SOURCE_DIR}/DriverManager/SQLColumnPrivilegesW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLColumns.c +${ODBC_SOURCE_DIR}/DriverManager/SQLColumnsW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLConnect.c +${ODBC_SOURCE_DIR}/DriverManager/SQLConnectW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLCopyDesc.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDataSources.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDataSourcesW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDescribeCol.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDescribeColW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDescribeParam.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDisconnect.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDriverConnect.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDriverConnectW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDrivers.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDriversW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLEndTran.c +${ODBC_SOURCE_DIR}/DriverManager/SQLError.c +${ODBC_SOURCE_DIR}/DriverManager/SQLErrorW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLExecDirect.c +${ODBC_SOURCE_DIR}/DriverManager/SQLExecDirectW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLExecute.c +${ODBC_SOURCE_DIR}/DriverManager/SQLExtendedFetch.c +${ODBC_SOURCE_DIR}/DriverManager/SQLFetch.c +${ODBC_SOURCE_DIR}/DriverManager/SQLFetchScroll.c +${ODBC_SOURCE_DIR}/DriverManager/SQLForeignKeys.c +${ODBC_SOURCE_DIR}/DriverManager/SQLForeignKeysW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLFreeConnect.c +${ODBC_SOURCE_DIR}/DriverManager/SQLFreeEnv.c +${ODBC_SOURCE_DIR}/DriverManager/SQLFreeHandle.c +${ODBC_SOURCE_DIR}/DriverManager/SQLFreeStmt.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetConnectAttr.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetConnectAttrW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetConnectOption.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetConnectOptionW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetCursorName.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetCursorNameW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetData.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetDescField.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetDescFieldW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetDescRec.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetDescRecW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetDiagField.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetDiagFieldW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetDiagRec.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetDiagRecW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetEnvAttr.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetFunctions.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetInfo.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetInfoW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetStmtAttr.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetStmtAttrW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetStmtOption.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetTypeInfo.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetTypeInfoW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLMoreResults.c +${ODBC_SOURCE_DIR}/DriverManager/SQLNativeSql.c +${ODBC_SOURCE_DIR}/DriverManager/SQLNativeSqlW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLNumParams.c +${ODBC_SOURCE_DIR}/DriverManager/SQLNumResultCols.c +${ODBC_SOURCE_DIR}/DriverManager/SQLParamData.c +${ODBC_SOURCE_DIR}/DriverManager/SQLParamOptions.c +${ODBC_SOURCE_DIR}/DriverManager/SQLPrepare.c +${ODBC_SOURCE_DIR}/DriverManager/SQLPrepareW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLPrimaryKeys.c +${ODBC_SOURCE_DIR}/DriverManager/SQLPrimaryKeysW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLProcedureColumns.c +${ODBC_SOURCE_DIR}/DriverManager/SQLProcedureColumnsW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLProcedures.c +${ODBC_SOURCE_DIR}/DriverManager/SQLProceduresW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLPutData.c +${ODBC_SOURCE_DIR}/DriverManager/SQLRowCount.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetConnectAttr.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetConnectAttrW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetConnectOption.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetConnectOptionW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetCursorName.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetCursorNameW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetDescField.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetDescFieldW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetDescRec.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetEnvAttr.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetParam.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetPos.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetScrollOptions.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetStmtAttr.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetStmtAttrW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetStmtOption.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetStmtOptionW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSpecialColumns.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSpecialColumnsW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLStatistics.c +${ODBC_SOURCE_DIR}/DriverManager/SQLStatisticsW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLTablePrivileges.c +${ODBC_SOURCE_DIR}/DriverManager/SQLTablePrivilegesW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLTables.c +${ODBC_SOURCE_DIR}/DriverManager/SQLTablesW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLTransact.c + +${ODBC_SOURCE_DIR}/odbcinst/_logging.c +${ODBC_SOURCE_DIR}/odbcinst/_odbcinst_ConfigModeINI.c +${ODBC_SOURCE_DIR}/odbcinst/ODBCINSTConstructProperties.c +${ODBC_SOURCE_DIR}/odbcinst/ODBCINSTDestructProperties.c +${ODBC_SOURCE_DIR}/odbcinst/_odbcinst_GetEntries.c +${ODBC_SOURCE_DIR}/odbcinst/_odbcinst_GetSections.c +${ODBC_SOURCE_DIR}/odbcinst/ODBCINSTSetProperty.c +${ODBC_SOURCE_DIR}/odbcinst/_odbcinst_SystemINI.c +${ODBC_SOURCE_DIR}/odbcinst/_odbcinst_UserINI.c +${ODBC_SOURCE_DIR}/odbcinst/ODBCINSTValidateProperties.c +${ODBC_SOURCE_DIR}/odbcinst/ODBCINSTValidateProperty.c +${ODBC_SOURCE_DIR}/odbcinst/SQLConfigDataSource.c +${ODBC_SOURCE_DIR}/odbcinst/SQLConfigDriver.c +${ODBC_SOURCE_DIR}/odbcinst/SQLCreateDataSource.c +${ODBC_SOURCE_DIR}/odbcinst/_SQLDriverConnectPrompt.c +${ODBC_SOURCE_DIR}/odbcinst/SQLGetAvailableDrivers.c +${ODBC_SOURCE_DIR}/odbcinst/SQLGetConfigMode.c +${ODBC_SOURCE_DIR}/odbcinst/_SQLGetInstalledDrivers.c +${ODBC_SOURCE_DIR}/odbcinst/SQLGetInstalledDrivers.c +${ODBC_SOURCE_DIR}/odbcinst/SQLGetPrivateProfileString.c +${ODBC_SOURCE_DIR}/odbcinst/SQLGetTranslator.c +${ODBC_SOURCE_DIR}/odbcinst/SQLInstallDriverEx.c +${ODBC_SOURCE_DIR}/odbcinst/SQLInstallDriverManager.c +${ODBC_SOURCE_DIR}/odbcinst/SQLInstallerError.c +${ODBC_SOURCE_DIR}/odbcinst/SQLInstallODBC.c +${ODBC_SOURCE_DIR}/odbcinst/SQLInstallTranslatorEx.c +${ODBC_SOURCE_DIR}/odbcinst/SQLManageDataSources.c +${ODBC_SOURCE_DIR}/odbcinst/SQLPostInstallerError.c +${ODBC_SOURCE_DIR}/odbcinst/SQLReadFileDSN.c +${ODBC_SOURCE_DIR}/odbcinst/SQLRemoveDriver.c +${ODBC_SOURCE_DIR}/odbcinst/SQLRemoveDriverManager.c +${ODBC_SOURCE_DIR}/odbcinst/SQLRemoveDSNFromIni.c +${ODBC_SOURCE_DIR}/odbcinst/SQLRemoveTranslator.c +${ODBC_SOURCE_DIR}/odbcinst/SQLSetConfigMode.c +${ODBC_SOURCE_DIR}/odbcinst/SQLValidDSN.c +${ODBC_SOURCE_DIR}/odbcinst/SQLWriteDSNToIni.c +${ODBC_SOURCE_DIR}/odbcinst/SQLWriteFileDSN.c +${ODBC_SOURCE_DIR}/odbcinst/_SQLWriteInstalledDrivers.c +${ODBC_SOURCE_DIR}/odbcinst/SQLWritePrivateProfileString.c + +${ODBC_SOURCE_DIR}/ini/iniAllTrim.c +${ODBC_SOURCE_DIR}/ini/iniAppend.c +${ODBC_SOURCE_DIR}/ini/iniClose.c +${ODBC_SOURCE_DIR}/ini/iniCommit.c +${ODBC_SOURCE_DIR}/ini/iniCursor.c +${ODBC_SOURCE_DIR}/ini/iniDelete.c +${ODBC_SOURCE_DIR}/ini/_iniDump.c +${ODBC_SOURCE_DIR}/ini/iniElement.c +${ODBC_SOURCE_DIR}/ini/iniElementCount.c +${ODBC_SOURCE_DIR}/ini/iniGetBookmark.c +${ODBC_SOURCE_DIR}/ini/iniGotoBookmark.c +${ODBC_SOURCE_DIR}/ini/iniObject.c +${ODBC_SOURCE_DIR}/ini/iniObjectDelete.c +${ODBC_SOURCE_DIR}/ini/iniObjectEOL.c +${ODBC_SOURCE_DIR}/ini/iniObjectFirst.c +${ODBC_SOURCE_DIR}/ini/iniObjectInsert.c +${ODBC_SOURCE_DIR}/ini/iniObjectLast.c +${ODBC_SOURCE_DIR}/ini/iniObjectNext.c +${ODBC_SOURCE_DIR}/ini/_iniObjectRead.c +${ODBC_SOURCE_DIR}/ini/iniObjectSeek.c +${ODBC_SOURCE_DIR}/ini/iniObjectSeekSure.c +${ODBC_SOURCE_DIR}/ini/iniObjectUpdate.c +${ODBC_SOURCE_DIR}/ini/iniOpen.c +${ODBC_SOURCE_DIR}/ini/iniProperty.c +${ODBC_SOURCE_DIR}/ini/iniPropertyDelete.c +${ODBC_SOURCE_DIR}/ini/iniPropertyEOL.c +${ODBC_SOURCE_DIR}/ini/iniPropertyFirst.c +${ODBC_SOURCE_DIR}/ini/iniPropertyInsert.c +${ODBC_SOURCE_DIR}/ini/iniPropertyLast.c +${ODBC_SOURCE_DIR}/ini/iniPropertyNext.c +${ODBC_SOURCE_DIR}/ini/_iniPropertyRead.c +${ODBC_SOURCE_DIR}/ini/iniPropertySeek.c +${ODBC_SOURCE_DIR}/ini/iniPropertySeekSure.c +${ODBC_SOURCE_DIR}/ini/iniPropertyUpdate.c +${ODBC_SOURCE_DIR}/ini/iniPropertyValue.c +${ODBC_SOURCE_DIR}/ini/_iniScanUntilObject.c +${ODBC_SOURCE_DIR}/ini/iniToUpper.c +${ODBC_SOURCE_DIR}/ini/iniValue.c + +${ODBC_SOURCE_DIR}/log/logClear.c +${ODBC_SOURCE_DIR}/log/logClose.c +${ODBC_SOURCE_DIR}/log/_logFreeMsg.c +${ODBC_SOURCE_DIR}/log/logOn.c +${ODBC_SOURCE_DIR}/log/logOpen.c +${ODBC_SOURCE_DIR}/log/logPeekMsg.c +${ODBC_SOURCE_DIR}/log/logPopMsg.c +${ODBC_SOURCE_DIR}/log/logPushMsg.c + +${ODBC_SOURCE_DIR}/lst/_lstAdjustCurrent.c +${ODBC_SOURCE_DIR}/lst/lstAppend.c +${ODBC_SOURCE_DIR}/lst/lstClose.c +${ODBC_SOURCE_DIR}/lst/lstDelete.c +${ODBC_SOURCE_DIR}/lst/_lstDump.c +${ODBC_SOURCE_DIR}/lst/lstEOL.c +${ODBC_SOURCE_DIR}/lst/lstFirst.c +${ODBC_SOURCE_DIR}/lst/_lstFreeItem.c +${ODBC_SOURCE_DIR}/lst/lstGetBookMark.c +${ODBC_SOURCE_DIR}/lst/lstGet.c +${ODBC_SOURCE_DIR}/lst/lstGotoBookMark.c +${ODBC_SOURCE_DIR}/lst/lstGoto.c +${ODBC_SOURCE_DIR}/lst/lstInsert.c +${ODBC_SOURCE_DIR}/lst/lstLast.c +${ODBC_SOURCE_DIR}/lst/lstNext.c +${ODBC_SOURCE_DIR}/lst/_lstNextValidItem.c +${ODBC_SOURCE_DIR}/lst/lstOpen.c +${ODBC_SOURCE_DIR}/lst/lstOpenCursor.c +${ODBC_SOURCE_DIR}/lst/lstPrev.c +${ODBC_SOURCE_DIR}/lst/_lstPrevValidItem.c +${ODBC_SOURCE_DIR}/lst/lstSeek.c +${ODBC_SOURCE_DIR}/lst/lstSeekItem.c +${ODBC_SOURCE_DIR}/lst/lstSet.c +${ODBC_SOURCE_DIR}/lst/lstSetFreeFunc.c +${ODBC_SOURCE_DIR}/lst/_lstVisible.c +) + +add_library(unixodbc STATIC ${SRCS}) + +target_link_libraries(unixodbc ltdl) + +# SYSTEM_FILE_PATH was changed to /etc + +target_include_directories(unixodbc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64) +target_include_directories(unixodbc PUBLIC ${ODBC_SOURCE_DIR}/include) + +target_compile_definitions(unixodbc PRIVATE -DHAVE_CONFIG_H) + +target_compile_options(unixodbc PRIVATE -Wno-dangling-else -Wno-parentheses -Wno-unknown-warning-option -O2) diff --git a/contrib/unixodbc-cmake/linux_x86_64/config.h b/contrib/unixodbc-cmake/linux_x86_64/config.h new file mode 100644 index 00000000000..d80a4da4665 --- /dev/null +++ b/contrib/unixodbc-cmake/linux_x86_64/config.h @@ -0,0 +1,496 @@ +/* config.h. Generated from config.h.in by configure. */ +/* config.h.in. Generated from configure.ac by autoheader. */ + +/* Encoding to use for CHAR */ +#define ASCII_ENCODING "auto-search" + +/* Install bindir */ +#define BIN_PREFIX "/usr/local/bin" + +/* Use a semaphore to allow ODBCConfig to display running counts */ +/* #undef COLLECT_STATS */ + +/* Define to one of `_getb67', `GETB67', `getb67' for Cray-2 and Cray-YMP + systems. This function is required for `alloca.c' support on those systems. + */ +/* #undef CRAY_STACKSEG_END */ + +/* Define to 1 if using `alloca.c'. */ +/* #undef C_ALLOCA */ + +/* Lib directory */ +#define DEFLIB_PATH "/usr/local/lib" + +/* Using ini cacheing */ +#define ENABLE_INI_CACHING /**/ + +/* Install exec_prefix */ +#define EXEC_PREFIX "/usr/local" + +/* Disable the precise but slow checking of the validity of handles */ +/* #undef FAST_HANDLE_VALIDATE */ + +/* Define to 1 if you have `alloca', as a function or macro. */ +#define HAVE_ALLOCA 1 + +/* Define to 1 if you have and it should be used (not on Ultrix). + */ +#define HAVE_ALLOCA_H 1 + +/* Define to 1 if you have the `argz_add' function. */ +#define HAVE_ARGZ_ADD 1 + +/* Define to 1 if you have the `argz_append' function. */ +#define HAVE_ARGZ_APPEND 1 + +/* Define to 1 if you have the `argz_count' function. */ +#define HAVE_ARGZ_COUNT 1 + +/* Define to 1 if you have the `argz_create_sep' function. */ +#define HAVE_ARGZ_CREATE_SEP 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARGZ_H 1 + +/* Define to 1 if you have the `argz_insert' function. */ +#define HAVE_ARGZ_INSERT 1 + +/* Define to 1 if you have the `argz_next' function. */ +#define HAVE_ARGZ_NEXT 1 + +/* Define to 1 if you have the `argz_stringify' function. */ +#define HAVE_ARGZ_STRINGIFY 1 + +/* Define to 1 if you have the `atoll' function. */ +#define HAVE_ATOLL 1 + +/* Define to 1 if you have the `closedir' function. */ +#define HAVE_CLOSEDIR 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_CRYPT_H 1 + +/* Define to 1 if you have the declaration of `cygwin_conv_path', and to 0 if + you don't. */ +/* #undef HAVE_DECL_CYGWIN_CONV_PATH */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +#define HAVE_DIRENT_H 1 + +/* Define if you have the GNU dld library. */ +/* #undef HAVE_DLD */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_DLD_H */ + +/* Define to 1 if you have the `dlerror' function. */ +#define HAVE_DLERROR 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_DL_H */ + +/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */ +/* #undef HAVE_DOPRNT */ + +/* Define if you have the _dyld_func_lookup function. */ +/* #undef HAVE_DYLD */ + +/* Define to 1 if you have the `endpwent' function. */ +#define HAVE_ENDPWENT 1 + +/* Define to 1 if the system has the type `error_t'. */ +#define HAVE_ERROR_T 1 + +/* Define to 1 if you have the `ftime' function. */ +#define HAVE_FTIME 1 + +/* Define to 1 if you have the `ftok' function. */ +/* #undef HAVE_FTOK */ + +/* Define to 1 if you have the `getpwuid' function. */ +#define HAVE_GETPWUID 1 + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY 1 + +/* Define to 1 if you have the `getuid' function. */ +#define HAVE_GETUID 1 + +/* Define if you have the iconv() function. */ +#define HAVE_ICONV 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define if you have and nl_langinfo(CODESET). */ +#define HAVE_LANGINFO_CODESET 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_LANGINFO_H 1 + +/* Add -lcrypt to lib list */ +#define HAVE_LIBCRYPT /**/ + +/* Define if you have the libdl library or equivalent. */ +#define HAVE_LIBDL 1 + +/* Define if libdlloader will be built on this platform */ +#define HAVE_LIBDLLOADER 1 + +/* Use the -lpth thread library */ +/* #undef HAVE_LIBPTH */ + +/* Use -lpthread threading lib */ +#define HAVE_LIBPTHREAD 1 + +/* Use the -lthread threading lib */ +/* #undef HAVE_LIBTHREAD */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_LOCALE_H 1 + +/* Use rentrant version of localtime */ +#define HAVE_LOCALTIME_R 1 + +/* Define if you have long long */ +#define HAVE_LONG_LONG 1 + +/* Define this if a modern libltdl is already installed */ +#define HAVE_LTDL 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_MACH_O_DYLD_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_MALLOC_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_MSQL_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. */ +/* #undef HAVE_NDIR_H */ + +/* Define to 1 if you have the `nl_langinfo' function. */ +#define HAVE_NL_LANGINFO 1 + +/* Define to 1 if you have the `opendir' function. */ +#define HAVE_OPENDIR 1 + +/* Define if libtool can extract symbol lists from object files. */ +#define HAVE_PRELOADED_SYMBOLS 1 + +/* Define to 1 if the system has the type `ptrdiff_t'. */ +#define HAVE_PTRDIFF_T 1 + +/* Define to 1 if you have the `putenv' function. */ +#define HAVE_PUTENV 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PWD_H 1 + +/* Define to 1 if you have the `readdir' function. */ +#define HAVE_READDIR 1 + +/* Add readline support */ +#define HAVE_READLINE 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_READLINE_HISTORY_H 1 + +/* Use the scandir lib */ +/* #undef HAVE_SCANDIR */ + +/* Define to 1 if you have the `semget' function. */ +/* #undef HAVE_SEMGET */ + +/* Define to 1 if you have the `semop' function. */ +/* #undef HAVE_SEMOP */ + +/* Define to 1 if you have the `setenv' function. */ +#define HAVE_SETENV 1 + +/* Define to 1 if you have the `setlocale' function. */ +#define HAVE_SETLOCALE 1 + +/* Define if you have the shl_load function. */ +/* #undef HAVE_SHL_LOAD */ + +/* Define to 1 if you have the `shmget' function. */ +/* #undef HAVE_SHMGET */ + +/* Define to 1 if you have the `snprintf' function. */ +/* #undef HAVE_SNPRINTF */ + +/* Define to 1 if you have the `socket' function. */ +#define HAVE_SOCKET 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDARG_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDDEF_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the `strcasecmp' function. */ +#define HAVE_STRCASECMP 1 + +/* Define to 1 if you have the `strchr' function. */ +#define HAVE_STRCHR 1 + +/* Define to 1 if you have the `strdup' function. */ +#define HAVE_STRDUP 1 + +/* Define to 1 if you have the `stricmp' function. */ +/* #undef HAVE_STRICMP */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the `strlcat' function. */ +/* #undef HAVE_STRLCAT */ + +/* Define to 1 if you have the `strlcpy' function. */ +/* #undef HAVE_STRLCPY */ + +/* Define to 1 if you have the `strncasecmp' function. */ +#define HAVE_STRNCASECMP 1 + +/* Define to 1 if you have the `strnicmp' function. */ +/* #undef HAVE_STRNICMP */ + +/* Define to 1 if you have the `strstr' function. */ +#define HAVE_STRSTR 1 + +/* Define to 1 if you have the `strtol' function. */ +#define HAVE_STRTOL 1 + +/* Define to 1 if you have the `strtoll' function. */ +#define HAVE_STRTOLL 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYNCH_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_DIR_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_DL_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_MALLOC_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SEM_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIMEB_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the `time' function. */ +#define HAVE_TIME 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_VARARGS_H */ + +/* Define to 1 if you have the `vprintf' function. */ +#define HAVE_VPRINTF 1 + +/* Define to 1 if you have the `vsnprintf' function. */ +#define HAVE_VSNPRINTF 1 + +/* This value is set to 1 to indicate that the system argz facility works */ +#define HAVE_WORKING_ARGZ 1 + +/* Define as const if the declaration of iconv() needs const. */ +#define ICONV_CONST + +/* Install includedir */ +#define INCLUDE_PREFIX "/usr/local/include" + +/* Lib directory */ +#define LIB_PREFIX "/usr/local/lib" + +/* Define if the OS needs help to load dependent libraries for dlopen(). */ +/* #undef LTDL_DLOPEN_DEPLIBS */ + +/* Define to the system default library search path. */ +#define LT_DLSEARCH_PATH "/lib:/usr/lib:/usr/lib/x86_64-linux-gnu/libfakeroot:/usr/local/lib:/usr/local/lib/x86_64-linux-gnu:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/lib32:/usr/lib32" + +/* The archive extension */ +#define LT_LIBEXT "a" + +/* The archive prefix */ +#define LT_LIBPREFIX "lib" + +/* Define to the extension used for runtime loadable modules, say, ".so". */ +#define LT_MODULE_EXT ".so" + +/* Define to the name of the environment variable that determines the run-time + module search path. */ +#define LT_MODULE_PATH_VAR "LD_LIBRARY_PATH" + +/* Define to the sub-directory where libtool stores uninstalled libraries. */ +#define LT_OBJDIR ".libs/" + +/* Define to the shared library suffix, say, ".dylib". */ +/* #undef LT_SHARED_EXT */ + +/* Define to the shared archive member specification, say "(shr.o)". */ +/* #undef LT_SHARED_LIB_MEMBER */ + +/* Define if you need semundo union */ +/* #undef NEED_SEMUNDO_UNION */ + +/* Define if dlsym() requires a leading underscore in symbol names. */ +/* #undef NEED_USCORE */ + +/* Using OSX */ +/* #undef OSXHEADER */ + +/* Name of package */ +#define PACKAGE "unixODBC" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "nick@unixodbc.org" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "unixODBC" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "unixODBC 2.3.6" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "unixODBC" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "2.3.6" + +/* Platform is 64 bit */ +#define PLATFORM64 /**/ + +/* Install prefix */ +#define PREFIX "/usr/local" + +/* Using QNX */ +/* #undef QNX_LIBLTDL */ + +/* Shared lib extension */ +#define SHLIBEXT ".so" + +/* The size of `long', as computed by sizeof. */ +#define SIZEOF_LONG 8 + +/* The size of `long int', as computed by sizeof. */ +#define SIZEOF_LONG_INT 8 + +/* If using the C implementation of alloca, define if you know the + direction of stack growth for your system; otherwise it will be + automatically deduced at runtime. + STACK_DIRECTION > 0 => grows toward higher addresses + STACK_DIRECTION < 0 => grows toward lower addresses + STACK_DIRECTION = 0 => direction of growth unknown */ +/* #undef STACK_DIRECTION */ + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* don't include unixODBC prefix in driver error messages */ +#define STRICT_ODBC_ERROR /**/ + +/* System file path */ +#define SYSTEM_FILE_PATH "/etc" + +/* Lib path */ +#define SYSTEM_LIB_PATH "/usr/local/lib" + +/* Define to 1 if you can safely include both and . */ +#define TIME_WITH_SYS_TIME 1 + +/* Define to 1 if your declares `struct tm'. */ +/* #undef TM_IN_SYS_TIME */ + +/* Encoding to use for UNICODE */ +#define UNICODE_ENCODING "auto-search" + +/* Flag that we are not using another DM */ +#define UNIXODBC /**/ + +/* We are building inside the unixODBC source tree */ +#define UNIXODBC_SOURCE /**/ + +/* Version number of package */ +#define VERSION "2.3.6" + +/* Work with IBM drivers that use 32 bit handles on 64 bit platforms */ +/* #undef WITH_HANDLE_REDIRECT */ + +/* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a + `char[]'. */ +/* #undef YYTEXT_POINTER */ + +/* Build flag for AIX */ +/* #undef _ALL_SOURCE */ + +/* Build flag for AIX */ +/* #undef _LONG_LONG */ + +/* Build flag for AIX */ +/* #undef _THREAD_SAFE */ + +/* Define so that glibc/gnulib argp.h does not typedef error_t. */ +/* #undef __error_t_defined */ + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Define to a type to use for 'error_t' if it is not otherwise available. */ +/* #undef error_t */ + +/* Define to `int' if doesn't define. */ +/* #undef gid_t */ + +/* Define to `unsigned int' if does not define. */ +/* #undef size_t */ + +/* Define to `int' if doesn't define. */ +/* #undef uid_t */ diff --git a/contrib/unixodbc-cmake/linux_x86_64/libltdl/config.h b/contrib/unixodbc-cmake/linux_x86_64/libltdl/config.h new file mode 100644 index 00000000000..194779b2b98 --- /dev/null +++ b/contrib/unixodbc-cmake/linux_x86_64/libltdl/config.h @@ -0,0 +1,181 @@ +/* config.h. Generated from config-h.in by configure. */ +/* config-h.in. Generated from configure.ac by autoheader. */ + +/* Define to 1 if you have the `argz_add' function. */ +#define HAVE_ARGZ_ADD 1 + +/* Define to 1 if you have the `argz_append' function. */ +#define HAVE_ARGZ_APPEND 1 + +/* Define to 1 if you have the `argz_count' function. */ +#define HAVE_ARGZ_COUNT 1 + +/* Define to 1 if you have the `argz_create_sep' function. */ +#define HAVE_ARGZ_CREATE_SEP 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARGZ_H 1 + +/* Define to 1 if you have the `argz_insert' function. */ +#define HAVE_ARGZ_INSERT 1 + +/* Define to 1 if you have the `argz_next' function. */ +#define HAVE_ARGZ_NEXT 1 + +/* Define to 1 if you have the `argz_stringify' function. */ +#define HAVE_ARGZ_STRINGIFY 1 + +/* Define to 1 if you have the `closedir' function. */ +#define HAVE_CLOSEDIR 1 + +/* Define to 1 if you have the declaration of `cygwin_conv_path', and to 0 if + you don't. */ +/* #undef HAVE_DECL_CYGWIN_CONV_PATH */ + +/* Define to 1 if you have the header file. */ +#define HAVE_DIRENT_H 1 + +/* Define if you have the GNU dld library. */ +/* #undef HAVE_DLD */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_DLD_H */ + +/* Define to 1 if you have the `dlerror' function. */ +#define HAVE_DLERROR 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_DL_H */ + +/* Define if you have the _dyld_func_lookup function. */ +/* #undef HAVE_DYLD */ + +/* Define to 1 if the system has the type `error_t'. */ +#define HAVE_ERROR_T 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define if you have the libdl library or equivalent. */ +#define HAVE_LIBDL 1 + +/* Define if libdlloader will be built on this platform */ +#define HAVE_LIBDLLOADER 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_MACH_O_DYLD_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the `opendir' function. */ +#define HAVE_OPENDIR 1 + +/* Define if libtool can extract symbol lists from object files. */ +#define HAVE_PRELOADED_SYMBOLS 1 + +/* Define to 1 if you have the `readdir' function. */ +#define HAVE_READDIR 1 + +/* Define if you have the shl_load function. */ +/* #undef HAVE_SHL_LOAD */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the `strlcat' function. */ +/* #undef HAVE_STRLCAT */ + +/* Define to 1 if you have the `strlcpy' function. */ +/* #undef HAVE_STRLCPY */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_DL_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* This value is set to 1 to indicate that the system argz facility works */ +#define HAVE_WORKING_ARGZ 1 + +/* Define if the OS needs help to load dependent libraries for dlopen(). */ +/* #undef LTDL_DLOPEN_DEPLIBS */ + +/* Define to the system default library search path. */ +#define LT_DLSEARCH_PATH "/lib:/usr/lib:/usr/lib/x86_64-linux-gnu/libfakeroot:/usr/local/lib:/usr/local/lib/x86_64-linux-gnu:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/lib32:/usr/lib32" + +/* The archive extension */ +#define LT_LIBEXT "a" + +/* The archive prefix */ +#define LT_LIBPREFIX "lib" + +/* Define to the extension used for runtime loadable modules, say, ".so". */ +#define LT_MODULE_EXT ".so" + +/* Define to the name of the environment variable that determines the run-time + module search path. */ +#define LT_MODULE_PATH_VAR "LD_LIBRARY_PATH" + +/* Define to the sub-directory where libtool stores uninstalled libraries. */ +#define LT_OBJDIR ".libs/" + +/* Define to the shared library suffix, say, ".dylib". */ +/* #undef LT_SHARED_EXT */ + +/* Define to the shared archive member specification, say "(shr.o)". */ +/* #undef LT_SHARED_LIB_MEMBER */ + +/* Define if dlsym() requires a leading underscore in symbol names. */ +/* #undef NEED_USCORE */ + +/* Name of package */ +#define PACKAGE "libltdl" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "bug-libtool@gnu.org" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "libltdl" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "libltdl 2.4.3a" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "libltdl" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "2.4.3a" + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Version number of package */ +#define VERSION "2.4.3a" + +/* Define so that glibc/gnulib argp.h does not typedef error_t. */ +/* #undef __error_t_defined */ + +/* Define to a type to use for 'error_t' if it is not otherwise available. */ +/* #undef error_t */ diff --git a/contrib/unixodbc-cmake/linux_x86_64/libltdl/libltdlcS.c b/contrib/unixodbc-cmake/linux_x86_64/libltdl/libltdlcS.c new file mode 100644 index 00000000000..ca866eb5986 --- /dev/null +++ b/contrib/unixodbc-cmake/linux_x86_64/libltdl/libltdlcS.c @@ -0,0 +1,53 @@ +/* libltdlcS.c - symbol resolution table for 'libltdlc' dlsym emulation. */ +/* Generated by libtool (GNU libtool) 2.4.6 */ + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined __GNUC__ && (((__GNUC__ == 4) && (__GNUC_MINOR__ >= 4)) || (__GNUC__ > 4)) +#pragma GCC diagnostic ignored "-Wstrict-prototypes" +#endif + +/* Keep this code in sync between libtool.m4, ltmain, lt_system.h, and tests. */ +#if defined _WIN32 || defined __CYGWIN__ || defined _WIN32_WCE +/* DATA imports from DLLs on WIN32 can't be const, because runtime + relocations are performed -- see ld's documentation on pseudo-relocs. */ +# define LT_DLSYM_CONST +#elif defined __osf__ +/* This system does not cope well with relocations in const data. */ +# define LT_DLSYM_CONST +#else +# define LT_DLSYM_CONST const +#endif + +#define STREQ(s1, s2) (strcmp ((s1), (s2)) == 0) + +/* External symbol declarations for the compiler. */ +extern int dlopen_LTX_get_vtable(); + +/* The mapping between symbol names and symbols. */ +typedef struct { + const char *name; + void *address; +} lt_dlsymlist; +extern LT_DLSYM_CONST lt_dlsymlist +lt_libltdlc_LTX_preloaded_symbols[]; +LT_DLSYM_CONST lt_dlsymlist +lt_libltdlc_LTX_preloaded_symbols[] = +{ {"libltdlc", (void *) 0}, + {"dlopen.a", (void *) 0}, + {"dlopen_LTX_get_vtable", (void *) &dlopen_LTX_get_vtable}, + {0, (void *) 0} +}; + +/* This works around a problem in FreeBSD linker */ +#ifdef FREEBSD_WORKAROUND +static const void *lt_preloaded_setup() { + return lt_libltdlc_LTX_preloaded_symbols; +} +#endif + +#ifdef __cplusplus +} +#endif diff --git a/contrib/unixodbc-cmake/linux_x86_64/unixodbc_conf.h b/contrib/unixodbc-cmake/linux_x86_64/unixodbc_conf.h new file mode 100644 index 00000000000..6597c85cea6 --- /dev/null +++ b/contrib/unixodbc-cmake/linux_x86_64/unixodbc_conf.h @@ -0,0 +1,60 @@ +/* unixodbc_conf.h. Generated from unixodbc_conf.h.in by configure. */ +#ifndef HAVE_UNISTD_H +#define HAVE_UNISTD_H 1 +#endif + +#ifndef HAVE_PWD_H +#define HAVE_PWD_H 1 +#endif + +#ifndef HAVE_SYS_TIME_H +#define HAVE_SYS_TIME_H 1 +#endif + +#ifndef ODBC_STD +/* #undef ODBC_STD */ +#endif + +#ifndef UNICODE +/* #undef UNICODE */ +#endif + +#ifndef GUID_DEFINED +/* #undef GUID_DEFINED */ +#endif + +#ifndef SQL_WCHART_CONVERT +/* #undef SQL_WCHART_CONVERT */ +#endif + +#ifndef HAVE_LONG_LONG +#define HAVE_LONG_LONG 1 +#endif + +#ifndef ODBCINT64_TYPEA +/* #undef ODBCINT64_TYPEA */ +#endif + +#ifndef UODBCINT64_TYPE +/* #undef UODBCINT64_TYPE */ +#endif + +#ifndef DISABLE_INI_CACHING +/* #undef DISABLE_INI_CACHING */ +#endif + +#ifndef SIZEOF_LONG_INT +#define SIZEOF_LONG_INT 8 +#endif + +#ifndef ALLREADY_HAVE_WINDOWS_TYPE +/* #undef ALLREADY_HAVE_WINDOWS_TYPE */ +#endif + +#ifndef DONT_TD_VOID +/* #undef DONT_TD_VOID */ +#endif + +#ifndef DO_YOU_KNOW_WHAT_YOUR_ARE_DOING +/* #undef DO_YOU_KNOW_WHAT_YOUR_ARE_DOING */ +#endif diff --git a/docs/en/development/build.md b/docs/en/development/build.md index 6bf67e58c9a..e1b1cfe5dde 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -46,7 +46,7 @@ export CXX=g++-7 ## Install required libraries from packages ```bash -sudo apt-get install libicu-dev libreadline-dev libssl-dev unixodbc-dev ninja-build +sudo apt-get install libicu-dev libreadline-dev libssl-dev ``` ## Checkout ClickHouse sources diff --git a/docs/en/development/build_osx.md b/docs/en/development/build_osx.md index b54fd7ac32c..3dfe503be8d 100644 --- a/docs/en/development/build_osx.md +++ b/docs/en/development/build_osx.md @@ -12,7 +12,7 @@ With appropriate changes, it should also work on any other Linux distribution. ## Install required compilers, tools, and libraries ```bash -brew install cmake ninja gcc icu4c mariadb-connector-c openssl unixodbc libtool gettext readline +brew install cmake ninja gcc icu4c mariadb-connector-c openssl libtool gettext readline ``` ## Checkout ClickHouse sources From b324f69ede547dec6b7f8f84a49a72511ff31c56 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 2 Aug 2018 11:35:38 +0300 Subject: [PATCH 20/20] Auto version update to [18.8.0] [54403] --- dbms/cmake/version.cmake | 12 ++++++------ debian/changelog | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index c5ac46ab75a..8ee16a16856 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -1,11 +1,11 @@ # This strings autochanged from release_lib.sh: -set(VERSION_REVISION 54402 CACHE STRING "") +set(VERSION_REVISION 54403 CACHE STRING "") set(VERSION_MAJOR 18 CACHE STRING "") -set(VERSION_MINOR 7 CACHE STRING "") -set(VERSION_PATCH 1 CACHE STRING "") -set(VERSION_GITHASH d799ba683dd992d45113106ab9d57d7fdea05d99 CACHE STRING "") -set(VERSION_DESCRIBE v18.7.1-testing CACHE STRING "") -set(VERSION_STRING 18.7.1 CACHE STRING "") +set(VERSION_MINOR 8 CACHE STRING "") +set(VERSION_PATCH 0 CACHE STRING "") +set(VERSION_GITHASH e6be3df322f24ff3aa9ae9a97b9b01b2c88ab7b0 CACHE STRING "") +set(VERSION_DESCRIBE v18.8.0-testing CACHE STRING "") +set(VERSION_STRING 18.8.0 CACHE STRING "") # end of autochange set(VERSION_EXTRA "" CACHE STRING "") diff --git a/debian/changelog b/debian/changelog index d5ac568a9b6..2d3eb28d872 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (18.7.1) unstable; urgency=low +clickhouse (18.8.0) unstable; urgency=low * Modified source code - -- Thu, 02 Aug 2018 01:24:01 +0300 + -- Thu, 02 Aug 2018 11:35:38 +0300