diff --git a/.gitignore b/.gitignore index 3fa4a095a1b..a21df2e8e0f 100644 --- a/.gitignore +++ b/.gitignore @@ -180,18 +180,10 @@ utils/zookeeper-create-entry-to-download-part/zookeeper-create-entry-to-download utils/zookeeper-dump-tree/zookeeper-dump-tree utils/zookeeper-remove-by-list/zookeeper-remove-by-list dbms/src/Storages/tests/remove_symlink_directory -debian/control -debian/copyright -debian/tmp/ libs/libcommon/src/tests/json_test utils/compressor/zstd_test utils/wikistat-loader/wikistat-loader dbms/src/Common/tests/pod_array -debian/clickhouse-benchmark/ -debian/clickhouse-client/ -debian/clickhouse-server-base/ -debian/clickhouse-server-common/ -debian/files dbms/src/Server/data/* dbms/src/Server/metadata/* @@ -210,9 +202,6 @@ vgcore* *.changes build-stamp configure-stamp -debian/*.debhelper.log -debian/*.debhelper -debian/*.substvars *.bin *.mrk diff --git a/.travis.yml.bak b/.travis.yml.bak deleted file mode 100644 index 0aed0a24427..00000000000 --- a/.travis.yml.bak +++ /dev/null @@ -1,50 +0,0 @@ -language: generic - -matrix: - fast_finish: true - include: - # We need to have gcc7 headers to compile c++17 code on clang -# - os: linux -# -# cache: -# ccache: true -# timeout: 1000 -# directories: -# - /home/travis/.ccache -# -# addons: -# apt: -# update: true -# sources: -# - ubuntu-toolchain-r-test -# - llvm-toolchain-trusty-5.0 -# packages: [ ninja-build, g++-7, clang-5.0, lld-5.0, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo, openssl] -# -# env: -# - MATRIX_EVAL="export CC=clang-5.0 CXX=clang++-5.0" -# -# script: -# - utils/travis/normal.sh - - - os: linux - - sudo: required - - cache: - timeout: 1000 - directories: - - /var/cache/pbuilder/ccache - - addons: - apt: - update: true - packages: [ pbuilder, fakeroot, debhelper ] - - script: - - utils/travis/pbuilder.sh - - allow_failures: - - os: osx - -before_script: - - eval "${MATRIX_EVAL}" diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index 34edab61470..ccb379a4a69 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -2,10 +2,10 @@ set(VERSION_REVISION 54413) set(VERSION_MAJOR 19) set(VERSION_MINOR 1) -set(VERSION_PATCH 1) -set(VERSION_GITHASH 4e7747117123f5a1b027a64865844b4faa10447d) -set(VERSION_DESCRIBE v19.1.1-testing) -set(VERSION_STRING 19.1.1) +set(VERSION_PATCH 3) +set(VERSION_GITHASH ac0060079ab221278338db343ca9eaf006fc4ee1) +set(VERSION_DESCRIBE v19.1.3-testing) +set(VERSION_STRING 19.1.3) # end of autochange set(VERSION_EXTRA "" CACHE STRING "") diff --git a/dbms/src/Interpreters/Compiler.cpp b/dbms/src/Interpreters/Compiler.cpp index 8a60b24a24b..0d8fc36f840 100644 --- a/dbms/src/Interpreters/Compiler.cpp +++ b/dbms/src/Interpreters/Compiler.cpp @@ -186,10 +186,10 @@ SharedLibraryPtr Compiler::getOrCount( static void addCodeToAssertHeadersMatch(WriteBuffer & out) { out << - "#define STRING2(x) #x\n" - "#define STRING(x) STRING2(x)\n" "#include \n" "#if VERSION_REVISION != " << ClickHouseRevision::get() << "\n" + "#define STRING2(x) #x\n" + "#define STRING(x) STRING2(x)\n" "#pragma message \"ClickHouse headers revision = \" STRING(VERSION_REVISION) \n" "#error \"ClickHouse headers revision doesn't match runtime revision of the server (" << ClickHouseRevision::get() << ").\"\n" "#endif\n\n"; diff --git a/dbms/src/Interpreters/SpecializedAggregator.h b/dbms/src/Interpreters/SpecializedAggregator.h index 615911a7224..de7ea6fd921 100644 --- a/dbms/src/Interpreters/SpecializedAggregator.h +++ b/dbms/src/Interpreters/SpecializedAggregator.h @@ -136,7 +136,7 @@ void NO_INLINE Aggregator::executeSpecializedCase( AggregateDataPtr overflow_row) const { /// For all rows. - typename Method::Key prev_key; + typename Method::Key prev_key{}; AggregateDataPtr value = nullptr; for (size_t i = 0; i < rows; ++i) { diff --git a/debian/.gitignore b/debian/.gitignore new file mode 100644 index 00000000000..3b10df86d93 --- /dev/null +++ b/debian/.gitignore @@ -0,0 +1,12 @@ +control +copyright +tmp/ +clickhouse-benchmark/ +clickhouse-client/ +clickhouse-server-base/ +clickhouse-server-common/ +files +*.debhelper.log +*.debhelper +*.substvars + diff --git a/debian/changelog b/debian/changelog index 08dd7ff59a8..0b436d08615 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (19.1.1) unstable; urgency=low +clickhouse (19.1.3) unstable; urgency=low * Modified source code - -- Wed, 16 Jan 2019 14:04:37 +0300 + -- Mon, 21 Jan 2019 16:26:13 +0300 diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 37d7ca0a7d1..c607bbb2f46 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.1.1 +ARG version=19.1.3 RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 4f2c8ae839d..4d40cbe7714 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.1.1 +ARG version=19.1.3 ARG gosu_ver=1.10 RUN apt-get update \ diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 6bd520a151b..c7b325e4d58 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.1.1 +ARG version=19.1.3 RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ diff --git a/docker/test/stateful/Dockerfile b/docker/test/stateful/Dockerfile new file mode 100644 index 00000000000..52275e2b43c --- /dev/null +++ b/docker/test/stateful/Dockerfile @@ -0,0 +1,29 @@ +FROM yandex/clickhouse-stateless-test + +RUN apt-get update -y \ + && env DEBIAN_FRONTEND=noninteractive \ + apt-get install --yes --no-install-recommends \ + python-requests + +COPY s3downloader /s3downloader + +ENV DATASETS="hits visits" + +CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \ + dpkg -i package_folder/clickhouse-server_*.deb; \ + dpkg -i package_folder/clickhouse-client_*.deb; \ + dpkg -i package_folder/clickhouse-test_*.deb; \ + service zookeeper start; sleep 5; \ + service clickhouse-server start && sleep 5 \ + && /s3downloader --dataset-names $DATASETS \ + && chmod 777 -R /var/lib/clickhouse \ + && clickhouse-client --query "SHOW DATABASES" \ + && clickhouse-client --query "CREATE DATABASE datasets" \ + && clickhouse-client --query "CREATE DATABASE test" \ + && service clickhouse-server restart && sleep 5 \ + && clickhouse-client --query "SHOW TABLES FROM datasets" \ + && clickhouse-client --query "SHOW TABLES FROM test" \ + && clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits" \ + && clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits" \ + && clickhouse-client --query "SHOW TABLES FROM test" \ + && clickhouse-test --shard --zookeeper --no-stateless $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt diff --git a/docker/test/stateful/listen.xml b/docker/test/stateful/listen.xml new file mode 100644 index 00000000000..675225f9575 --- /dev/null +++ b/docker/test/stateful/listen.xml @@ -0,0 +1 @@ +:: diff --git a/docker/test/stateful/log_queries.xml b/docker/test/stateful/log_queries.xml new file mode 100644 index 00000000000..25261072ade --- /dev/null +++ b/docker/test/stateful/log_queries.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/docker/test/stateful/part_log.xml b/docker/test/stateful/part_log.xml new file mode 100644 index 00000000000..6c6fc9c6982 --- /dev/null +++ b/docker/test/stateful/part_log.xml @@ -0,0 +1,8 @@ + + + system + part_log
+ + 7500 + + diff --git a/docker/test/stateful/s3downloader b/docker/test/stateful/s3downloader new file mode 100755 index 00000000000..f8e2bf3cbe4 --- /dev/null +++ b/docker/test/stateful/s3downloader @@ -0,0 +1,86 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import os +import sys +import tarfile +import logging +import argparse +import requests +import tempfile + + +DEFAULT_URL = 'https://clickhouse-datasets.s3.yandex.net' + +AVAILABLE_DATASETS = { + 'hits': 'hits_v1.tar', + 'visits': 'visits_v1.tar', +} + +def _get_temp_file_name(): + return os.path.join(tempfile._get_default_tempdir(), next(tempfile._get_candidate_names())) + +def build_url(base_url, dataset): + return os.path.join(base_url, dataset, 'partitions', AVAILABLE_DATASETS[dataset]) + +def dowload_with_progress(url, path): + logging.info("Downloading from %s to temp path %s", url, path) + with open(path, 'w') as f: + response = requests.get(url, stream=True) + response.raise_for_status() + total_length = response.headers.get('content-length') + if total_length is None or int(total_length) == 0: + logging.info("No content-length, will download file without progress") + f.write(response.content) + else: + dl = 0 + total_length = int(total_length) + logging.info("Content length is %ld bytes", total_length) + for data in response.iter_content(chunk_size=4096): + dl += len(data) + f.write(data) + if sys.stdout.isatty(): + done = int(50 * dl / total_length) + percent = int(100 * float(dl) / total_length) + sys.stdout.write("\r[{}{}] {}%".format('=' * done, ' ' * (50-done), percent)) + sys.stdout.flush() + sys.stdout.write("\n") + logging.info("Downloading finished") + +def unpack_to_clickhouse_directory(tar_path, clickhouse_path): + logging.info("Will unpack data from temp path %s to clickhouse db %s", tar_path, clickhouse_path) + with tarfile.open(tar_path, 'r') as comp_file: + comp_file.extractall(path=clickhouse_path) + logging.info("Unpack finished") + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser( + description="Simple tool for dowloading datasets for clickhouse from S3") + + parser.add_argument('--dataset-names', required=True, nargs='+', choices=AVAILABLE_DATASETS.keys()) + parser.add_argument('--url-prefix', default=DEFAULT_URL) + parser.add_argument('--clickhouse-data-path', default='/var/lib/clickhouse/') + + args = parser.parse_args() + datasets = args.dataset_names + logging.info("Will fetch following datasets: %s", ', '.join(datasets)) + for dataset in datasets: + logging.info("Processing %s", dataset) + temp_archive_path = _get_temp_file_name() + try: + download_url_for_dataset = build_url(args.url_prefix, dataset) + dowload_with_progress(download_url_for_dataset, temp_archive_path) + unpack_to_clickhouse_directory(temp_archive_path, args.clickhouse_data_path) + except Exception as ex: + logging.info("Some exception occured %s", str(ex)) + raise + finally: + logging.info("Will remove dowloaded file %s from filesystem if it exists", temp_archive_path) + if os.path.exists(temp_archive_path): + os.remove(temp_archive_path) + logging.info("Processing of %s finished", dataset) + logging.info("Fetch finished, enjoy your tables!") + + diff --git a/docker/test/stateful/zookeeper.xml b/docker/test/stateful/zookeeper.xml new file mode 100644 index 00000000000..095f4be78c1 --- /dev/null +++ b/docker/test/stateful/zookeeper.xml @@ -0,0 +1,16 @@ + + + + localhost + 2181 + + + yandex.ru + 2181 + + + 111.0.1.2 + 2181 + + +