mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-10 01:25:21 +00:00
Update deb-builder ubuntu version. Add clang-11, gcc-10. Add unbundled docker. Add test-base image.
This commit is contained in:
Yatsishin Ilya
parent
edcd6d5502
commit
ccbfd2e5df
@ -2,6 +2,7 @@
|
||||
"docker/packager/deb": {
|
||||
"name": "yandex/clickhouse-deb-builder",
|
||||
"dependent": [
|
||||
"docker/packager/unbundled",
|
||||
"docker/test/stateless",
|
||||
"docker/test/stateless_with_coverage",
|
||||
"docker/test/stateless_pytest",
|
||||
@ -15,6 +16,10 @@
|
||||
"docker/test/pvs"
|
||||
]
|
||||
},
|
||||
"docker/packager/unbundled": {
|
||||
"name": "yandex/clickhouse-unbundled-builder",
|
||||
"dependent": []
|
||||
},
|
||||
"docker/test/coverage": {
|
||||
"name": "yandex/clickhouse-coverage",
|
||||
"dependent": []
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
# docker build -t yandex/clickhouse-deb-builder .
|
||||
FROM ubuntu:19.10
|
||||
FROM ubuntu:20.04
|
||||
|
||||
RUN apt-get --allow-unauthenticated update -y && apt-get install --yes wget gnupg
|
||||
RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
|
||||
RUN echo "deb [trusted=yes] http://apt.llvm.org/eoan/ llvm-toolchain-eoan-10 main" >> /etc/apt/sources.list
|
||||
RUN echo "deb [trusted=yes] http://apt.llvm.org/focal/ llvm-toolchain-focal-11 main" >> /etc/apt/sources.list
|
||||
|
||||
# initial packages
|
||||
RUN apt-get --allow-unauthenticated update -y \
|
||||
@ -25,13 +25,17 @@ RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb
|
||||
RUN chmod +x dpkg-deb
|
||||
RUN cp dpkg-deb /usr/bin
|
||||
|
||||
|
||||
# Libraries from OS are only needed to test the "unbundled" build (that is not used in production).
|
||||
RUN apt-get --allow-unauthenticated update -y \
|
||||
&& env DEBIAN_FRONTEND=noninteractive \
|
||||
apt-get --allow-unauthenticated install --yes --no-install-recommends \
|
||||
gcc-10 \
|
||||
g++-10 \
|
||||
gcc-9 \
|
||||
g++-9 \
|
||||
llvm-11 \
|
||||
clang-11 \
|
||||
lld-11 \
|
||||
clang-tidy-11 \
|
||||
llvm-10 \
|
||||
clang-10 \
|
||||
lld-10 \
|
||||
@ -39,54 +43,19 @@ RUN apt-get --allow-unauthenticated update -y \
|
||||
clang-9 \
|
||||
lld-9 \
|
||||
clang-tidy-9 \
|
||||
libicu-dev \
|
||||
libreadline-dev \
|
||||
gperf \
|
||||
ninja-build \
|
||||
perl \
|
||||
pkg-config \
|
||||
devscripts \
|
||||
debhelper \
|
||||
git \
|
||||
libc++-dev \
|
||||
libc++abi-dev \
|
||||
libboost-program-options-dev \
|
||||
libboost-system-dev \
|
||||
libboost-filesystem-dev \
|
||||
libboost-thread-dev \
|
||||
libboost-iostreams-dev \
|
||||
libboost-regex-dev \
|
||||
zlib1g-dev \
|
||||
liblz4-dev \
|
||||
libdouble-conversion-dev \
|
||||
librdkafka-dev \
|
||||
libpoconetssl62 \
|
||||
libpoco-dev \
|
||||
libgoogle-perftools-dev \
|
||||
libzstd-dev \
|
||||
libltdl-dev \
|
||||
libre2-dev \
|
||||
libjemalloc-dev \
|
||||
libmsgpack-dev \
|
||||
libcurl4-openssl-dev \
|
||||
opencl-headers \
|
||||
ocl-icd-libopencl1 \
|
||||
intel-opencl-icd \
|
||||
unixodbc-dev \
|
||||
odbcinst \
|
||||
tzdata \
|
||||
gperf \
|
||||
alien \
|
||||
libcapnp-dev \
|
||||
cmake \
|
||||
gdb \
|
||||
pigz \
|
||||
moreutils \
|
||||
libcctz-dev \
|
||||
libldap2-dev \
|
||||
libsasl2-dev \
|
||||
heimdal-multidev \
|
||||
libhyperscan-dev
|
||||
pigz
|
||||
|
||||
|
||||
# This symlink required by gcc to find lld compiler
|
||||
|
||||
@ -11,6 +11,7 @@ SCRIPT_PATH = os.path.realpath(__file__)
|
||||
IMAGE_MAP = {
|
||||
"deb": "yandex/clickhouse-deb-builder",
|
||||
"binary": "yandex/clickhouse-binary-builder",
|
||||
"unbundled": "yandex/clickhouse-unbundled-builder"
|
||||
}
|
||||
|
||||
def check_image_exists_locally(image_name):
|
||||
@ -176,7 +177,9 @@ if __name__ == "__main__":
|
||||
parser.add_argument("--clickhouse-repo-path", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir))
|
||||
parser.add_argument("--output-dir", required=True)
|
||||
parser.add_argument("--build-type", choices=("debug", ""), default="")
|
||||
parser.add_argument("--compiler", choices=("clang-10-darwin", "clang-10-aarch64", "clang-10-freebsd", "gcc-9", "clang-10"), default="gcc-9")
|
||||
parser.add_argument("--compiler", choices=("clang-10", "clang-10-darwin", "clang-10-aarch64", "clang-10-freebsd",
|
||||
"clang-11", "clang-11-darwin", "clang-11-aarch64", "clang-11-freebsd",
|
||||
"gcc-9", "gcc-10"), default="gcc-9")
|
||||
parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="")
|
||||
parser.add_argument("--unbundled", action="store_true")
|
||||
parser.add_argument("--split-binary", action="store_true")
|
||||
@ -197,7 +200,7 @@ if __name__ == "__main__":
|
||||
if not os.path.isabs(args.output_dir):
|
||||
args.output_dir = os.path.abspath(os.path.join(os.getcwd(), args.output_dir))
|
||||
|
||||
image_type = 'binary' if args.package_type == 'performance' else args.package_type
|
||||
image_type = 'binary' if args.package_type == 'performance' else 'unbundled' if args.unbundled else args.package_type
|
||||
image_name = IMAGE_MAP[image_type]
|
||||
|
||||
if not os.path.isabs(args.clickhouse_repo_path):
|
||||
|
||||
56
docker/packager/unbundled/Dockerfile
Normal file
56
docker/packager/unbundled/Dockerfile
Normal file
@ -0,0 +1,56 @@
|
||||
# docker build -t yandex/clickhouse-unbundled-builder .
|
||||
FROM yandex/clickhouse-deb-builder
|
||||
|
||||
# Libraries from OS are only needed to test the "unbundled" build (that is not used in production).
|
||||
RUN apt-get --allow-unauthenticated update -y \
|
||||
&& env DEBIAN_FRONTEND=noninteractive \
|
||||
apt-get --allow-unauthenticated install --yes --no-install-recommends \
|
||||
libicu-dev \
|
||||
libreadline-dev \
|
||||
gperf \
|
||||
perl \
|
||||
pkg-config \
|
||||
devscripts \
|
||||
libc++-dev \
|
||||
libc++abi-dev \
|
||||
libboost-program-options-dev \
|
||||
libboost-system-dev \
|
||||
libboost-filesystem-dev \
|
||||
libboost-thread-dev \
|
||||
libboost-iostreams-dev \
|
||||
libboost-regex-dev \
|
||||
zlib1g-dev \
|
||||
liblz4-dev \
|
||||
libdouble-conversion-dev \
|
||||
librdkafka-dev \
|
||||
libpoconetssl62 \
|
||||
libpoco-dev \
|
||||
libgoogle-perftools-dev \
|
||||
libzstd-dev \
|
||||
libltdl-dev \
|
||||
libre2-dev \
|
||||
libjemalloc-dev \
|
||||
libmsgpack-dev \
|
||||
libcurl4-openssl-dev \
|
||||
opencl-headers \
|
||||
ocl-icd-libopencl1 \
|
||||
intel-opencl-icd \
|
||||
unixodbc-dev \
|
||||
odbcinst \
|
||||
tzdata \
|
||||
gperf \
|
||||
alien \
|
||||
libcapnp-dev \
|
||||
cmake \
|
||||
gdb \
|
||||
pigz \
|
||||
moreutils \
|
||||
libcctz-dev \
|
||||
libldap2-dev \
|
||||
libsasl2-dev \
|
||||
heimdal-multidev \
|
||||
libhyperscan-dev
|
||||
|
||||
COPY build.sh /
|
||||
|
||||
CMD ["/bin/bash", "/build.sh"]
|
||||
18
docker/packager/unbundled/build.sh
Executable file
18
docker/packager/unbundled/build.sh
Executable file
@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -x -e
|
||||
|
||||
# Update tzdata to the latest version. It is embedded into clickhouse binary.
|
||||
sudo apt-get update && sudo apt-get install tzdata
|
||||
|
||||
ccache --show-stats ||:
|
||||
ccache --zero-stats ||:
|
||||
build/release --no-pbuilder $ALIEN_PKGS | ts '%Y-%m-%d %H:%M:%S'
|
||||
mv /*.deb /output
|
||||
mv *.changes /output
|
||||
mv *.buildinfo /output
|
||||
mv /*.rpm /output ||: # if exists
|
||||
mv /*.tgz /output ||: # if exists
|
||||
|
||||
ccache --show-stats ||:
|
||||
ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||:
|
||||
51
docker/test/base/Dockerfile
Normal file
51
docker/test/base/Dockerfile
Normal file
@ -0,0 +1,51 @@
|
||||
# docker build -t yandex/clickhouse-test-base .
|
||||
FROM ubuntu:19.10
|
||||
|
||||
RUN apt-get --allow-unauthenticated update -y && apt-get install --yes wget gnupg
|
||||
RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
|
||||
RUN echo "deb [trusted=yes] http://apt.llvm.org/eoan/ llvm-toolchain-eoan-10 main" >> /etc/apt/sources.list
|
||||
|
||||
# initial packages
|
||||
RUN apt-get --allow-unauthenticated update -y \
|
||||
&& env DEBIAN_FRONTEND=noninteractive \
|
||||
apt-get --allow-unauthenticated install --yes --no-install-recommends \
|
||||
apt-transport-https \
|
||||
bash \
|
||||
ca-certificates \
|
||||
curl \
|
||||
fakeroot \
|
||||
gnupg \
|
||||
software-properties-common
|
||||
|
||||
# Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able
|
||||
# to compress files using pigz (https://zlib.net/pigz/) instead of gzip.
|
||||
# Significantly increase deb packaging speed and compatible with old systems
|
||||
RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb
|
||||
RUN chmod +x dpkg-deb
|
||||
RUN cp dpkg-deb /usr/bin
|
||||
|
||||
RUN apt-get --allow-unauthenticated update -y \
|
||||
&& env DEBIAN_FRONTEND=noninteractive \
|
||||
apt-get --allow-unauthenticated install --yes --no-install-recommends \
|
||||
clang-10 \
|
||||
debhelper \
|
||||
devscripts \
|
||||
gdb \
|
||||
git \
|
||||
gperf \
|
||||
lcov \
|
||||
llvm-10 \
|
||||
moreutils \
|
||||
perl \
|
||||
perl \
|
||||
pigz \
|
||||
pkg-config \
|
||||
tzdata
|
||||
|
||||
# Sanitizer options
|
||||
RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7'" >> /etc/environment; \
|
||||
echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment; \
|
||||
echo "MSAN_OPTIONS='abort_on_error=1'" >> /etc/environment; \
|
||||
ln -s /usr/lib/llvm-10/bin/llvm-symbolizer /usr/bin/llvm-symbolizer;
|
||||
|
||||
CMD sleep 1
|
||||
Loading…
Reference in New Issue
Block a user