release script: convert .deb to .rpm and .tgz (#5740)

debian/clickhouse-server.config

@@ -1,6 +1,6 @@
 #!/bin/sh -e
 
-. /usr/share/debconf/confmodule
+test -f /usr/share/debconf/confmodule && . /usr/share/debconf/confmodule
 
 db_fget clickhouse-server/default-password seen || true
 password_seen="$RET"

debian/clickhouse-server.postinst

@@ -11,17 +11,25 @@ CLICKHOUSE_GENERIC_PROGRAM=${CLICKHOUSE_GENERIC_PROGRAM:=clickhouse}
 EXTRACT_FROM_CONFIG=${CLICKHOUSE_GENERIC_PROGRAM}-extract-from-config
 CLICKHOUSE_CONFIG=$CLICKHOUSE_CONFDIR/config.xml
 
-OS=${OS=`lsb_release -is 2>/dev/null || uname -s ||:`}
+OS=${OS=`lsb_release -is 2>/dev/null ||:`}
+if [ -z "$OS" ]; then
+    test -f /etc/os-release && . /etc/os-release && OS=$ID
+fi
+OS=${OS=`uname -s ||:`}
 
-. /usr/share/debconf/confmodule
+test -f /usr/share/debconf/confmodule && . /usr/share/debconf/confmodule
 
 test -f /etc/default/clickhouse && . /etc/default/clickhouse
 
-if [ "$1" = configure ]; then
+if [ "$OS" = "rhel" ] || [ "$OS" = "centos" ] || [ "$OS" = "fedora" ] || [ "$OS" = "CentOS" ] || [ "$OS" = "Fedora" ]; then
+    is_rh=1
+fi
+
+if [ "$1" = configure ] || [ -n "$is_rh" ]; then
     if [ -x "/bin/systemctl" ] && [ -f /etc/systemd/system/clickhouse-server.service ] && [ -d /run/systemd/system ]; then
         # if old rc.d service present - remove it
-        if [ -x "/etc/init.d/clickhouse-server" ]; then
+        if [ -x "/etc/init.d/clickhouse-server" ] && [ -x "/usr/sbin/update-rc.d" ]; then
-            update-rc.d clickhouse-server remove
+            /usr/sbin/update-rc.d clickhouse-server remove
             echo "ClickHouse init script has migrated to systemd. Please manually stop old server and restart the service: sudo killall clickhouse-server && sleep 5 && sudo service clickhouse-server restart"
         fi
 
@@ -30,17 +38,17 @@ if [ "$1" = configure ]; then
     else
         # If you downgrading to version older than 1.1.54336 run: systemctl disable clickhouse-server
         if [ -x "/etc/init.d/clickhouse-server" ]; then
-            if [ "$OS" = "rhel" ] || [ "$OS" = "centos" ] || [ "$OS" = "fedora" ]; then
+            if [ -x "/usr/sbin/update-rc.d" ]; then
-                echo # TODO
+                /usr/sbin/update-rc.d clickhouse-server defaults 19 19 >/dev/null || exit $?
             else
-                update-rc.d clickhouse-server defaults 19 19 >/dev/null || exit $?
+                echo # TODO [ "$OS" = "rhel" ] || [ "$OS" = "centos" ] || [ "$OS" = "fedora" ]
             fi
         fi
     fi
 
     # Make sure the administrative user exists
     if ! getent passwd ${CLICKHOUSE_USER} > /dev/null; then
-        if [ "$OS" = "rhel" ] || [ "$OS" = "centos" ] || [ "$OS" = "fedora" ]; then
+        if [ -n "$is_rh" ]; then
             adduser --system --no-create-home --home /nonexistent \
                 --shell /bin/false ${CLICKHOUSE_USER} > /dev/null
         else
@@ -123,14 +131,16 @@ Please fix this and reinstall this package." >&2
         rm -f ${CLICKHOUSE_DATADIR_FROM_CONFIG}/build/*.cpp ${CLICKHOUSE_DATADIR_FROM_CONFIG}/build/*.so ||:
     fi
 
-    db_get clickhouse-server/default-password
+    if [ -f /usr/share/debconf/confmodule ]; then
-    defaultpassword="$RET"
+        db_get clickhouse-server/default-password
-    if [ -n "$defaultpassword" ]; then
+        defaultpassword="$RET"
-        echo "<yandex><users><default><password>$defaultpassword</password></default></users></yandex>" > ${CLICKHOUSE_CONFDIR}/users.d/default-password.xml
+        if [ -n "$defaultpassword" ]; then
-    fi
+            echo "<yandex><users><default><password>$defaultpassword</password></default></users></yandex>" > ${CLICKHOUSE_CONFDIR}/users.d/default-password.xml
+        fi
 
-    # everything went well, so now let's reset the password
+        # everything went well, so now let's reset the password
-    db_set clickhouse-server/default-password ""
+        db_set clickhouse-server/default-password ""
-    # ... done with debconf here
+        # ... done with debconf here
-    db_stop
+        db_stop
+    fi
 fi

release

@@ -31,8 +31,8 @@
 
 set -e
 
-CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-cd $CURDIR
+cd $CUR_DIR
 
 source "./utils/release/release_lib.sh"
 
@@ -68,6 +68,12 @@ do
         # Wrong but fast pbuilder mode: create base package with all depends
         EXTRAPACKAGES="$EXTRAPACKAGES debhelper cmake ninja-build gcc-7 g++-7 libc6-dev libicu-dev libreadline-dev psmisc bash expect python python-lxml python-termcolor python-requests curl perl sudo openssl netcat-openbsd"
         shift
+    elif [[ $1 == '--rpm' ]]; then
+        MAKE_RPM=1
+        shift
+    elif [[ $1 == '--tgz' ]]; then
+        MAKE_TGZ=1
+        shift
     else
         echo "Unknown option $1"
         exit 2
@@ -109,40 +115,50 @@ echo -e "\nCurrent version is $VERSION_STRING"
 
 gen_changelog "$VERSION_STRING" "" "$AUTHOR" ""
 
-if [ -z "$USE_PBUILDER" ] ; then
+if [ -z "$NO_BUILD" ] ; then
-    DEB_CC=${DEB_CC:=`which gcc-7 gcc-8 gcc | head -n1`}
+    if [ -z "$USE_PBUILDER" ] ; then
-    DEB_CXX=${DEB_CXX:=`which g++-7 g++-8 g++ | head -n1`}
+        DEB_CC=${DEB_CC:=`which gcc-7 gcc-8 gcc | head -n1`}
-    # Build (only binary packages).
+        DEB_CXX=${DEB_CXX:=`which g++-7 g++-8 g++ | head -n1`}
-    debuild --preserve-env -e PATH \
+        # Build (only binary packages).
-    -e DEB_CC=$DEB_CC -e DEB_CXX=$DEB_CXX -e CMAKE_FLAGS="$CMAKE_FLAGS" \
+        debuild --preserve-env -e PATH \
-    -b ${DEBUILD_NOSIGN_OPTIONS} ${DEBUILD_NODEPS_OPTIONS}
+        -e DEB_CC=$DEB_CC -e DEB_CXX=$DEB_CXX -e CMAKE_FLAGS="$CMAKE_FLAGS" \
-else
+        -b ${DEBUILD_NOSIGN_OPTIONS} ${DEBUILD_NODEPS_OPTIONS}
-    export DIST=${DIST:=bionic}
+    else
-    export SET_BUILDRESULT=${SET_BUILDRESULT:=$CURDIR/..}
+        export DIST=${DIST:=bionic}
+        export SET_BUILDRESULT=${SET_BUILDRESULT:=$CUR_DIR/..}
 
-    if [[ -z `which pbuilder` ]] ; then
+        if [[ -z `which pbuilder` ]] ; then
-        sudo apt install -y pbuilder devscripts ccache fakeroot debhelper debian-archive-keyring debian-keyring lsb-release
+            sudo apt install -y pbuilder devscripts ccache fakeroot debhelper debian-archive-keyring debian-keyring lsb-release
-    fi
-
-    . $CURDIR/debian/.pbuilderrc
-
-    if [[ ! -e "/usr/share/debootstrap/scripts/${DIST}" ]] ; then
-        sudo ln -s gutsy /usr/share/debootstrap/scripts/${DIST}
-    fi
-
-    if [[ -n "$FORCE_PBUILDER_CREATE" || ! -e "$BASETGZ" ]] ; then
-        echo Creating base system $BASETGZ
-        [ ! -e "/usr/share/debootstrap/scripts/${DIST}" ] && sudo ln -s gutsy /usr/share/debootstrap/scripts/${DIST}
-        sudo --preserve-env bash -x pbuilder create --configfile $CURDIR/debian/.pbuilderrc $PBUILDER_OPT
-    fi
-
-    if [ "$PBUILDER_AUTOUPDATE" -gt 0 ]; then
-        # Update every 3 days (60*24*3 minutes)
-        if [[ -n "$PBUILDER_UPDATE" ]] || test `find "$BASETGZ" -mmin +$PBUILDER_AUTOUPDATE` ; then
-            echo Updating base system $BASETGZ
-            sudo --preserve-env pbuilder update --configfile $CURDIR/debian/.pbuilderrc $PBUILDER_OPT
         fi
-    fi
 
-    pdebuild --configfile $CURDIR/debian/.pbuilderrc -- $PBUILDER_OPT
+        . $CUR_DIR/debian/.pbuilderrc
+
+        if [[ ! -e "/usr/share/debootstrap/scripts/${DIST}" ]] ; then
+            sudo ln -s gutsy /usr/share/debootstrap/scripts/${DIST}
+        fi
+
+        if [[ -n "$FORCE_PBUILDER_CREATE" || ! -e "$BASETGZ" ]] ; then
+            echo Creating base system $BASETGZ
+            [ ! -e "/usr/share/debootstrap/scripts/${DIST}" ] && sudo ln -s gutsy /usr/share/debootstrap/scripts/${DIST}
+            sudo --preserve-env bash -x pbuilder create --configfile $CUR_DIR/debian/.pbuilderrc $PBUILDER_OPT
+        fi
+
+        if [ "$PBUILDER_AUTOUPDATE" -gt 0 ]; then
+            # Update every 3 days (60*24*3 minutes)
+            if [[ -n "$PBUILDER_UPDATE" ]] || test `find "$BASETGZ" -mmin +$PBUILDER_AUTOUPDATE` ; then
+                echo Updating base system $BASETGZ
+                sudo --preserve-env pbuilder update --configfile $CUR_DIR/debian/.pbuilderrc $PBUILDER_OPT
+            fi
+        fi
+
+        pdebuild --configfile $CUR_DIR/debian/.pbuilderrc -- $PBUILDER_OPT
+    fi
+fi
+
+if [ -n "$MAKE_RPM" ]; then
+    make_rpm
+fi
+
+if [ -n "$MAKE_TGZ" ]; then
+    make_tgz
 fi

utils/release/release_lib.sh

@@ -179,3 +179,92 @@ function gen_dockerfiles {
     VERSION_STRING="$1"
     ls -1 docker/*/Dockerfile | xargs sed -i -r -e 's/ARG version=.+$/ARG version='$VERSION_STRING'/'
 }
+
+function make_rpm {
+    get_version
+    VERSION_STRING+=$VERSION_POSTFIX
+    VERSION=$VERSION_STRING
+    PACKAGE_DIR=../
+
+    function deb_unpack {
+        rm -rf $PACKAGE-$VERSION
+        alien --verbose --generate --to-rpm --scripts ${PACKAGE_DIR}${PACKAGE}_${VERSION}_${ARCH}.deb
+        cd $PACKAGE-$VERSION
+        mv ${PACKAGE}-$VERSION-2.spec ${PACKAGE}-$VERSION-2.spec.tmp
+        cat ${PACKAGE}-$VERSION-2.spec.tmp \
+            | grep -vF '%dir "/"' \
+            | grep -vF '%dir "/usr/"' \
+            | grep -vF '%dir "/usr/bin/"' \
+            | grep -vF '%dir "/usr/lib/"' \
+            | grep -vF '%dir "/usr/lib/debug/"' \
+            | grep -vF '%dir "/usr/lib/.build-id/"' \
+            | grep -vF '%dir "/usr/share/"' \
+            | grep -vF '%dir "/usr/share/doc/"' \
+            | grep -vF '%dir "/lib/"' \
+            | grep -vF '%dir "/lib/systemd/"' \
+            | grep -vF '%dir "/lib/systemd/system/"' \
+            | grep -vF '%dir "/etc/"' \
+            | grep -vF '%dir "/etc/security/"' \
+            | grep -vF '%dir "/etc/security/limits.d/"' \
+            | grep -vF '%dir "/etc/init.d/"' \
+            | grep -vF '%dir "/etc/cron.d/"' \
+            | grep -vF '%dir "/etc/systemd/system/"' \
+            | grep -vF '%dir "/etc/systemd/"' \
+            > ${PACKAGE}-$VERSION-2.spec
+    }
+
+    function rpm_pack {
+        rpmbuild --buildroot="$CUR_DIR/${PACKAGE}-$VERSION" -bb --target ${TARGET} "${PACKAGE}-$VERSION-2.spec"
+        cd $CUR_DIR
+    }
+
+    function unpack_pack {
+        deb_unpack
+        rpm_pack
+    }
+
+    PACKAGE=clickhouse-server
+    ARCH=all
+    TARGET=noarch
+    unpack_pack
+
+    PACKAGE=clickhouse-client
+    ARCH=all
+    TARGET=noarch
+    unpack_pack
+
+    PACKAGE=clickhouse-test
+    ARCH=all
+    TARGET=noarch
+    deb_unpack
+    mv ${PACKAGE}-$VERSION-2.spec ${PACKAGE}-$VERSION-2.spec_tmp
+    echo "Requires: python2" >> ${PACKAGE}-$VERSION-2.spec
+    #echo "Requires: python2-termcolor" >> ${PACKAGE}-$VERSION-2.spec
+    cat ${PACKAGE}-$VERSION-2.spec_tmp >> ${PACKAGE}-$VERSION-2.spec
+    rpm_pack
+
+    PACKAGE=clickhouse-common-static
+    ARCH=amd64
+    TARGET=x86_64
+    unpack_pack
+
+    PACKAGE=clickhouse-common-static-dbg
+    ARCH=amd64
+    TARGET=x86_64
+    unpack_pack
+
+    mv clickhouse-*-${VERSION_STRING}-2.*.rpm ${PACKAGE_DIR}
+}
+
+function make_tgz {
+    get_version
+    VERSION_STRING+=$VERSION_POSTFIX
+    VERSION=$VERSION_STRING
+    PACKAGE_DIR=../
+
+    for PACKAGE in clickhouse-server clickhouse-client clickhouse-test clickhouse-common-static clickhouse-common-static-dbg; do
+        alien --verbose --to-tgz ${PACKAGE_DIR}${PACKAGE}_${VERSION}_*.deb
+    done
+
+    mv clickhouse-*-${VERSION_STRING}.tgz ${PACKAGE_DIR}
+}