Merge pull request #48183 from ClickHouse/contrib_sparse_checkout

Add scripts for sparse checkout of some contribs

.github/workflows/backport_branches.yml

@@ -349,6 +349,13 @@ jobs:
         with:
           clear-repository: true
           submodules: true
+      - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
+        run: |
+          rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
+          git -C "$GITHUB_WORKSPACE" checkout .  && echo 'restored'
+          "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK'
+          du -hs "$GITHUB_WORKSPACE/contrib" ||:
+          find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||:
       - name: Build
         run: |
           sudo rm -fr "$TEMP_PATH"

.github/workflows/master.yml

@@ -487,6 +487,13 @@ jobs:
         with:
           clear-repository: true
           submodules: true
+      - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
+        run: |
+          rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
+          git -C "$GITHUB_WORKSPACE" checkout .  && echo 'restored'
+          "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK'
+          du -hs "$GITHUB_WORKSPACE/contrib" ||:
+          find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||:
       - name: Build
         run: |
           sudo rm -fr "$TEMP_PATH"

.github/workflows/pull_request.yml

@@ -550,6 +550,13 @@ jobs:
         with:
           clear-repository: true
           submodules: true
+      - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
+        run: |
+          rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
+          git -C "$GITHUB_WORKSPACE" checkout .  && echo 'restored'
+          "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK'
+          du -hs "$GITHUB_WORKSPACE/contrib" ||:
+          find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||:
       - name: Build
         run: |
           sudo rm -fr "$TEMP_PATH"

.github/workflows/release_branches.yml

@@ -406,6 +406,13 @@ jobs:
         with:
           clear-repository: true
           submodules: true
+      - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
+        run: |
+          rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
+          git -C "$GITHUB_WORKSPACE" checkout .  && echo 'restored'
+          "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK'
+          du -hs "$GITHUB_WORKSPACE/contrib" ||:
+          find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||:
       - name: Build
         run: |
           sudo rm -fr "$TEMP_PATH"

contrib/sparse-checkout/setup-sparse-checkout.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -e
+
+git config submodule."contrib/llvm-project".update '!../sparse-checkout/update-llvm-project.sh'
+git config submodule."contrib/croaring".update '!../sparse-checkout/update-croaring.sh'
+git config submodule."contrib/aws".update '!../sparse-checkout/update-aws.sh'
+git config submodule."contrib/openssl".update '!../sparse-checkout/update-openssl.sh'
+git config submodule."contrib/boringssl".update '!../sparse-checkout/update-boringssl.sh'
+git config submodule."contrib/arrow".update '!../sparse-checkout/update-arrow.sh'
+git config submodule."contrib/grpc".update '!../sparse-checkout/update-grpc.sh'
+git config submodule."contrib/orc".update '!../sparse-checkout/update-orc.sh'
+git config submodule."contrib/h3".update '!../sparse-checkout/update-h3.sh'
+git config submodule."contrib/icu".update '!../sparse-checkout/update-icu.sh'
+git config submodule."contrib/boost".update '!../sparse-checkout/update-boost.sh'
+git config submodule."contrib/aws-s2n-tls".update '!../sparse-checkout/update-aws-s2n-tls.sh'
+git config submodule."contrib/protobuf".update '!../sparse-checkout/update-protobuf.sh'
+git config submodule."contrib/libxml2".update '!../sparse-checkout/update-libxml2.sh'
+git config submodule."contrib/brotli".update '!../sparse-checkout/update-brotli.sh'

contrib/sparse-checkout/update-arrow.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+echo "Using sparse checkout for arrow"
+
+FILES_TO_CHECKOUT=$(git rev-parse --git-dir)/info/sparse-checkout
+echo '/*' > $FILES_TO_CHECKOUT
+echo '!/*/*' >> $FILES_TO_CHECKOUT
+echo '/cpp/*' >> $FILES_TO_CHECKOUT
+
+git config core.sparsecheckout true
+git checkout $1
+git read-tree -mu HEAD

contrib/sparse-checkout/update-aws-s2n-tls.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+echo "Using sparse checkout for aws-s2n-tls"
+
+FILES_TO_CHECKOUT=$(git rev-parse --git-dir)/info/sparse-checkout
+echo '/*' > $FILES_TO_CHECKOUT
+echo '!/test/*' >> $FILES_TO_CHECKOUT
+echo '!/docs/*' >> $FILES_TO_CHECKOUT
+echo '!/compliance/*' >> $FILES_TO_CHECKOUT
+
+git config core.sparsecheckout true
+git checkout $1
+git read-tree -mu HEAD

contrib/sparse-checkout/update-aws.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+echo "Using sparse checkout for aws"
+
+FILES_TO_CHECKOUT=$(git rev-parse --git-dir)/info/sparse-checkout
+echo '/*' > $FILES_TO_CHECKOUT
+echo '!/*/*' >> $FILES_TO_CHECKOUT
+echo '/aws-cpp-sdk-core/*' >> $FILES_TO_CHECKOUT
+echo '/aws-cpp-sdk-s3/*' >> $FILES_TO_CHECKOUT
+
+git config core.sparsecheckout true
+git checkout $1
+git read-tree -mu HEAD

contrib/sparse-checkout/update-boost.sh

@@ -0,0 +1,85 @@
+#!/bin/sh
+
+echo "Using sparse checkout for boost"
+
+FILES_TO_CHECKOUT=$(git rev-parse --git-dir)/info/sparse-checkout
+echo '/*' > $FILES_TO_CHECKOUT
+echo '!/*/*' >> $FILES_TO_CHECKOUT
+echo '/boost/*' > $FILES_TO_CHECKOUT
+echo '!/boost/*/*' >> $FILES_TO_CHECKOUT
+echo '/boost/algorithm/*' >> $FILES_TO_CHECKOUT
+echo '/boost/any/*' >> $FILES_TO_CHECKOUT
+echo '/boost/atomic/*' >> $FILES_TO_CHECKOUT
+echo '/boost/assert/*' >> $FILES_TO_CHECKOUT
+echo '/boost/bind/*' >> $FILES_TO_CHECKOUT
+echo '/boost/concept/*' >> $FILES_TO_CHECKOUT
+echo '/boost/config/*' >> $FILES_TO_CHECKOUT
+echo '/boost/container/*' >> $FILES_TO_CHECKOUT
+echo '/boost/container_hash/*' >> $FILES_TO_CHECKOUT
+echo '/boost/context/*' >> $FILES_TO_CHECKOUT
+echo '/boost/convert/*' >> $FILES_TO_CHECKOUT
+echo '/boost/coroutine/*' >> $FILES_TO_CHECKOUT
+echo '/boost/core/*' >> $FILES_TO_CHECKOUT
+echo '/boost/detail/*' >> $FILES_TO_CHECKOUT
+echo '/boost/dynamic_bitset/*' >> $FILES_TO_CHECKOUT
+echo '/boost/exception/*' >> $FILES_TO_CHECKOUT
+echo '/boost/filesystem/*' >> $FILES_TO_CHECKOUT
+echo '/boost/functional/*' >> $FILES_TO_CHECKOUT
+echo '/boost/function/*' >> $FILES_TO_CHECKOUT
+echo '/boost/geometry/*' >> $FILES_TO_CHECKOUT
+echo '/boost/graph/*' >> $FILES_TO_CHECKOUT
+echo '/boost/heap/*' >> $FILES_TO_CHECKOUT
+echo '/boost/integer/*' >> $FILES_TO_CHECKOUT
+echo '/boost/intrusive/*' >> $FILES_TO_CHECKOUT
+echo '/boost/iostreams/*' >> $FILES_TO_CHECKOUT
+echo '/boost/io/*' >> $FILES_TO_CHECKOUT
+echo '/boost/iterator/*' >> $FILES_TO_CHECKOUT
+echo '/boost/math/*' >> $FILES_TO_CHECKOUT
+echo '/boost/move/*' >> $FILES_TO_CHECKOUT
+echo '/boost/mpl/*' >> $FILES_TO_CHECKOUT
+echo '/boost/multi_index/*' >> $FILES_TO_CHECKOUT
+echo '/boost/multiprecision/*' >> $FILES_TO_CHECKOUT
+echo '/boost/numeric/*' >> $FILES_TO_CHECKOUT
+echo '/boost/predef/*' >> $FILES_TO_CHECKOUT
+echo '/boost/preprocessor/*' >> $FILES_TO_CHECKOUT
+echo '/boost/program_options/*' >> $FILES_TO_CHECKOUT
+echo '/boost/range/*' >> $FILES_TO_CHECKOUT
+echo '/boost/regex/*' >> $FILES_TO_CHECKOUT
+echo '/boost/smart_ptr/*' >> $FILES_TO_CHECKOUT
+echo '/boost/type_index/*' >> $FILES_TO_CHECKOUT
+echo '/boost/type_traits/*' >> $FILES_TO_CHECKOUT
+echo '/boost/system/*' >> $FILES_TO_CHECKOUT
+echo '/boost/tti/*' >> $FILES_TO_CHECKOUT
+echo '/boost/utility/*' >> $FILES_TO_CHECKOUT
+echo '/boost/lexical_cast/*' >> $FILES_TO_CHECKOUT
+echo '/boost/optional/*' >> $FILES_TO_CHECKOUT
+echo '/boost/property_map/*' >> $FILES_TO_CHECKOUT
+echo '/boost/pending/*' >> $FILES_TO_CHECKOUT
+echo '/boost/multi_array/*' >> $FILES_TO_CHECKOUT
+echo '/boost/tuple/*' >> $FILES_TO_CHECKOUT
+echo '/boost/icl/*' >> $FILES_TO_CHECKOUT
+echo '/boost/unordered/*' >> $FILES_TO_CHECKOUT
+echo '/boost/typeof/*' >> $FILES_TO_CHECKOUT
+echo '/boost/parameter/*' >> $FILES_TO_CHECKOUT
+echo '/boost/mp11/*' >> $FILES_TO_CHECKOUT
+echo '/boost/archive/*' >> $FILES_TO_CHECKOUT
+echo '/boost/function_types/*' >> $FILES_TO_CHECKOUT
+echo '/boost/serialization/*' >> $FILES_TO_CHECKOUT
+echo '/boost/fusion/*' >> $FILES_TO_CHECKOUT
+echo '/boost/variant/*' >> $FILES_TO_CHECKOUT
+echo '/boost/format/*' >> $FILES_TO_CHECKOUT
+echo '/boost/locale/*' >> $FILES_TO_CHECKOUT
+echo '/boost/random/*' >> $FILES_TO_CHECKOUT
+echo '/boost/spirit/*' >> $FILES_TO_CHECKOUT
+echo '/boost/uuid/*' >> $FILES_TO_CHECKOUT
+echo '/boost/xpressive/*' >> $FILES_TO_CHECKOUT
+echo '/boost/asio/*' >> $FILES_TO_CHECKOUT
+echo '/boost/circular_buffer/*' >> $FILES_TO_CHECKOUT
+echo '/boost/proto/*' >> $FILES_TO_CHECKOUT
+echo '/boost/qvm/*' >> $FILES_TO_CHECKOUT
+echo '/boost/property_tree/*' >> $FILES_TO_CHECKOUT
+echo '/libs/*' >> $FILES_TO_CHECKOUT
+
+git config core.sparsecheckout true
+git checkout $1
+git read-tree -mu HEAD

contrib/sparse-checkout/update-boringssl.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+echo "Using sparse checkout for boringsll"
+
+FILES_TO_CHECKOUT=$(git rev-parse --git-dir)/info/sparse-checkout
+echo '/*' > $FILES_TO_CHECKOUT
+echo '!/fuzz/*' >> $FILES_TO_CHECKOUT
+echo '!/crypto/cipher_extra/test/*' >> $FILES_TO_CHECKOUT
+echo '!/third_party/wycheproof_testvectors/*' >> $FILES_TO_CHECKOUT
+echo '!/third_party/googletest/*' >> $FILES_TO_CHECKOUT
+
+git config core.sparsecheckout true
+git checkout $1
+git read-tree -mu HEAD

contrib/sparse-checkout/update-brotli.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+echo "Using sparse checkout for brotli"
+
+FILES_TO_CHECKOUT=$(git rev-parse --git-dir)/info/sparse-checkout
+echo '/*' > $FILES_TO_CHECKOUT
+echo '!/*/*' >> $FILES_TO_CHECKOUT
+echo '/c/*' >> $FILES_TO_CHECKOUT
+
+git config core.sparsecheckout true
+git checkout $1
+git read-tree -mu HEAD

contrib/sparse-checkout/update-croaring.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+echo "Using sparse checkout for croaring"
+
+FILES_TO_CHECKOUT=$(git rev-parse --git-dir)/info/sparse-checkout
+echo '/*' > $FILES_TO_CHECKOUT
+echo '!/benchmarks/*' >> $FILES_TO_CHECKOUT
+echo '!/tests/*' >> $FILES_TO_CHECKOUT
+
+git config core.sparsecheckout true
+git checkout $1
+git read-tree -mu HEAD

contrib/sparse-checkout/update-grpc.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+echo "Using sparse checkout for grpc"
+
+FILES_TO_CHECKOUT=$(git rev-parse --git-dir)/info/sparse-checkout
+echo '/*' > $FILES_TO_CHECKOUT
+echo '!/test/*' >> $FILES_TO_CHECKOUT
+echo '/test/build/*' >> $FILES_TO_CHECKOUT
+echo '!/tools/*' >> $FILES_TO_CHECKOUT
+echo '/tools/codegen/*' >> $FILES_TO_CHECKOUT
+echo '!/examples/*' >> $FILES_TO_CHECKOUT
+echo '!/doc/*' >> $FILES_TO_CHECKOUT
+# FIXME why do we need csharp?
+#echo '!/src/csharp/*' >> $FILES_TO_CHECKOUT
+echo '!/src/python/*' >> $FILES_TO_CHECKOUT
+echo '!/src/objective-c/*' >> $FILES_TO_CHECKOUT
+echo '!/src/php/*' >> $FILES_TO_CHECKOUT
+echo '!/src/ruby/*' >> $FILES_TO_CHECKOUT
+
+git config core.sparsecheckout true
+git checkout $1
+git read-tree -mu HEAD

contrib/sparse-checkout/update-h3.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+echo "Using sparse checkout for h3"
+
+FILES_TO_CHECKOUT=$(git rev-parse --git-dir)/info/sparse-checkout
+echo '/*' > $FILES_TO_CHECKOUT
+echo '!/tests/*' >> $FILES_TO_CHECKOUT
+echo '!/website/*' >> $FILES_TO_CHECKOUT
+
+git config core.sparsecheckout true
+git checkout $1
+git read-tree -mu HEAD

contrib/sparse-checkout/update-icu.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+echo "Using sparse checkout for icu"
+
+FILES_TO_CHECKOUT=$(git rev-parse --git-dir)/info/sparse-checkout
+echo '/*' > $FILES_TO_CHECKOUT
+echo '!/*/*' >> $FILES_TO_CHECKOUT
+echo '/icu4c/*' >> $FILES_TO_CHECKOUT
+
+git config core.sparsecheckout true
+git checkout $1
+git read-tree -mu HEAD

contrib/sparse-checkout/update-libxml2.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+echo "Using sparse checkout for libxml2"
+
+FILES_TO_CHECKOUT=$(git rev-parse --git-dir)/info/sparse-checkout
+echo '/*' > $FILES_TO_CHECKOUT
+echo '!/result/*' >> $FILES_TO_CHECKOUT
+echo '!/test/*' >> $FILES_TO_CHECKOUT
+echo '!/doc/*' >> $FILES_TO_CHECKOUT
+echo '!/os400/*' >> $FILES_TO_CHECKOUT
+echo '!/fuzz/*' >> $FILES_TO_CHECKOUT
+echo '!/python/*' >> $FILES_TO_CHECKOUT
+
+git config core.sparsecheckout true
+git checkout $1
+git read-tree -mu HEAD

contrib/sparse-checkout/update-llvm-project.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+echo "Using sparse checkout for llvm-project"
+
+FILES_TO_CHECKOUT=$(git rev-parse --git-dir)/info/sparse-checkout
+echo '/*' > $FILES_TO_CHECKOUT
+echo '!/*/*' >> $FILES_TO_CHECKOUT
+echo '/llvm/*' >> $FILES_TO_CHECKOUT
+echo '!/llvm/*/*' >> $FILES_TO_CHECKOUT
+echo '/llvm/cmake/*' >> $FILES_TO_CHECKOUT
+echo '/llvm/projects/*' >> $FILES_TO_CHECKOUT
+echo '/llvm/include/*' >> $FILES_TO_CHECKOUT
+echo '/llvm/lib/*' >> $FILES_TO_CHECKOUT
+echo '/llvm/utils/TableGen/*' >> $FILES_TO_CHECKOUT
+echo '/libcxxabi/*' >> $FILES_TO_CHECKOUT
+echo '!/libcxxabi/test/*' >> $FILES_TO_CHECKOUT
+echo '/libcxx/*' >> $FILES_TO_CHECKOUT
+echo '!/libcxx/test/*' >> $FILES_TO_CHECKOUT
+echo '/libunwind/*' >> $FILES_TO_CHECKOUT
+echo '!/libunwind/test/*' >> $FILES_TO_CHECKOUT
+echo '/compiler-rt/*' >> $FILES_TO_CHECKOUT
+echo '!/compiler-rt/test/*' >> $FILES_TO_CHECKOUT
+echo '/cmake/*' >> $FILES_TO_CHECKOUT
+
+git config core.sparsecheckout true
+git checkout $1
+git read-tree -mu HEAD

contrib/sparse-checkout/update-openssl.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+echo "Using sparse checkout for openssl"
+
+FILES_TO_CHECKOUT=$(git rev-parse --git-dir)/info/sparse-checkout
+echo '/*' > $FILES_TO_CHECKOUT
+echo '!/fuzz/*' >> $FILES_TO_CHECKOUT
+echo '!/test/*' >> $FILES_TO_CHECKOUT
+echo '!/doc/*' >> $FILES_TO_CHECKOUT
+echo '!/providers/*' >> $FILES_TO_CHECKOUT
+echo '!/apps/*' >> $FILES_TO_CHECKOUT
+
+git config core.sparsecheckout true
+git checkout $1
+git read-tree -mu HEAD

contrib/sparse-checkout/update-orc.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+echo "Using sparse checkout for orc"
+
+FILES_TO_CHECKOUT=$(git rev-parse --git-dir)/info/sparse-checkout
+echo '/*' > $FILES_TO_CHECKOUT
+echo '!/*/*' >> $FILES_TO_CHECKOUT
+echo '/c++/*' >> $FILES_TO_CHECKOUT
+echo '/proto/*' >> $FILES_TO_CHECKOUT
+
+git config core.sparsecheckout true
+git checkout $1
+git read-tree -mu HEAD

contrib/sparse-checkout/update-protobuf.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+echo "Using sparse checkout for protobuf"
+
+FILES_TO_CHECKOUT=$(git rev-parse --git-dir)/info/sparse-checkout
+echo '!/*' > $FILES_TO_CHECKOUT
+echo '/*/*' >> $FILES_TO_CHECKOUT
+echo '/src/*' >> $FILES_TO_CHECKOUT
+echo '/cmake/*' >> $FILES_TO_CHECKOUT
+
+git config core.sparsecheckout true
+git checkout $1
+git read-tree -mu HEAD

contrib/update-submodules.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+set -e
+
+WORKDIR=$(dirname "$0")
+WORKDIR=$(readlink -f "${WORKDIR}")
+
+"$WORKDIR/sparse-checkout/setup-sparse-checkout.sh"
+git submodule init
+git submodule sync
+git submodule update --depth=1

docs/en/development/developer-instruction.md

@@ -39,9 +39,15 @@ Next, you need to download the source files onto your working machine. This is c
 
 In the command line terminal run:
 
-    git clone --recursive --shallow-submodules git@github.com:your_github_username/ClickHouse.git
+    git clone --shallow-submodules git@github.com:your_github_username/ClickHouse.git
     cd ClickHouse
 
+Or (if you'd like to use sparse checkout for submodules and avoid checking out unneeded files):
+
+    git clone git@github.com:your_github_username/ClickHouse.git
+    cd ClickHouse
+    ./contrib/update-submodules.sh
+
 Note: please, substitute *your_github_username* with what is appropriate!
 
 This command will create a directory `ClickHouse` containing the working copy of the project.

docs/ru/development/developer-instruction.md

@@ -41,9 +41,15 @@ ClickHouse не работает и не собирается на 32-битны
 
 Выполните в терминале:
 
-    git clone git@github.com:your_github_username/ClickHouse.git --recursive
+    git clone --shallow-submodules git@github.com:your_github_username/ClickHouse.git
     cd ClickHouse
 
+Или (если вы хотите использовать sparse checkout для submodules):
+
+    git clone git@github.com:your_github_username/ClickHouse.git
+    cd ClickHouse
+    ./contrib/update-submodules.sh
+
 Замените слово `your_github_username` в команде для git на имя вашего аккаунта на GitHub.
 
 Эта команда создаст директорию ClickHouse, содержащую рабочую копию проекта.