thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 23:52:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into disable-hedged-requests-under-tsan

Browse Source
This commit is contained in:
Kruglov Pavel 2023-07-04 17:37:14 +02:00 committed by GitHub
commit bab423acc2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
47 changed files with 750 additions and 875 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
base/poco/Foundation/CMakeLists.txt
View File

@ -87,7 +87,6 @@ set (SRCS
src/LoggingRegistry.cpp
src/LogStream.cpp
src/MD5Engine.cpp
src/MemoryPool.cpp
src/MemoryStream.cpp
src/Message.cpp
src/Mutex.cpp

116
base/poco/Foundation/include/Poco/MemoryPool.h
View File

@ -1,116 +0,0 @@
//
// MemoryPool.h
//
// Library: Foundation
// Package: Core
// Module: MemoryPool
//
// Definition of the MemoryPool class.
//
// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef Foundation_MemoryPool_INCLUDED
#define Foundation_MemoryPool_INCLUDED
#include <cstddef>
#include <vector>
#include "Poco/Foundation.h"
#include "Poco/Mutex.h"
namespace Poco
{
class Foundation_API MemoryPool
/// A simple pool for fixed-size memory blocks.
///
/// The main purpose of this class is to speed-up
/// memory allocations, as well as to reduce memory
/// fragmentation in situations where the same blocks
/// are allocated all over again, such as in server
/// applications.
///
/// All allocated blocks are retained for future use.
/// A limit on the number of blocks can be specified.
/// Blocks can be preallocated.
{
public:
MemoryPool(std::size_t blockSize, int preAlloc = 0, int maxAlloc = 0);
/// Creates a MemoryPool for blocks with the given blockSize.
/// The number of blocks given in preAlloc are preallocated.
~MemoryPool();
void * get();
/// Returns a memory block. If there are no more blocks
/// in the pool, a new block will be allocated.
///
/// If maxAlloc blocks are already allocated, an
/// OutOfMemoryException is thrown.
void release(void * ptr);
/// Releases a memory block and returns it to the pool.
std::size_t blockSize() const;
/// Returns the block size.
int allocated() const;
/// Returns the number of allocated blocks.
int available() const;
/// Returns the number of available blocks in the pool.
private:
MemoryPool();
MemoryPool(const MemoryPool &);
MemoryPool & operator=(const MemoryPool &);
void clear();
enum
{
BLOCK_RESERVE = 128
};
typedef std::vector<char *> BlockVec;
std::size_t _blockSize;
int _maxAlloc;
int _allocated;
BlockVec _blocks;
FastMutex _mutex;
};
//
// inlines
//
inline std::size_t MemoryPool::blockSize() const
{
return _blockSize;
}
inline int MemoryPool::allocated() const
{
return _allocated;
}
inline int MemoryPool::available() const
{
return (int)_blocks.size();
}
} // namespace Poco
#endif // Foundation_MemoryPool_INCLUDED

105
base/poco/Foundation/src/MemoryPool.cpp
View File

@ -1,105 +0,0 @@
//
// MemoryPool.cpp
//
// Library: Foundation
// Package: Core
// Module: MemoryPool
//
// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/MemoryPool.h"
#include "Poco/Exception.h"
namespace Poco {
MemoryPool::MemoryPool(std::size_t blockSize, int preAlloc, int maxAlloc):
_blockSize(blockSize),
_maxAlloc(maxAlloc),
_allocated(preAlloc)
{
poco_assert (maxAlloc == 0 || maxAlloc >= preAlloc);
poco_assert (preAlloc >= 0 && maxAlloc >= 0);
int r = BLOCK_RESERVE;
if (preAlloc > r)
r = preAlloc;
if (maxAlloc > 0 && maxAlloc < r)
r = maxAlloc;
_blocks.reserve(r);
try
{
for (int i = 0; i < preAlloc; ++i)
{
_blocks.push_back(new char[_blockSize]);
}
}
catch (...)
{
clear();
throw;
}
}
MemoryPool::~MemoryPool()
{
clear();
}
void MemoryPool::clear()
{
for (BlockVec::iterator it = _blocks.begin(); it != _blocks.end(); ++it)
{
delete [] *it;
}
_blocks.clear();
}
void* MemoryPool::get()
{
FastMutex::ScopedLock lock(_mutex);
if (_blocks.empty())
{
if (_maxAlloc == 0 || _allocated < _maxAlloc)
{
++_allocated;
return new char[_blockSize];
}
else throw OutOfMemoryException("MemoryPool exhausted");
}
else
{
char* ptr = _blocks.back();
_blocks.pop_back();
return ptr;
}
}
void MemoryPool::release(void* ptr)
{
FastMutex::ScopedLock lock(_mutex);
try
{
_blocks.push_back(reinterpret_cast<char*>(ptr));
}
catch (...)
{
delete [] reinterpret_cast<char*>(ptr);
}
}
} // namespace Poco

5
base/poco/Net/include/Poco/Net/HTTPBasicStreamBuf.h
View File

@ -19,7 +19,6 @@
#include "Poco/BufferedStreamBuf.h"
#include "Poco/Net/HTTPBufferAllocator.h"
#include "Poco/Net/Net.h"
@ -27,9 +26,9 @@ namespace Poco
{
namespace Net
{
constexpr size_t HTTP_DEFAULT_BUFFER_SIZE = 8 * 1024;
typedef Poco::BasicBufferedStreamBuf<char, std::char_traits<char>, HTTPBufferAllocator> HTTPBasicStreamBuf;
typedef Poco::BasicBufferedStreamBuf<char, std::char_traits<char>> HTTPBasicStreamBuf;
}

53
base/poco/Net/include/Poco/Net/HTTPBufferAllocator.h
View File

@ -1,53 +0,0 @@
//
// HTTPBufferAllocator.h
//
// Library: Net
// Package: HTTP
// Module: HTTPBufferAllocator
//
// Definition of the HTTPBufferAllocator class.
//
// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef Net_HTTPBufferAllocator_INCLUDED
#define Net_HTTPBufferAllocator_INCLUDED
#include <ios>
#include "Poco/MemoryPool.h"
#include "Poco/Net/Net.h"
namespace Poco
{
namespace Net
{
class Net_API HTTPBufferAllocator
/// A BufferAllocator for HTTP streams.
{
public:
static char * allocate(std::streamsize size);
static void deallocate(char * ptr, std::streamsize size);
enum
{
BUFFER_SIZE = 128 * 1024
};
private:
static Poco::MemoryPool _pool;
};
}
} // namespace Poco::Net
#endif // Net_HTTPBufferAllocator_INCLUDED

13
base/poco/Net/include/Poco/Net/HTTPChunkedStream.h
View File

@ -21,7 +21,6 @@
#include <cstddef>
#include <istream>
#include <ostream>
#include "Poco/MemoryPool.h"
#include "Poco/Net/HTTPBasicStreamBuf.h"
#include "Poco/Net/Net.h"
@ -80,12 +79,6 @@ namespace Net
public:
HTTPChunkedInputStream(HTTPSession & session);
~HTTPChunkedInputStream();
void * operator new(std::size_t size);
void operator delete(void * ptr);
private:
static Poco::MemoryPool _pool;
};
@ -95,12 +88,6 @@ namespace Net
public:
HTTPChunkedOutputStream(HTTPSession & session);
~HTTPChunkedOutputStream();
void * operator new(std::size_t size);
void operator delete(void * ptr);
private:
static Poco::MemoryPool _pool;
};

12
base/poco/Net/include/Poco/Net/HTTPFixedLengthStream.h
View File

@ -78,12 +78,6 @@ namespace Net
public:
HTTPFixedLengthInputStream(HTTPSession & session, HTTPFixedLengthStreamBuf::ContentLength length);
~HTTPFixedLengthInputStream();
void * operator new(std::size_t size);
void operator delete(void * ptr);
private:
static Poco::MemoryPool _pool;
};
@ -93,12 +87,6 @@ namespace Net
public:
HTTPFixedLengthOutputStream(HTTPSession & session, HTTPFixedLengthStreamBuf::ContentLength length);
~HTTPFixedLengthOutputStream();
void * operator new(std::size_t size);
void operator delete(void * ptr);
private:
static Poco::MemoryPool _pool;
};

13
base/poco/Net/include/Poco/Net/HTTPHeaderStream.h
View File

@ -21,7 +21,6 @@
#include <cstddef>
#include <istream>
#include <ostream>
#include "Poco/MemoryPool.h"
#include "Poco/Net/HTTPBasicStreamBuf.h"
#include "Poco/Net/Net.h"
@ -74,12 +73,6 @@ namespace Net
public:
HTTPHeaderInputStream(HTTPSession & session);
~HTTPHeaderInputStream();
void * operator new(std::size_t size);
void operator delete(void * ptr);
private:
static Poco::MemoryPool _pool;
};
@ -89,12 +82,6 @@ namespace Net
public:
HTTPHeaderOutputStream(HTTPSession & session);
~HTTPHeaderOutputStream();
void * operator new(std::size_t size);
void operator delete(void * ptr);
private:
static Poco::MemoryPool _pool;
};

2
base/poco/Net/include/Poco/Net/HTTPSession.h
View File

@ -192,7 +192,7 @@ namespace Net
HTTPSession & operator=(const HTTPSession &);
StreamSocket _socket;
char * _pBuffer;
std::unique_ptr<char[]> _pBuffer;
char * _pCurrent;
char * _pEnd;
bool _keepAlive;

13
base/poco/Net/include/Poco/Net/HTTPStream.h
View File

@ -21,7 +21,6 @@
#include <cstddef>
#include <istream>
#include <ostream>
#include "Poco/MemoryPool.h"
#include "Poco/Net/HTTPBasicStreamBuf.h"
#include "Poco/Net/Net.h"
@ -75,12 +74,6 @@ namespace Net
public:
HTTPInputStream(HTTPSession & session);
~HTTPInputStream();
void * operator new(std::size_t size);
void operator delete(void * ptr);
private:
static Poco::MemoryPool _pool;
};
@ -90,12 +83,6 @@ namespace Net
public:
HTTPOutputStream(HTTPSession & session);
~HTTPOutputStream();
void * operator new(std::size_t size);
void operator delete(void * ptr);
private:
static Poco::MemoryPool _pool;
};

44
base/poco/Net/src/HTTPBufferAllocator.cpp
View File

@ -1,44 +0,0 @@
//
// HTTPBufferAllocator.cpp
//
// Library: Net
// Package: HTTP
// Module: HTTPBufferAllocator
//
// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/Net/HTTPBufferAllocator.h"
using Poco::MemoryPool;
namespace Poco {
namespace Net {
MemoryPool HTTPBufferAllocator::_pool(HTTPBufferAllocator::BUFFER_SIZE, 16);
char* HTTPBufferAllocator::allocate(std::streamsize size)
{
poco_assert_dbg (size == BUFFER_SIZE);
return reinterpret_cast<char*>(_pool.get());
}
void HTTPBufferAllocator::deallocate(char* ptr, std::streamsize size)
{
poco_assert_dbg (size == BUFFER_SIZE);
_pool.release(ptr);
}
} } // namespace Poco::Net

50
base/poco/Net/src/HTTPChunkedStream.cpp
View File

@ -34,7 +34,7 @@ namespace Net {
HTTPChunkedStreamBuf::HTTPChunkedStreamBuf(HTTPSession& session, openmode mode):
HTTPBasicStreamBuf(HTTPBufferAllocator::BUFFER_SIZE, mode),
HTTPBasicStreamBuf(HTTP_DEFAULT_BUFFER_SIZE, mode),
_session(session),
_mode(mode),
_chunk(0)
@ -181,10 +181,6 @@ HTTPChunkedStreamBuf* HTTPChunkedIOS::rdbuf()
// HTTPChunkedInputStream
//
Poco::MemoryPool HTTPChunkedInputStream::_pool(sizeof(HTTPChunkedInputStream));
HTTPChunkedInputStream::HTTPChunkedInputStream(HTTPSession& session):
HTTPChunkedIOS(session, std::ios::in),
std::istream(&_buf)
@ -196,34 +192,10 @@ HTTPChunkedInputStream::~HTTPChunkedInputStream()
{
}
void* HTTPChunkedInputStream::operator new(std::size_t size)
{
return _pool.get();
}
void HTTPChunkedInputStream::operator delete(void* ptr)
{
try
{
_pool.release(ptr);
}
catch (...)
{
poco_unexpected();
}
}
//
// HTTPChunkedOutputStream
//
Poco::MemoryPool HTTPChunkedOutputStream::_pool(sizeof(HTTPChunkedOutputStream));
HTTPChunkedOutputStream::HTTPChunkedOutputStream(HTTPSession& session):
HTTPChunkedIOS(session, std::ios::out),
std::ostream(&_buf)
@ -235,24 +207,4 @@ HTTPChunkedOutputStream::~HTTPChunkedOutputStream()
{
}
void* HTTPChunkedOutputStream::operator new(std::size_t size)
{
return _pool.get();
}
void HTTPChunkedOutputStream::operator delete(void* ptr)
{
try
{
_pool.release(ptr);
}
catch (...)
{
poco_unexpected();
}
}
} } // namespace Poco::Net

47
base/poco/Net/src/HTTPFixedLengthStream.cpp
View File

@ -30,7 +30,7 @@ namespace Net {
HTTPFixedLengthStreamBuf::HTTPFixedLengthStreamBuf(HTTPSession& session, ContentLength length, openmode mode):
HTTPBasicStreamBuf(HTTPBufferAllocator::BUFFER_SIZE, mode),
HTTPBasicStreamBuf(HTTP_DEFAULT_BUFFER_SIZE, mode),
_session(session),
_length(length),
_count(0)
@ -109,9 +109,6 @@ HTTPFixedLengthStreamBuf* HTTPFixedLengthIOS::rdbuf()
//
Poco::MemoryPool HTTPFixedLengthInputStream::_pool(sizeof(HTTPFixedLengthInputStream));
HTTPFixedLengthInputStream::HTTPFixedLengthInputStream(HTTPSession& session, HTTPFixedLengthStreamBuf::ContentLength length):
HTTPFixedLengthIOS(session, length, std::ios::in),
std::istream(&_buf)
@ -124,33 +121,10 @@ HTTPFixedLengthInputStream::~HTTPFixedLengthInputStream()
}
void* HTTPFixedLengthInputStream::operator new(std::size_t size)
{
return _pool.get();
}
void HTTPFixedLengthInputStream::operator delete(void* ptr)
{
try
{
_pool.release(ptr);
}
catch (...)
{
poco_unexpected();
}
}
//
// HTTPFixedLengthOutputStream
//
Poco::MemoryPool HTTPFixedLengthOutputStream::_pool(sizeof(HTTPFixedLengthOutputStream));
HTTPFixedLengthOutputStream::HTTPFixedLengthOutputStream(HTTPSession& session, HTTPFixedLengthStreamBuf::ContentLength length):
HTTPFixedLengthIOS(session, length, std::ios::out),
std::ostream(&_buf)
@ -163,23 +137,4 @@ HTTPFixedLengthOutputStream::~HTTPFixedLengthOutputStream()
}
void* HTTPFixedLengthOutputStream::operator new(std::size_t size)
{
return _pool.get();
}
void HTTPFixedLengthOutputStream::operator delete(void* ptr)
{
try
{
_pool.release(ptr);
}
catch (...)
{
poco_unexpected();
}
}
} } // namespace Poco::Net

50
base/poco/Net/src/HTTPHeaderStream.cpp
View File

@ -26,7 +26,7 @@ namespace Net {
HTTPHeaderStreamBuf::HTTPHeaderStreamBuf(HTTPSession& session, openmode mode):
HTTPBasicStreamBuf(HTTPBufferAllocator::BUFFER_SIZE, mode),
HTTPBasicStreamBuf(HTTP_DEFAULT_BUFFER_SIZE, mode),
_session(session),
_end(false)
{
@ -101,10 +101,6 @@ HTTPHeaderStreamBuf* HTTPHeaderIOS::rdbuf()
// HTTPHeaderInputStream
//
Poco::MemoryPool HTTPHeaderInputStream::_pool(sizeof(HTTPHeaderInputStream));
HTTPHeaderInputStream::HTTPHeaderInputStream(HTTPSession& session):
HTTPHeaderIOS(session, std::ios::in),
std::istream(&_buf)
@ -116,34 +112,10 @@ HTTPHeaderInputStream::~HTTPHeaderInputStream()
{
}
void* HTTPHeaderInputStream::operator new(std::size_t size)
{
return _pool.get();
}
void HTTPHeaderInputStream::operator delete(void* ptr)
{
try
{
_pool.release(ptr);
}
catch (...)
{
poco_unexpected();
}
}
//
// HTTPHeaderOutputStream
//
Poco::MemoryPool HTTPHeaderOutputStream::_pool(sizeof(HTTPHeaderOutputStream));
HTTPHeaderOutputStream::HTTPHeaderOutputStream(HTTPSession& session):
HTTPHeaderIOS(session, std::ios::out),
std::ostream(&_buf)
@ -155,24 +127,4 @@ HTTPHeaderOutputStream::~HTTPHeaderOutputStream()
{
}
void* HTTPHeaderOutputStream::operator new(std::size_t size)
{
return _pool.get();
}
void HTTPHeaderOutputStream::operator delete(void* ptr)
{
try
{
_pool.release(ptr);
}
catch (...)
{
poco_unexpected();
}
}
} } // namespace Poco::Net

18
base/poco/Net/src/HTTPSession.cpp
View File

@ -13,8 +13,8 @@
#include "Poco/Net/HTTPSession.h"
#include "Poco/Net/HTTPBufferAllocator.h"
#include "Poco/Net/NetException.h"
#include "Poco/Net/HTTPBasicStreamBuf.h"
#include <cstring>
@ -68,14 +68,6 @@ HTTPSession::HTTPSession(const StreamSocket& socket, bool keepAlive):
HTTPSession::~HTTPSession()
{
try
{
if (_pBuffer) HTTPBufferAllocator::deallocate(_pBuffer, HTTPBufferAllocator::BUFFER_SIZE);
}
catch (...)
{
poco_unexpected();
}
try
{
close();
@ -177,10 +169,10 @@ void HTTPSession::refill()
{
if (!_pBuffer)
{
_pBuffer = HTTPBufferAllocator::allocate(HTTPBufferAllocator::BUFFER_SIZE);
_pBuffer = std::make_unique<char[]>(HTTP_DEFAULT_BUFFER_SIZE);
}
_pCurrent = _pEnd = _pBuffer;
int n = receive(_pBuffer, HTTPBufferAllocator::BUFFER_SIZE);
_pCurrent = _pEnd = _pBuffer.get();
int n = receive(_pBuffer.get(), HTTP_DEFAULT_BUFFER_SIZE);
_pEnd += n;
}
@ -199,7 +191,7 @@ void HTTPSession::connect(const SocketAddress& address)
_socket.setNoDelay(true);
// There may be leftover data from a previous (failed) request in the buffer,
// so we clear it.
_pCurrent = _pEnd = _pBuffer;
_pCurrent = _pEnd = _pBuffer.get();
}

48
base/poco/Net/src/HTTPStream.cpp
View File

@ -26,7 +26,7 @@ namespace Net {
HTTPStreamBuf::HTTPStreamBuf(HTTPSession& session, openmode mode):
HTTPBasicStreamBuf(HTTPBufferAllocator::BUFFER_SIZE, mode),
HTTPBasicStreamBuf(HTTP_DEFAULT_BUFFER_SIZE, mode),
_session(session),
_mode(mode)
{
@ -96,10 +96,6 @@ HTTPStreamBuf* HTTPIOS::rdbuf()
// HTTPInputStream
//
Poco::MemoryPool HTTPInputStream::_pool(sizeof(HTTPInputStream));
HTTPInputStream::HTTPInputStream(HTTPSession& session):
HTTPIOS(session, std::ios::in),
std::istream(&_buf)
@ -112,33 +108,11 @@ HTTPInputStream::~HTTPInputStream()
}
void* HTTPInputStream::operator new(std::size_t size)
{
return _pool.get();
}
void HTTPInputStream::operator delete(void* ptr)
{
try
{
_pool.release(ptr);
}
catch (...)
{
poco_unexpected();
}
}
//
// HTTPOutputStream
//
Poco::MemoryPool HTTPOutputStream::_pool(sizeof(HTTPOutputStream));
HTTPOutputStream::HTTPOutputStream(HTTPSession& session):
HTTPIOS(session, std::ios::out),
std::ostream(&_buf)
@ -150,24 +124,4 @@ HTTPOutputStream::~HTTPOutputStream()
{
}
void* HTTPOutputStream::operator new(std::size_t size)
{
return _pool.get();
}
void HTTPOutputStream::operator delete(void* ptr)
{
try
{
_pool.release(ptr);
}
catch (...)
{
poco_unexpected();
}
}
} } // namespace Poco::Net

2
docker/packager/README.md
View File

@ -6,7 +6,7 @@ Usage:
Build deb package with `clang-14` in `debug` mode:
```
$ mkdir deb/test_output
$ ./packager --output-dir deb/test_output/ --package-type deb --compiler=clang-14 --debug-build
$ ./packager --output-dir deb/test_output/ --package-type deb --compiler=clang-14 --build-type=debug
$ ls -l deb/test_output
-rw-r--r-- 1 root root 3730 clickhouse-client_22.2.2+debug_all.deb
-rw-r--r-- 1 root root 84221888 clickhouse-common-static_22.2.2+debug_amd64.deb

16
docker/packager/packager
View File

@ -112,12 +112,12 @@ def run_docker_image_with_env(
subprocess.check_call(cmd, shell=True)
def is_release_build(debug_build: bool, package_type: str, sanitizer: str) -> bool:
return not debug_build and package_type == "deb" and sanitizer == ""
def is_release_build(build_type: str, package_type: str, sanitizer: str) -> bool:
return build_type == "" and package_type == "deb" and sanitizer == ""
def parse_env_variables(
debug_build: bool,
build_type: str,
compiler: str,
sanitizer: str,
package_type: str,
@ -233,7 +233,7 @@ def parse_env_variables(
build_target = (
f"{build_target} clickhouse-odbc-bridge clickhouse-library-bridge"
)
if is_release_build(debug_build, package_type, sanitizer):
if is_release_build(build_type, package_type, sanitizer):
cmake_flags.append("-DSPLIT_DEBUG_SYMBOLS=ON")
result.append("WITH_PERFORMANCE=1")
if is_cross_arm:
@ -253,8 +253,8 @@ def parse_env_variables(
if sanitizer:
result.append(f"SANITIZER={sanitizer}")
if debug_build:
result.append("BUILD_TYPE=DEBUG")
if build_type:
result.append(f"BUILD_TYPE={build_type.capitalize()}")
else:
result.append("BUILD_TYPE=None")
@ -359,7 +359,7 @@ def parse_args() -> argparse.Namespace:
help="ClickHouse git repository",
)
parser.add_argument("--output-dir", type=dir_name, required=True)
parser.add_argument("--debug-build", action="store_true")
parser.add_argument("--build-type", choices=("debug", ""), default="")
parser.add_argument(
"--compiler",
@ -464,7 +464,7 @@ def main():
build_image(image_with_version, dockerfile)
env_prepared = parse_env_variables(
args.debug_build,
args.build_type,
args.compiler,
args.sanitizer,
args.package_type,

2
docker/test/style/Dockerfile
View File

@ -18,7 +18,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
python3-pip \
shellcheck \
yamllint \
&& pip3 install black==23.1.0 boto3 codespell==2.2.1 dohq-artifactory mypy PyGithub unidiff pylint==2.6.2 \
&& pip3 install black==23.1.0 boto3 codespell==2.2.1 mypy==1.3.0 PyGithub unidiff pylint==2.6.2 \
&& apt-get clean \
&& rm -rf /root/.cache/pip

14
docs/en/development/build.md
View File

@ -13,20 +13,6 @@ Supported platforms:
- AArch64
- Power9 (experimental)
## Building in docker
We use the docker image `clickhouse/binary-builder` for our CI builds. It contains everything necessary to build the binary and packages. There is a script `docker/packager/packager` to ease the image usage:
```bash
# define a directory for the output artifacts
output_dir="build_results"
# a simplest build
./docker/packager/packager --package-type=binary --output-dir "$output_dir"
# build debian packages
./docker/packager/packager --package-type=deb --output-dir "$output_dir"
# by default, debian packages use thin LTO, so we can override it to speed up the build
CMAKE_FLAGS='-DENABLE_THINLTO=' ./docker/packager/packager --package-type=deb --output-dir "$output_dir"
```
## Building on Ubuntu
The following tutorial is based on Ubuntu Linux.

4
docs/en/getting-started/install.md
View File

@ -378,6 +378,10 @@ request](https://github.com/ClickHouse/ClickHouse/commits/master) and find CI ch
https://s3.amazonaws.com/clickhouse/builds/PRs/.../.../binary_aarch64_v80compat/clickhouse". You can then click the link to download the
build.
### macOS-only: Install with Homebrew
To install ClickHouse using the popular `brew` package manager, follow the instructions listed in the [ClickHouse Homebrew tap](https://github.com/ClickHouse/homebrew-clickhouse).
## Launch {#launch}
To start the server as a daemon, run:

360
rust/skim/Cargo.lock generated
View File

@ -14,13 +14,19 @@ dependencies = [
[[package]]
name = "aho-corasick"
version = "0.7.20"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
dependencies = [
"memchr",
]
[[package]]
name = "android-tzdata"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
[[package]]
name = "android_system_properties"
version = "0.1.5"
@ -32,9 +38,9 @@ dependencies = [
[[package]]
name = "arrayvec"
version = "0.7.2"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
[[package]]
name = "atty"
@ -42,7 +48,7 @@ version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [
"hermit-abi",
"hermit-abi 0.1.19",
"libc",
"winapi",
]
@ -67,15 +73,15 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bumpalo"
version = "3.11.1"
version = "3.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba"
checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
[[package]]
name = "cc"
version = "1.0.77"
version = "1.0.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4"
checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
[[package]]
name = "cfg-if"
@ -85,13 +91,13 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chrono"
version = "0.4.23"
version = "0.4.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f"
checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5"
dependencies = [
"android-tzdata",
"iana-time-zone",
"js-sys",
"num-integer",
"num-traits",
"time 0.1.45",
"wasm-bindgen",
@ -100,9 +106,9 @@ dependencies = [
[[package]]
name = "clap"
version = "3.2.23"
version = "3.2.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"
checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123"
dependencies = [
"atty",
"bitflags",
@ -135,9 +141,9 @@ dependencies = [
[[package]]
name = "core-foundation-sys"
version = "0.8.3"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
[[package]]
name = "crossbeam"
@ -155,9 +161,9 @@ dependencies = [
[[package]]
name = "crossbeam-channel"
version = "0.5.6"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
dependencies = [
"cfg-if",
"crossbeam-utils",
@ -165,9 +171,9 @@ dependencies = [
[[package]]
name = "crossbeam-deque"
version = "0.8.2"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc"
checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
dependencies = [
"cfg-if",
"crossbeam-epoch",
@ -176,14 +182,14 @@ dependencies = [
[[package]]
name = "crossbeam-epoch"
version = "0.9.13"
version = "0.9.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a"
checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"memoffset 0.7.1",
"memoffset 0.9.0",
"scopeguard",
]
@ -199,18 +205,18 @@ dependencies = [
[[package]]
name = "crossbeam-utils"
version = "0.8.14"
version = "0.8.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f"
checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
dependencies = [
"cfg-if",
]
[[package]]
name = "cxx"
version = "1.0.83"
version = "1.0.97"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bdf07d07d6531bfcdbe9b8b739b104610c6508dcc4d63b410585faf338241daf"
checksum = "e88abab2f5abbe4c56e8f1fb431b784d710b709888f35755a160e62e33fe38e8"
dependencies = [
"cc",
"cxxbridge-flags",
@ -220,9 +226,9 @@ dependencies = [
[[package]]
name = "cxx-build"
version = "1.0.83"
version = "1.0.97"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2eb5b96ecdc99f72657332953d4d9c50135af1bac34277801cc3937906ebd39"
checksum = "5c0c11acd0e63bae27dcd2afced407063312771212b7a823b4fd72d633be30fb"
dependencies = [
"cc",
"codespan-reporting",
@ -230,31 +236,31 @@ dependencies = [
"proc-macro2",
"quote",
"scratch",
"syn",
"syn 2.0.23",
]
[[package]]
name = "cxxbridge-flags"
version = "1.0.83"
version = "1.0.97"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac040a39517fd1674e0f32177648334b0f4074625b5588a64519804ba0553b12"
checksum = "8d3816ed957c008ccd4728485511e3d9aaf7db419aa321e3d2c5a2f3411e36c8"
[[package]]
name = "cxxbridge-macro"
version = "1.0.83"
version = "1.0.97"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1362b0ddcfc4eb0a1f57b68bd77dd99f0e826958a96abd0ae9bd092e114ffed6"
checksum = "a26acccf6f445af85ea056362561a24ef56cdc15fcc685f03aec50b9c702cb6d"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.23",
]
[[package]]
name = "darling"
version = "0.14.2"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0dd3cd20dc6b5a876612a6e5accfe7f3dd883db6d07acfbf14c128f61550dfa"
checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
dependencies = [
"darling_core",
"darling_macro",
@ -262,27 +268,27 @@ dependencies = [
[[package]]
name = "darling_core"
version = "0.14.2"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a784d2ccaf7c98501746bf0be29b2022ba41fd62a2e622af997a03e9f972859f"
checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn",
"syn 1.0.109",
]
[[package]]
name = "darling_macro"
version = "0.14.2"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e"
checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
dependencies = [
"darling_core",
"quote",
"syn",
"syn 1.0.109",
]
[[package]]
@ -313,7 +319,7 @@ dependencies = [
"darling",
"proc-macro2",
"quote",
"syn",
"syn 1.0.109",
]
[[package]]
@ -323,7 +329,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68"
dependencies = [
"derive_builder_core",
"syn",
"syn 1.0.109",
]
[[package]]
@ -349,9 +355,9 @@ dependencies = [
[[package]]
name = "either"
version = "1.8.0"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797"
checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
[[package]]
name = "env_logger"
@ -383,9 +389,9 @@ dependencies = [
[[package]]
name = "getrandom"
version = "0.2.8"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
dependencies = [
"cfg-if",
"libc",
@ -407,6 +413,12 @@ dependencies = [
"libc",
]
[[package]]
name = "hermit-abi"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
[[package]]
name = "humantime"
version = "2.1.0"
@ -415,26 +427,25 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "iana-time-zone"
version = "0.1.53"
version = "0.1.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765"
checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613"
dependencies = [
"android_system_properties",
"core-foundation-sys",
"iana-time-zone-haiku",
"js-sys",
"wasm-bindgen",
"winapi",
"windows",
]
[[package]]
name = "iana-time-zone-haiku"
version = "0.1.1"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca"
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
dependencies = [
"cxx",
"cxx-build",
"cc",
]
[[package]]
@ -445,9 +456,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]]
name = "indexmap"
version = "1.9.2"
version = "1.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399"
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
dependencies = [
"autocfg",
"hashbrown",
@ -455,9 +466,9 @@ dependencies = [
[[package]]
name = "js-sys"
version = "0.3.60"
version = "0.3.64"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47"
checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a"
dependencies = [
"wasm-bindgen",
]
@ -470,27 +481,24 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.138"
version = "0.2.147"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8"
checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
[[package]]
name = "link-cplusplus"
version = "1.0.7"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369"
checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5"
dependencies = [
"cc",
]
[[package]]
name = "log"
version = "0.4.17"
version = "0.4.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
dependencies = [
"cfg-if",
]
checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
[[package]]
name = "memchr"
@ -509,9 +517,9 @@ dependencies = [
[[package]]
name = "memoffset"
version = "0.7.1"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
dependencies = [
"autocfg",
]
@ -541,16 +549,6 @@ dependencies = [
"pin-utils",
]
[[package]]
name = "num-integer"
version = "0.1.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
dependencies = [
"autocfg",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.15"
@ -562,25 +560,25 @@ dependencies = [
[[package]]
name = "num_cpus"
version = "1.14.0"
version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5"
checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
dependencies = [
"hermit-abi",
"hermit-abi 0.3.1",
"libc",
]
[[package]]
name = "once_cell"
version = "1.16.0"
version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
[[package]]
name = "os_str_bytes"
version = "6.4.1"
version = "6.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee"
checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac"
[[package]]
name = "pin-utils"
@ -590,27 +588,27 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "proc-macro2"
version = "1.0.47"
version = "1.0.63"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725"
checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.21"
version = "1.0.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105"
dependencies = [
"proc-macro2",
]
[[package]]
name = "rayon"
version = "1.6.1"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7"
checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
dependencies = [
"either",
"rayon-core",
@ -618,9 +616,9 @@ dependencies = [
[[package]]
name = "rayon-core"
version = "1.10.1"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3"
checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
@ -650,9 +648,9 @@ dependencies = [
[[package]]
name = "regex"
version = "1.7.0"
version = "1.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a"
checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f"
dependencies = [
"aho-corasick",
"memchr",
@ -661,15 +659,15 @@ dependencies = [
[[package]]
name = "regex-syntax"
version = "0.6.28"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
[[package]]
name = "rustversion"
version = "1.0.9"
version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8"
checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06"
[[package]]
name = "scopeguard"
@ -679,15 +677,15 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "scratch"
version = "1.0.2"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898"
checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1"
[[package]]
name = "serde"
version = "1.0.149"
version = "1.0.164"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "256b9932320c590e707b94576e3cc1f7c9024d0ee6612dfbcf1cb106cbe8e055"
checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d"
[[package]]
name = "shlex"
@ -697,9 +695,9 @@ checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
[[package]]
name = "skim"
version = "0.10.2"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cebed5f897cd6c0d80fbe30adb36c0abf7400e93043a63ae56458495642b3485"
checksum = "e5d28de0a6cb2cdd83a076f1de9d965b973ae08b244df1aa70b432946dda0f32"
dependencies = [
"atty",
"beef",
@ -717,7 +715,7 @@ dependencies = [
"rayon",
"regex",
"shlex",
"time 0.3.17",
"time 0.3.22",
"timer",
"tuikit",
"unicode-width",
@ -732,9 +730,20 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "syn"
version = "1.0.105"
version = "1.0.109"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908"
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "syn"
version = "2.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59fb7d6d8281a51045d62b8eb3a7d1ce347b76f312af50cd3dc0af39c87c1737"
dependencies = [
"proc-macro2",
"quote",
@ -754,9 +763,9 @@ dependencies = [
[[package]]
name = "termcolor"
version = "1.1.3"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
dependencies = [
"winapi-util",
]
@ -769,30 +778,31 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
[[package]]
name = "thiserror"
version = "1.0.37"
version = "1.0.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e"
checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.37"
version = "1.0.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb"
checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.23",
]
[[package]]
name = "thread_local"
version = "1.1.4"
version = "1.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180"
checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152"
dependencies = [
"cfg-if",
"once_cell",
]
@ -809,9 +819,9 @@ dependencies = [
[[package]]
name = "time"
version = "0.3.17"
version = "0.3.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376"
checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd"
dependencies = [
"serde",
"time-core",
@ -819,9 +829,9 @@ dependencies = [
[[package]]
name = "time-core"
version = "0.1.0"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd"
checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
[[package]]
name = "timer"
@ -848,9 +858,9 @@ dependencies = [
[[package]]
name = "unicode-ident"
version = "1.0.5"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0"
[[package]]
name = "unicode-width"
@ -860,15 +870,15 @@ checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
[[package]]
name = "utf8parse"
version = "0.2.0"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372"
checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
[[package]]
name = "vte"
version = "0.11.0"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aae21c12ad2ec2d168c236f369c38ff332bc1134f7246350dca641437365045"
checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197"
dependencies = [
"arrayvec",
"utf8parse",
@ -899,9 +909,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.83"
version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"
checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
@ -909,24 +919,24 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.83"
version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"
checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd"
dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2",
"quote",
"syn",
"syn 2.0.23",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.83"
version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"
checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@ -934,22 +944,22 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.83"
version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.23",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.83"
version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
[[package]]
name = "winapi"
@ -981,3 +991,69 @@ name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-targets"
version = "0.48.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
[[package]]
name = "windows_aarch64_msvc"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
[[package]]
name = "windows_i686_gnu"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
[[package]]
name = "windows_i686_msvc"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
[[package]]
name = "windows_x86_64_gnu"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
[[package]]
name = "windows_x86_64_msvc"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"

16
src/Access/ContextAccess.cpp
View File

@ -228,6 +228,12 @@ ContextAccess::ContextAccess(const AccessControl & access_control_, const Params
}
ContextAccess::ContextAccess(FullAccess)
: is_full_access(true), access(std::make_shared<AccessRights>(AccessRights::getFullAccess())), access_with_implicit(access)
{
}
ContextAccess::~ContextAccess()
{
enabled_settings.reset();
@ -413,14 +419,8 @@ std::optional<QuotaUsage> ContextAccess::getQuotaUsage() const
std::shared_ptr<const ContextAccess> ContextAccess::getFullAccess()
{
static const std::shared_ptr<const ContextAccess> res = []
{
auto full_access = std::make_shared<ContextAccess>();
full_access->is_full_access = true;
full_access->access = std::make_shared<AccessRights>(AccessRights::getFullAccess());
full_access->access_with_implicit = full_access->access;
return full_access;
}();
static const std::shared_ptr<const ContextAccess> res =
[] { return std::shared_ptr<ContextAccess>(new ContextAccess{kFullAccess}); }();
return res;
}

45
src/Access/ContextAccess.h
View File

@ -69,7 +69,6 @@ public:
using Params = ContextAccessParams;
const Params & getParams() const { return params; }
ContextAccess() { } /// NOLINT
ContextAccess(const AccessControl & access_control_, const Params & params_);
/// Returns the current user. Throws if user is nullptr.
@ -171,10 +170,17 @@ public:
private:
friend class AccessControl;
struct FullAccess {};
static const FullAccess kFullAccess;
/// Makes an instance of ContextAccess which provides full access to everything
/// without any limitations. This is used for the global context.
explicit ContextAccess(FullAccess);
void initialize();
void setUser(const UserPtr & user_) const;
void setRolesInfo(const std::shared_ptr<const EnabledRolesInfo> & roles_info_) const;
void calculateAccessRights() const;
void setUser(const UserPtr & user_) const TSA_REQUIRES(mutex);
void setRolesInfo(const std::shared_ptr<const EnabledRolesInfo> & roles_info_) const TSA_REQUIRES(mutex);
void calculateAccessRights() const TSA_REQUIRES(mutex);
template <bool throw_if_denied, bool grant_option>
bool checkAccessImpl(const AccessFlags & flags) const;
@ -217,20 +223,23 @@ private:
const AccessControl * access_control = nullptr;
const Params params;
bool is_full_access = false;
mutable Poco::Logger * trace_log = nullptr;
mutable UserPtr user;
mutable String user_name;
mutable bool user_was_dropped = false;
mutable scope_guard subscription_for_user_change;
mutable std::shared_ptr<const EnabledRoles> enabled_roles;
mutable scope_guard subscription_for_roles_changes;
mutable std::shared_ptr<const EnabledRolesInfo> roles_info;
mutable std::shared_ptr<const AccessRights> access;
mutable std::shared_ptr<const AccessRights> access_with_implicit;
mutable std::shared_ptr<const EnabledRowPolicies> enabled_row_policies;
mutable std::shared_ptr<const EnabledQuota> enabled_quota;
mutable std::shared_ptr<const EnabledSettings> enabled_settings;
const bool is_full_access = false;
mutable std::atomic<bool> user_was_dropped = false;
mutable std::atomic<Poco::Logger *> trace_log = nullptr;
mutable UserPtr user TSA_GUARDED_BY(mutex);
mutable String user_name TSA_GUARDED_BY(mutex);
mutable scope_guard subscription_for_user_change TSA_GUARDED_BY(mutex);
mutable std::shared_ptr<const EnabledRoles> enabled_roles TSA_GUARDED_BY(mutex);
mutable scope_guard subscription_for_roles_changes TSA_GUARDED_BY(mutex);
mutable std::shared_ptr<const EnabledRolesInfo> roles_info TSA_GUARDED_BY(mutex);
mutable std::shared_ptr<const AccessRights> access TSA_GUARDED_BY(mutex);
mutable std::shared_ptr<const AccessRights> access_with_implicit TSA_GUARDED_BY(mutex);
mutable std::shared_ptr<const EnabledRowPolicies> enabled_row_policies TSA_GUARDED_BY(mutex);
mutable std::shared_ptr<const EnabledQuota> enabled_quota TSA_GUARDED_BY(mutex);
mutable std::shared_ptr<const EnabledSettings> enabled_settings TSA_GUARDED_BY(mutex);
mutable std::mutex mutex;
};

1
src/Common/CurrentMetrics.cpp
View File

@ -188,6 +188,7 @@
M(CacheDetachedFileSegments, "Number of existing detached cache file segments") \
M(FilesystemCacheSize, "Filesystem cache size in bytes") \
M(FilesystemCacheElements, "Filesystem cache elements (file segments)") \
M(FilesystemCacheDownloadQueueElements, "Filesystem cache elements in download queue") \
M(AsyncInsertCacheSize, "Number of async insert hash id in cache") \
M(S3Requests, "S3 requests") \
M(KeeperAliveConnections, "Number of alive connections") \

50
src/Interpreters/Cache/FileCache.cpp
View File

@ -11,6 +11,7 @@
#include <base/hex.h>
#include <pcg-random/pcg_random.hpp>
#include <Common/randomSeed.h>
#include <Common/ThreadPool.h>
#include <Common/ElapsedTimeProfileEventIncrement.h>
#include <filesystem>
@ -54,9 +55,10 @@ FileCache::FileCache(const FileCacheSettings & settings)
: max_file_segment_size(settings.max_file_segment_size)
, bypass_cache_threshold(settings.enable_bypass_cache_with_threashold ? settings.bypass_cache_threashold : 0)
, delayed_cleanup_interval_ms(settings.delayed_cleanup_interval_ms)
, boundary_alignment(settings.boundary_alignment)
, background_download_threads(settings.background_download_threads)
, log(&Poco::Logger::get("FileCache"))
, metadata(settings.base_path)
, boundary_alignment(settings.boundary_alignment)
{
main_priority = std::make_unique<LRUFileCachePriority>(settings.max_size, settings.max_elements);
@ -129,6 +131,9 @@ void FileCache::initialize()
is_initialized = true;
for (size_t i = 0; i < background_download_threads; ++i)
download_threads.emplace_back([this] { metadata.downloadThreadFunc(); });
cleanup_task = Context::getGlobalContextInstance()->getSchedulePool().createTask("FileCacheCleanup", [this]{ cleanupThreadFunc(); });
cleanup_task->activate();
cleanup_task->scheduleAfter(delayed_cleanup_interval_ms);
@ -423,7 +428,12 @@ FileSegmentsHolderPtr FileCache::set(
}
FileSegmentsHolderPtr
FileCache::getOrSet(const Key & key, size_t offset, size_t size, size_t file_size, const CreateFileSegmentSettings & settings)
FileCache::getOrSet(
const Key & key,
size_t offset,
size_t size,
size_t file_size,
const CreateFileSegmentSettings & settings)
{
ProfileEventTimeIncrement<Microseconds> watch(ProfileEvents::FilesystemCacheGetOrSetMicroseconds);
@ -645,27 +655,14 @@ bool FileCache::tryReserve(FileSegment & file_segment, const size_t size)
if (segment_metadata->releasable())
{
auto segment = segment_metadata->file_segment;
if (segment->state() == FileSegment::State::DOWNLOADED)
{
const auto & key = segment->key();
const auto & key = segment_metadata->file_segment->key();
auto it = to_delete.find(key);
if (it == to_delete.end())
it = to_delete.emplace(key, locked_key.getKeyMetadata()).first;
it->second.add(segment_metadata);
auto it = to_delete.find(key);
if (it == to_delete.end())
it = to_delete.emplace(key, locked_key.getKeyMetadata()).first;
it->second.add(segment_metadata);
freeable_space += segment_metadata->size();
++freeable_count;
return PriorityIterationResult::CONTINUE;
}
ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictedFileSegments);
ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictedBytes, segment->getDownloadedSize(false));
locked_key.removeFileSegment(segment->offset(), segment->lock());
return PriorityIterationResult::REMOVE_AND_CONTINUE;
freeable_space += segment_metadata->size();
++freeable_count;
}
return PriorityIterationResult::CONTINUE;
};
@ -1027,6 +1024,11 @@ void FileCache::deactivateBackgroundOperations()
{
if (cleanup_task)
cleanup_task->deactivate();
metadata.cancelDownload();
for (auto & thread : download_threads)
if (thread.joinable())
thread.join();
}
void FileCache::cleanup()
@ -1038,10 +1040,6 @@ void FileCache::cleanupThreadFunc()
{
try
{
#ifdef ABORT_ON_LOGICAL_ERROR
assertCacheCorrectness();
#endif
cleanup();
}
catch (...)

6
src/Interpreters/Cache/FileCache.h
View File

@ -136,6 +136,8 @@ private:
const size_t max_file_segment_size;
const size_t bypass_cache_threshold = 0;
const size_t delayed_cleanup_interval_ms;
const size_t boundary_alignment;
const size_t background_download_threads;
Poco::Logger * log;
@ -180,9 +182,9 @@ private:
*/
BackgroundSchedulePool::TaskHolder cleanup_task;
void assertInitialized() const;
std::vector<ThreadFromGlobalPool> download_threads;
size_t boundary_alignment;
void assertInitialized() const;
void assertCacheCorrectness();

11
src/Interpreters/Cache/FileCacheSettings.cpp
View File

@ -31,10 +31,9 @@ void FileCacheSettings::loadFromConfig(const Poco::Util::AbstractConfiguration &
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Disk Cache requires non-empty `path` field (cache base path) in config");
max_elements = config.getUInt64(config_prefix + ".max_elements", FILECACHE_DEFAULT_MAX_ELEMENTS);
if (config.has(config_prefix + ".max_file_segment_size"))
max_file_segment_size = parseWithSizeSuffix<uint64_t>(config.getString(config_prefix + ".max_file_segment_size"));
else
max_file_segment_size = FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE;
cache_on_write_operations = config.getUInt64(config_prefix + ".cache_on_write_operations", false);
enable_filesystem_query_cache_limit = config.getUInt64(config_prefix + ".enable_filesystem_query_cache_limit", false);
@ -44,10 +43,12 @@ void FileCacheSettings::loadFromConfig(const Poco::Util::AbstractConfiguration &
if (config.has(config_prefix + ".bypass_cache_threashold"))
bypass_cache_threashold = parseWithSizeSuffix<uint64_t>(config.getString(config_prefix + ".bypass_cache_threashold"));
else
bypass_cache_threashold = FILECACHE_BYPASS_THRESHOLD;
boundary_alignment = config.getUInt64(config_prefix + ".boundary_alignment", DBMS_DEFAULT_BUFFER_SIZE);
if (config.has(config_prefix + ".boundary_alignment"))
boundary_alignment = parseWithSizeSuffix<uint64_t>(config.getString(config_prefix + ".boundary_alignment"));
if (config.has(config_prefix + ".background_download_threads"))
background_download_threads = config.getUInt(config_prefix + ".background_download_threads");
delayed_cleanup_interval_ms = config.getUInt64(config_prefix + ".delayed_cleanup_interval_ms", FILECACHE_DELAYED_CLEANUP_INTERVAL_MS);
}

3
src/Interpreters/Cache/FileCacheSettings.h
View File

@ -26,7 +26,8 @@ struct FileCacheSettings
size_t bypass_cache_threashold = FILECACHE_BYPASS_THRESHOLD;
size_t delayed_cleanup_interval_ms = FILECACHE_DELAYED_CLEANUP_INTERVAL_MS;
size_t boundary_alignment = DBMS_DEFAULT_BUFFER_SIZE;
size_t boundary_alignment = FILECACHE_DEFAULT_FILE_SEGMENT_ALIGNMENT;
size_t background_download_threads = FILECACHE_DEFAULT_BACKGROUND_DOWNLOAD_THREADS;
void loadFromConfig(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix);
};

4
src/Interpreters/Cache/FileCache_fwd.h
View File

@ -4,7 +4,9 @@
namespace DB
{
static constexpr int FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE = 8 * 1024 * 1024;
static constexpr int FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE = 32 * 1024 * 1024; /// 32Mi
static constexpr int FILECACHE_DEFAULT_FILE_SEGMENT_ALIGNMENT = 4 * 1024 * 1024; /// 4Mi
static constexpr int FILECACHE_DEFAULT_BACKGROUND_DOWNLOAD_THREADS = 2;
static constexpr int FILECACHE_DEFAULT_MAX_ELEMENTS = 10000000;
static constexpr int FILECACHE_DEFAULT_HITS_THRESHOLD = 0;
static constexpr size_t FILECACHE_BYPASS_THRESHOLD = 256 * 1024 * 1024;

64
src/Interpreters/Cache/FileSegment.cpp
View File

@ -237,8 +237,10 @@ void FileSegment::resetDownloadingStateUnlocked(const FileSegmentGuard::Lock & l
/// range().size() can equal 0 in case of write-though cache.
if (!is_unbound && current_downloaded_size != 0 && current_downloaded_size == range().size())
setDownloadedUnlocked(lock);
else
else if (current_downloaded_size)
setDownloadState(State::PARTIALLY_DOWNLOADED, lock);
else
setDownloadState(State::EMPTY, lock);
}
void FileSegment::resetDownloader()
@ -303,22 +305,13 @@ void FileSegment::resetRemoteFileReader()
FileSegment::RemoteFileReaderPtr FileSegment::extractRemoteFileReader()
{
auto locked_key = lockKeyMetadata(false);
if (!locked_key)
auto lock = lockFileSegment();
if (remote_file_reader && (download_state == State::DOWNLOADED
|| download_state == State::PARTIALLY_DOWNLOADED_NO_CONTINUATION))
{
assert(isDetached());
return std::move(remote_file_reader);
}
auto segment_lock = lockFileSegment();
assert(download_state != State::DETACHED);
bool is_last_holder = locked_key->isLastOwnerOfFileSegment(offset());
if (!downloader_id.empty() || !is_last_holder)
return nullptr;
return std::move(remote_file_reader);
return nullptr;
}
void FileSegment::setRemoteFileReader(RemoteFileReaderPtr remote_file_reader_)
@ -635,16 +628,6 @@ void FileSegment::complete()
resetDownloaderUnlocked(segment_lock);
}
if (is_downloader || is_last_holder)
{
if (cache_writer)
{
cache_writer->finalize();
cache_writer.reset();
}
remote_file_reader.reset();
}
if (segment_kind == FileSegmentKind::Temporary && is_last_holder)
{
LOG_TEST(log, "Removing temporary file segment: {}", getInfoForLogUnlocked(segment_lock));
@ -659,6 +642,7 @@ void FileSegment::complete()
chassert(current_downloaded_size == range().size());
chassert(current_downloaded_size == fs::file_size(getPathInLocalCache()));
chassert(!cache_writer);
chassert(!remote_file_reader);
break;
}
case State::DOWNLOADING:
@ -667,7 +651,33 @@ void FileSegment::complete()
break;
}
case State::EMPTY:
{
if (is_last_holder)
locked_key->removeFileSegment(offset(), segment_lock);
break;
}
case State::PARTIALLY_DOWNLOADED:
{
chassert(current_downloaded_size > 0);
if (is_last_holder)
{
if (remote_file_reader)
{
LOG_TEST(
log, "Submitting file segment for background download "
"(having {}/{})", downloaded_size, range().size());
locked_key->addToDownloadQueue(offset(), segment_lock); /// Finish download in background.
}
else
{
locked_key->shrinkFileSegmentToDownloadedSize(offset(), segment_lock);
setDetachedState(segment_lock); /// See comment below.
}
}
break;
}
case State::PARTIALLY_DOWNLOADED_NO_CONTINUATION:
{
chassert(current_downloaded_size != range().size());
@ -676,7 +686,6 @@ void FileSegment::complete()
{
if (current_downloaded_size == 0)
{
LOG_TEST(log, "Remove file segment {} (nothing downloaded)", range().toString());
locked_key->removeFileSegment(offset(), segment_lock);
}
else
@ -698,9 +707,8 @@ void FileSegment::complete()
/// We mark current file segment with state DETACHED, even though the data is still in cache
/// (but a separate file segment) because is_last_holder is satisfied, so it does not matter.
setDetachedState(segment_lock);
}
setDetachedState(segment_lock);
}
break;
}
@ -870,6 +878,8 @@ void FileSegment::setDetachedState(const FileSegmentGuard::Lock & lock)
key_metadata.reset();
cache = nullptr;
queue_iterator = nullptr;
cache_writer.reset();
remote_file_reader.reset();
}
void FileSegment::detach(const FileSegmentGuard::Lock & lock, const LockedKey &)

194
src/Interpreters/Cache/Metadata.cpp
View File

@ -1,12 +1,18 @@
#include <Interpreters/Cache/Metadata.h>
#include <Interpreters/Cache/FileCache.h>
#include <Interpreters/Cache/FileSegment.h>
#include "Common/Exception.h"
#include <Common/logger_useful.h>
#include <Common/ElapsedTimeProfileEventIncrement.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace CurrentMetrics
{
extern const Metric FilesystemCacheDownloadQueueElements;
}
namespace ProfileEvents
{
extern const Event FilesystemCacheLockKeyMicroseconds;
@ -53,11 +59,13 @@ KeyMetadata::KeyMetadata(
const Key & key_,
const std::string & key_path_,
CleanupQueue & cleanup_queue_,
DownloadQueue & download_queue_,
Poco::Logger * log_,
bool created_base_directory_)
: key(key_)
, key_path(key_path_)
, cleanup_queue(cleanup_queue_)
, download_queue(download_queue_)
, created_base_directory(created_base_directory_)
, log(log_)
{
@ -134,6 +142,7 @@ private:
CacheMetadata::CacheMetadata(const std::string & path_)
: path(path_)
, cleanup_queue(std::make_unique<CleanupQueue>())
, download_queue(std::make_unique<DownloadQueue>())
, log(&Poco::Logger::get("CacheMetadata"))
{
}
@ -188,7 +197,7 @@ LockedKeyPtr CacheMetadata::lockKeyMetadata(
it = emplace(
key, std::make_shared<KeyMetadata>(
key, getPathForKey(key), *cleanup_queue, log, is_initial_load)).first;
key, getPathForKey(key), *cleanup_queue, *download_queue, log, is_initial_load)).first;
}
key_metadata = it->second;
@ -279,6 +288,7 @@ void CacheMetadata::doCleanup()
continue;
}
chassert(it->second->empty());
locked_metadata->markAsRemoved();
erase(it);
LOG_DEBUG(log, "Key {} is removed from metadata", cleanup_key);
@ -319,6 +329,180 @@ void CacheMetadata::doCleanup()
}
}
class DownloadQueue
{
friend struct CacheMetadata;
public:
void add(std::weak_ptr<FileSegment> file_segment)
{
{
std::lock_guard lock(mutex);
queue.push(file_segment);
}
CurrentMetrics::add(CurrentMetrics::FilesystemCacheDownloadQueueElements);
cv.notify_one();
}
private:
void cancel()
{
{
std::lock_guard lock(mutex);
cancelled = true;
}
cv.notify_all();
}
std::mutex mutex;
std::condition_variable cv;
std::queue<std::weak_ptr<FileSegment>> queue;
bool cancelled = false;
};
void CacheMetadata::downloadThreadFunc()
{
std::optional<Memory<>> memory;
while (true)
{
std::weak_ptr<FileSegment> file_segment_weak;
{
std::unique_lock lock(download_queue->mutex);
if (download_queue->cancelled)
return;
if (download_queue->queue.empty())
{
download_queue->cv.wait(lock);
continue;
}
file_segment_weak = download_queue->queue.front();
download_queue->queue.pop();
}
CurrentMetrics::sub(CurrentMetrics::FilesystemCacheDownloadQueueElements);
FileSegmentsHolderPtr holder;
try
{
{
auto file_segment = file_segment_weak.lock();
if (!file_segment
|| file_segment->state() != FileSegment::State::PARTIALLY_DOWNLOADED)
continue;
auto locked_key = lockKeyMetadata(file_segment->key(), KeyNotFoundPolicy::RETURN_NULL);
if (!locked_key)
continue;
auto file_segment_metadata = locked_key->tryGetByOffset(file_segment->offset());
if (!file_segment_metadata || file_segment_metadata->evicting())
continue;
holder = std::make_unique<FileSegmentsHolder>(FileSegments{file_segment});
}
downloadImpl(holder->front(), memory);
}
catch (...)
{
if (holder)
{
const auto & file_segment = holder->front();
LOG_ERROR(
log, "Error during background download of {}:{} ({}): {}",
file_segment.key(), file_segment.offset(),
file_segment.getInfoForLog(), getCurrentExceptionMessage(true));
}
else
{
tryLogCurrentException(__PRETTY_FUNCTION__);
chassert(false);
}
}
}
}
void CacheMetadata::downloadImpl(FileSegment & file_segment, std::optional<Memory<>> & memory)
{
chassert(file_segment.assertCorrectness());
if (file_segment.getOrSetDownloader() != FileSegment::getCallerId())
return;
if (file_segment.getDownloadedSize(false) == file_segment.range().size())
throw Exception(ErrorCodes::LOGICAL_ERROR, "File segment is already fully downloaded");
LOG_TEST(
log, "Downloading {} bytes for file segment {}",
file_segment.range().size() - file_segment.getDownloadedSize(false), file_segment.getInfoForLog());
auto reader = file_segment.getRemoteFileReader();
if (!reader)
{
throw Exception(
ErrorCodes::LOGICAL_ERROR, "No reader. "
"File segment should not have been submitted for background download ({})",
file_segment.getInfoForLog());
}
/// If remote_fs_read_method == 'threadpool',
/// reader itself never owns/allocates the buffer.
if (reader->internalBuffer().empty())
{
if (!memory)
memory.emplace(DBMS_DEFAULT_BUFFER_SIZE);
reader->set(memory->data(), memory->size());
}
size_t offset = file_segment.getCurrentWriteOffset(false);
if (offset != static_cast<size_t>(reader->getPosition()))
reader->seek(offset, SEEK_SET);
while (!reader->eof())
{
auto size = reader->available();
if (!file_segment.reserve(size))
{
LOG_TEST(
log, "Failed to reserve space during background download "
"for {}:{} (downloaded size: {}/{})",
file_segment.key(), file_segment.offset(),
file_segment.getDownloadedSize(false), file_segment.range().size());
return;
}
try
{
file_segment.write(reader->position(), size, offset);
offset += size;
reader->position() += size;
}
catch (ErrnoException & e)
{
int code = e.getErrno();
if (code == /* No space left on device */28 || code == /* Quota exceeded */122)
{
LOG_INFO(log, "Insert into cache is skipped due to insufficient disk space. ({})", e.displayText());
return;
}
throw;
}
}
LOG_TEST(log, "Downloaded file segment: {}", file_segment.getInfoForLog());
}
void CacheMetadata::cancelDownload()
{
download_queue->cancel();
}
LockedKey::LockedKey(std::shared_ptr<KeyMetadata> key_metadata_)
: key_metadata(key_metadata_)
, lock(key_metadata->guard.lock())
@ -456,6 +640,14 @@ void LockedKey::shrinkFileSegmentToDownloadedSize(
chassert(file_segment->assertCorrectnessUnlocked(segment_lock));
}
void LockedKey::addToDownloadQueue(size_t offset, const FileSegmentGuard::Lock &)
{
auto it = key_metadata->find(offset);
if (it == key_metadata->end())
throw Exception(ErrorCodes::LOGICAL_ERROR, "There is not offset {}", offset);
key_metadata->download_queue.add(it->second->file_segment);
}
std::optional<FileSegment::Range> LockedKey::hasIntersectingRange(const FileSegment::Range & range) const
{
if (key_metadata->empty())

15
src/Interpreters/Cache/Metadata.h
View File

@ -8,8 +8,12 @@
namespace DB
{
class CleanupQueue;
using CleanupQueuePtr = std::shared_ptr<CleanupQueue>;
class DownloadQueue;
using DownloadQueuePtr = std::shared_ptr<DownloadQueue>;
using FileSegmentsHolderPtr = std::unique_ptr<FileSegmentsHolder>;
struct FileSegmentMetadata : private boost::noncopyable
@ -44,6 +48,7 @@ struct KeyMetadata : public std::map<size_t, FileSegmentMetadataPtr>,
const Key & key_,
const std::string & key_path_,
CleanupQueue & cleanup_queue_,
DownloadQueue & download_queue_,
Poco::Logger * log_,
bool created_base_directory_ = false);
@ -70,6 +75,7 @@ private:
KeyState key_state = KeyState::ACTIVE;
KeyGuard guard;
CleanupQueue & cleanup_queue;
DownloadQueue & download_queue;
std::atomic<bool> created_base_directory = false;
Poco::Logger * log;
};
@ -111,12 +117,19 @@ public:
void doCleanup();
void downloadThreadFunc();
void cancelDownload();
private:
CacheMetadataGuard::Lock lockMetadata() const;
const std::string path; /// Cache base path
mutable CacheMetadataGuard guard;
const CleanupQueuePtr cleanup_queue;
const DownloadQueuePtr download_queue;
Poco::Logger * log;
void downloadImpl(FileSegment & file_segment, std::optional<Memory<>> & memory);
};
@ -162,6 +175,8 @@ struct LockedKey : private boost::noncopyable
void shrinkFileSegmentToDownloadedSize(size_t offset, const FileSegmentGuard::Lock &);
void addToDownloadQueue(size_t offset, const FileSegmentGuard::Lock &);
bool isLastOwnerOfFileSegment(size_t offset) const;
std::optional<FileSegment::Range> hasIntersectingRange(const FileSegment::Range & range) const;

25
src/Interpreters/InterpreterDescribeCacheQuery.cpp
View File

@ -19,11 +19,15 @@ static Block getSampleBlock()
ColumnWithTypeAndName{std::make_shared<DataTypeUInt64>(), "max_size"},
ColumnWithTypeAndName{std::make_shared<DataTypeUInt64>(), "max_elements"},
ColumnWithTypeAndName{std::make_shared<DataTypeUInt64>(), "max_file_segment_size"},
ColumnWithTypeAndName{std::make_shared<DataTypeUInt64>(), "boundary_alignment"},
ColumnWithTypeAndName{std::make_shared<DataTypeNumber<UInt8>>(), "cache_on_write_operations"},
ColumnWithTypeAndName{std::make_shared<DataTypeNumber<UInt8>>(), "cache_hits_threshold"},
ColumnWithTypeAndName{std::make_shared<DataTypeUInt64>(), "current_size"},
ColumnWithTypeAndName{std::make_shared<DataTypeUInt64>(), "current_elements"},
ColumnWithTypeAndName{std::make_shared<DataTypeString>(), "path"},
ColumnWithTypeAndName{std::make_shared<DataTypeNumber<UInt64>>(), "delayed_cleanup_interval_ms"},
ColumnWithTypeAndName{std::make_shared<DataTypeNumber<UInt64>>(), "background_download_threads"},
ColumnWithTypeAndName{std::make_shared<DataTypeNumber<UInt64>>(), "enable_bypass_cache_with_threshold"},
};
return Block(columns);
}
@ -40,14 +44,19 @@ BlockIO InterpreterDescribeCacheQuery::execute()
const auto & settings = cache_data.settings;
const auto & cache = cache_data.cache;
res_columns[0]->insert(settings.max_size);
res_columns[1]->insert(settings.max_elements);
res_columns[2]->insert(settings.max_file_segment_size);
res_columns[3]->insert(settings.cache_on_write_operations);
res_columns[4]->insert(settings.cache_hits_threshold);
res_columns[5]->insert(cache->getUsedCacheSize());
res_columns[6]->insert(cache->getFileSegmentsNum());
res_columns[7]->insert(cache->getBasePath());
size_t i = 0;
res_columns[i++]->insert(settings.max_size);
res_columns[i++]->insert(settings.max_elements);
res_columns[i++]->insert(settings.max_file_segment_size);
res_columns[i++]->insert(settings.boundary_alignment);
res_columns[i++]->insert(settings.cache_on_write_operations);
res_columns[i++]->insert(settings.cache_hits_threshold);
res_columns[i++]->insert(cache->getUsedCacheSize());
res_columns[i++]->insert(cache->getFileSegmentsNum());
res_columns[i++]->insert(cache->getBasePath());
res_columns[i++]->insert(settings.delayed_cleanup_interval_ms);
res_columns[i++]->insert(settings.background_download_threads);
res_columns[i++]->insert(settings.enable_bypass_cache_with_threashold);
BlockIO res;
size_t num_rows = res_columns[0]->size();

4
src/Interpreters/tests/gtest_lru_file_cache.cpp
View File

@ -544,8 +544,8 @@ TEST_F(FileCacheTest, get)
cv.notify_one();
file_segment2.wait(file_segment2.range().left);
ASSERT_TRUE(file_segment2.state() == DB::FileSegment::State::PARTIALLY_DOWNLOADED);
ASSERT_TRUE(file_segment2.getOrSetDownloader() == DB::FileSegment::getCallerId());
ASSERT_EQ(file_segment2.state(), DB::FileSegment::State::EMPTY);
ASSERT_EQ(file_segment2.getOrSetDownloader(), DB::FileSegment::getCallerId());
download(file_segment2);
});

6
tests/ci/build_check.py
View File

@ -45,7 +45,7 @@ def _can_export_binaries(build_config: BuildConfig) -> bool:
return False
if build_config["sanitizer"] != "":
return True
if build_config["debug_build"]:
if build_config["build_type"] != "":
return True
return False
@ -66,8 +66,8 @@ def get_packager_cmd(
f"--package-type={package_type} --compiler={comp}"
)
if build_config["debug_build"]:
cmd += " --debug-build"
if build_config["build_type"]:
cmd += f" --build-type={build_config['build_type']}"
if build_config["sanitizer"]:
cmd += f" --sanitizer={build_config['sanitizer']}"
if build_config["tidy"] == "enable":

4
tests/ci/build_report_check.py
View File

@ -70,7 +70,7 @@ def get_failed_report(
message = f"{job_name} failed"
build_result = BuildResult(
compiler="unknown",
debug_build=False,
build_type="unknown",
sanitizer="unknown",
status=message,
elapsed_seconds=0,
@ -85,7 +85,7 @@ def process_report(
build_config = build_report["build_config"]
build_result = BuildResult(
compiler=build_config["compiler"],
debug_build=build_config["debug_build"],
build_type=build_config["build_type"],
sanitizer=build_config["sanitizer"],
status="success" if build_report["status"] else "failure",
elapsed_seconds=build_report["elapsed_seconds"],

34
tests/ci/ci_config.py
View File

@ -10,7 +10,7 @@ CI_CONFIG = {
"build_config": {
"package_release": {
"compiler": "clang-16",
"debug_build": False,
"build_type": "",
"sanitizer": "",
"package_type": "deb",
"static_binary_name": "amd64",
@ -21,7 +21,7 @@ CI_CONFIG = {
},
"coverity": {
"compiler": "clang-16",
"debug_build": False,
"build_type": "",
"sanitizer": "",
"package_type": "coverity",
"tidy": "disable",
@ -31,7 +31,7 @@ CI_CONFIG = {
},
"package_aarch64": {
"compiler": "clang-16-aarch64",
"debug_build": False,
"build_type": "",
"sanitizer": "",
"package_type": "deb",
"static_binary_name": "aarch64",
@ -42,7 +42,7 @@ CI_CONFIG = {
},
"package_asan": {
"compiler": "clang-16",
"debug_build": False,
"build_type": "",
"sanitizer": "address",
"package_type": "deb",
"tidy": "disable",
@ -51,7 +51,7 @@ CI_CONFIG = {
},
"package_ubsan": {
"compiler": "clang-16",
"debug_build": False,
"build_type": "",
"sanitizer": "undefined",
"package_type": "deb",
"tidy": "disable",
@ -60,7 +60,7 @@ CI_CONFIG = {
},
"package_tsan": {
"compiler": "clang-16",
"debug_build": False,
"build_type": "",
"sanitizer": "thread",
"package_type": "deb",
"tidy": "disable",
@ -69,7 +69,7 @@ CI_CONFIG = {
},
"package_msan": {
"compiler": "clang-16",
"debug_build": False,
"build_type": "",
"sanitizer": "memory",
"package_type": "deb",
"tidy": "disable",
@ -78,7 +78,7 @@ CI_CONFIG = {
},
"package_debug": {
"compiler": "clang-16",
"debug_build": True,
"build_type": "debug",
"sanitizer": "",
"package_type": "deb",
"tidy": "disable",
@ -87,7 +87,7 @@ CI_CONFIG = {
},
"binary_release": {
"compiler": "clang-16",
"debug_build": False,
"build_type": "",
"sanitizer": "",
"package_type": "binary",
"tidy": "disable",
@ -96,7 +96,7 @@ CI_CONFIG = {
},
"binary_tidy": {
"compiler": "clang-16",
"debug_build": True,
"build_type": "debug",
"sanitizer": "",
"package_type": "binary",
"static_binary_name": "debug-amd64",
@ -106,7 +106,7 @@ CI_CONFIG = {
},
"binary_darwin": {
"compiler": "clang-16-darwin",
"debug_build": False,
"build_type": "",
"sanitizer": "",
"package_type": "binary",
"static_binary_name": "macos",
@ -116,7 +116,7 @@ CI_CONFIG = {
},
"binary_aarch64": {
"compiler": "clang-16-aarch64",
"debug_build": False,
"build_type": "",
"sanitizer": "",
"package_type": "binary",
"tidy": "disable",
@ -125,7 +125,7 @@ CI_CONFIG = {
},
"binary_aarch64_v80compat": {
"compiler": "clang-16-aarch64-v80compat",
"debug_build": False,
"build_type": "",
"sanitizer": "",
"package_type": "binary",
"static_binary_name": "aarch64v80compat",
@ -135,7 +135,7 @@ CI_CONFIG = {
},
"binary_freebsd": {
"compiler": "clang-16-freebsd",
"debug_build": False,
"build_type": "",
"sanitizer": "",
"package_type": "binary",
"static_binary_name": "freebsd",
@ -145,7 +145,7 @@ CI_CONFIG = {
},
"binary_darwin_aarch64": {
"compiler": "clang-16-darwin-aarch64",
"debug_build": False,
"build_type": "",
"sanitizer": "",
"package_type": "binary",
"static_binary_name": "macos-aarch64",
@ -155,7 +155,7 @@ CI_CONFIG = {
},
"binary_ppc64le": {
"compiler": "clang-16-ppc64le",
"debug_build": False,
"build_type": "",
"sanitizer": "",
"package_type": "binary",
"static_binary_name": "powerpc64le",
@ -165,7 +165,7 @@ CI_CONFIG = {
},
"binary_amd64_compat": {
"compiler": "clang-16-amd64-compat",
"debug_build": False,
"build_type": "",
"sanitizer": "",
"package_type": "binary",
"static_binary_name": "amd64compat",

2
tests/ci/commit_status_helper.py
View File

@ -7,7 +7,7 @@ from typing import Dict, List, Literal, Optional, Union
import logging
from github import Github
from github.GithubObject import _NotSetType, NotSet as NotSet # type: ignore
from github.GithubObject import _NotSetType, NotSet as NotSet
from github.Commit import Commit
from github.CommitStatus import CommitStatus
from github.IssueComment import IssueComment

2
tests/ci/github_helper.py
View File

@ -111,7 +111,7 @@ class GitHub(github.Github):
# See https://github.com/PyGithub/PyGithub/issues/2202,
# obj._rawData doesn't spend additional API requests
# pylint: disable=protected-access
repo_url = issue._rawData["repository_url"] # type: ignore
repo_url = issue._rawData["repository_url"]
if repo_url not in repos:
repos[repo_url] = issue.repository
prs.append(

2
tests/ci/merge_pr.py
View File

@ -154,7 +154,7 @@ def get_workflows_for_head(repo: Repository, head_sha: str) -> List[WorkflowRun]
return list(
PaginatedList(
WorkflowRun,
repo._requester, # type:ignore # pylint:disable=protected-access
repo._requester, # pylint:disable=protected-access
f"{repo.url}/actions/runs",
{"head_sha": head_sha},
list_item="workflow_runs",

6
tests/ci/report.py
View File

@ -239,7 +239,7 @@ def read_test_results(results_path: Path, with_raw_logs: bool = True) -> TestRes
@dataclass
class BuildResult:
compiler: str
debug_build: bool
build_type: str
sanitizer: str
status: str
elapsed_seconds: int
@ -484,8 +484,8 @@ def create_build_html_report(
):
row = "<tr>"
row += f"<td>{build_result.compiler}</td>"
if build_result.debug_build:
row += "<td>debug</td>"
if build_result.build_type:
row += f"<td>{build_result.build_type}</td>"
else:
row += "<td>relwithdebuginfo</td>"
if build_result.sanitizer:

2
tests/queries/0_stateless/00900_orc_load.sh
View File

@ -12,6 +12,6 @@ ${CLICKHOUSE_CLIENT} --query="select * from orc_load FORMAT ORC" > "${CLICKHOUSE
${CLICKHOUSE_CLIENT} --query="truncate table orc_load"
cat "${CLICKHOUSE_TMP}"/test.orc | ${CLICKHOUSE_CLIENT} -q "insert into orc_load format ORC"
timeout 3 ${CLICKHOUSE_CLIENT} -q "insert into orc_load format ORC" < "${CLICKHOUSE_TMP}"/test.orc
${CLICKHOUSE_CLIENT} -q "insert into orc_load format ORC" < "${CLICKHOUSE_TMP}"/test.orc
${CLICKHOUSE_CLIENT} --query="select * from orc_load"
${CLICKHOUSE_CLIENT} --query="drop table orc_load"

5
tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh
View File

@ -12,7 +12,10 @@ $CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 --query "CREATE DATABA
function thread1()
{
while true; do $CLICKHOUSE_CLIENT -n --query "CREATE TABLE test_01320.r (x UInt64) ENGINE = ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table', 'r') ORDER BY x; DROP TABLE test_01320.r;"; done
while true; do
$CLICKHOUSE_CLIENT -n --query "CREATE TABLE test_01320.r (x UInt64) ENGINE = ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table', 'r') ORDER BY x;
DROP TABLE test_01320.r;" 2>&1 | grep -F "Code:" | grep -v "UNKNOWN_DATABASE"
done
}
function thread2()

4
tests/queries/0_stateless/02344_describe_cache.reference
View File

@ -1,2 +1,2 @@
134217728 10000000 8388608 1 0 0 0 /var/lib/clickhouse/caches/s3_cache/
134217728 10000000 104857600 0 0 0 0 /var/lib/clickhouse/caches/s3_cache_2/
134217728 10000000 33554432 4194304 1 0 0 0 /var/lib/clickhouse/caches/s3_cache/ 100 2 0
134217728 10000000 104857600 4194304 0 0 0 0 /var/lib/clickhouse/caches/s3_cache_2/ 100 2 0

7
tests/queries/0_stateless/02789_filesystem_cache_alignment.reference Normal file
View File

@ -0,0 +1,7 @@
0
OK
OK
0
0
OK
OK

125
tests/queries/0_stateless/02789_filesystem_cache_alignment.sh Executable file
View File

@ -0,0 +1,125 @@
#!/usr/bin/env bash
# Tags: no-fasttest, no-parallel, no-random-settings, no-random-merge-tree-settings
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -nm -q "
DROP TABLE IF EXISTS test;
CREATE TABLE test (a Int32, b String)
ENGINE = MergeTree()
ORDER BY a
SETTINGS disk = disk(type = cache,
max_size = '1Gi',
max_file_segment_size = '40Mi',
boundary_alignment = '20Mi',
path = '$CLICKHOUSE_TEST_UNIQUE_NAME',
disk = 's3_disk');
INSERT INTO test SELECT number, randomString(100) FROM numbers(1000000);
"
QUERY_ID=$RANDOM
$CLICKHOUSE_CLIENT --query_id "$QUERY_ID" -nm -q "
SET enable_filesystem_cache_log = 1;
SYSTEM DROP FILESYSTEM CACHE;
SELECT * FROM test WHERE NOT ignore() LIMIT 1 FORMAT Null;
SYSTEM FLUSH LOGS;
"
query="
SELECT cache_path, file_size,
tupleElement(file_segment_range, 2) - tupleElement(file_segment_range, 1) + 1 as file_segment_size,
formatReadableSize(file_size) as formatted_file_size,
formatReadableSize(file_segment_size) as formatted_file_segment_size,
tupleElement(file_segment_range, 2) as end_offset
FROM (
SELECT arrayJoin(cache_paths) AS cache_path,
local_path,
remote_path,
size as file_size
FROM system.remote_data_paths
WHERE endsWith(local_path, '.bin')
) AS data_paths
INNER JOIN system.filesystem_cache_log AS cache_log
ON data_paths.remote_path = cache_log.source_file_path
WHERE query_id = '$QUERY_ID' "
# File segments cannot be less that 20Mi,
# except for last file segment in a file or if file size is less.
$CLICKHOUSE_CLIENT -nm -q "
SELECT count() FROM ($query)
WHERE file_segment_size < file_size
AND end_offset + 1 != file_size
AND file_segment_size < 20 * 1024 * 1024;
"
all=$($CLICKHOUSE_CLIENT -nm -q "
SELECT count() FROM ($query)
WHERE file_segment_size < file_size AND end_offset + 1 != file_size;
")
#echo $all
if [ "$all" -gt "1" ]; then
echo "OK"
else
echo "FAIL"
fi
count=$($CLICKHOUSE_CLIENT -nm -q "
SELECT count() FROM ($query)
WHERE file_segment_size < file_size
AND end_offset + 1 != file_size
AND formatted_file_segment_size in ('20.00 MiB', '40.00 MiB')
")
if [ "$count" = "$all" ]; then
echo "OK"
else
echo "FAIL"
fi
query2="
SELECT *
FROM (SELECT * FROM ($query)) AS cache_log
INNER JOIN system.filesystem_cache AS cache
ON cache_log.cache_path = cache.cache_path "
$CLICKHOUSE_CLIENT -nm -q "
SELECT count() FROM ($query2)
WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size
AND file_segment_range_end + 1 != file_size
AND downloaded_size < 20 * 1024 * 1024;
"
$CLICKHOUSE_CLIENT -nm -q "
SELECT count() FROM ($query2)
WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size
AND file_segment_range_end + 1 != file_size
AND formatReadableSize(downloaded_size) not in ('20.00 MiB', '40.00 MiB');
"
all=$($CLICKHOUSE_CLIENT -nm -q "
SELECT count() FROM ($query2)
WHERE file_segment_size < file_size AND file_segment_range_end + 1 != file_size;
")
if [ "$all" -gt "1" ]; then
echo "OK"
else
echo "FAIL"
fi
count2=$($CLICKHOUSE_CLIENT -nm -q "
SELECT count() FROM ($query2)
WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size
AND file_segment_range_end + 1 != file_size
AND formatReadableSize(downloaded_size) in ('20.00 MiB', '40.00 MiB');
")
if [ "$count2" = "$all" ]; then
echo "OK"
else
echo "FAIL"
fi