thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 09:32:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix remove in memory disk, fix style, change s3 object naming

Browse Source
This commit is contained in:
Pervakov Grigory 2020-01-16 18:22:58 +03:00
parent b0b3842b57
commit 36e5322320
4 changed files with 27 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
dbms/src/Disks/DiskMemory.cpp
View File

@ -239,15 +239,23 @@ void DiskMemory::remove(const String & path, bool recursive)
if (fileIt == files.end())
return;
if (fileIt->second.type == FileType::Directory)
if (fileIt->second.type == FileType::File)
{
if (recursive)
clearDirectory(path);
else if (std::any_of(files.begin(), files.end(), [path](const auto & file) { return parentPath(file.first) == path; }))
throw Exception("Directory " + path + "is not empty", ErrorCodes::CANNOT_DELETE_DIRECTORY);
files.erase(fileIt);
return;
}
files.erase(fileIt);
if (!recursive && std::any_of(files.begin(), files.end(), [path](const auto & file) { return parentPath(file.first) == path; }))
throw Exception("Directory " + path + "is not empty", ErrorCodes::CANNOT_DELETE_DIRECTORY);
if (recursive)
for (auto iter = files.begin(); iter != files.end();)
{
if (iter->first.size() >= path.size() && std::string_view(iter->first.data(), path.size()) == path)
iter = files.erase(iter);
else
++iter;
}
}
void registerDiskMemory(DiskFactory & factory)

6
dbms/src/Disks/DiskS3.cpp
View File

@ -168,7 +168,7 @@ void DiskS3::copyFile(const String & from_path, const String & to_path)
remove(to_path, false);
String s3_from_path;
String s3_to_path = s3_root_path + to_path + getRandomSuffix();
String s3_to_path = s3_root_path + getRandomName();
Poco::FileInputStream(metadata_path + from_path) >> s3_from_path;
Aws::S3::Model::CopyObjectRequest req;
@ -188,7 +188,7 @@ std::unique_ptr<WriteBuffer> DiskS3::writeFile(const String & path, size_t buf_s
{
if (!exists(path) || mode == WriteMode::Rewrite)
{
String new_s3_path = s3_root_path + path + '.' + getRandomSuffix();
String new_s3_path = s3_root_path + getRandomName();
return std::make_unique<WriteIndirectBufferFromS3>(client, bucket, metadata_path + path, new_s3_path, buf_size);
}
else
@ -237,7 +237,7 @@ String DiskS3::getS3Path(const String & path) const
return s3_path;
}
String DiskS3::getRandomSuffix() const
String DiskS3::getRandomName() const
{
std::mt19937 random{std::random_device{}()};
std::uniform_int_distribution<int> distribution('a', 'z');

9
dbms/src/Disks/DiskS3.h
View File

@ -11,6 +11,11 @@
namespace DB
{
/**
* Storage for persisting data in S3 and metadata on the local disk.
* Files are represented by file in local filesystem (clickhouse_root/disks/disk_name/path/to/file)
* that contains S3 object key with actual data.
*/
class DiskS3 : public IDisk
{
public:
@ -47,7 +52,7 @@ public:
void clearDirectory(const String & path) override
{
for (auto it{iterateDirectory(path)}; it->isValid(); it->next())
if(isFile(it->path()))
if (isFile(it->path()))
remove(it->path(), false);
}
@ -70,7 +75,7 @@ public:
private:
String getS3Path(const String & path) const;
String getRandomSuffix() const;
String getRandomName() const;
bool tryReserve(UInt64 bytes);

10
dbms/tests/integration/test_tinylog_s3/test.py
View File

@ -41,13 +41,9 @@ def test_tinylog_s3(cluster):
node.query("CREATE TABLE s3_test (id UInt64) Engine=TinyLog")
node.query("INSERT INTO s3_test SELECT number FROM numbers(3)")
assert node.query("SELECT * FROM s3_test") == "0\n1\n2\n"
assert list(map(lambda obj: obj.object_name[:-17],
minio.list_objects(cluster.minio_bucket, 'data/data/default/s3_test/'))) == [
'data/data/default/s3_test/id.bin', 'data/data/default/s3_test/tmp_sizes.json']
assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == 2
node.query("INSERT INTO s3_test SELECT number + 3 FROM numbers(3)")
assert node.query("SELECT * FROM s3_test") == "0\n1\n2\n3\n4\n5\n"
assert list(map(lambda obj: obj.object_name[:-17],
minio.list_objects(cluster.minio_bucket, 'data/data/default/s3_test/'))) == [
'data/data/default/s3_test/id.bin', 'data/data/default/s3_test/tmp_sizes.json']
assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == 2
node.query("DROP TABLE s3_test")
assert len(list(minio.list_objects(cluster.minio_bucket, 'data/data/default/s3_test/'))) == 0
assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == 0