thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 23:52:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add recursive version for storage file

Browse Source
This commit is contained in:
stavrolia 2019-08-02 18:00:12 +03:00
parent 7addd50c16
commit 95cb95105d
2 changed files with 43 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
dbms/src/Storages/StorageFile.cpp
View File

@ -28,6 +28,7 @@
#include <re2/re2.h>
#include <re2/stringpiece.h>
#include <boost/filesystem.hpp>
#include <boost/filesystem/operations.hpp>
namespace fs = boost::filesystem;
@ -46,6 +47,43 @@ namespace ErrorCodes
extern const int EMPTY_LIST_OF_COLUMNS_PASSED;
}
namespace
{
Strings LSWithRegexpMatching(const String & path_for_ls, const String & for_match)
{
Strings result;
size_t next_slash = for_match.find('/', 1);
String cur_item_for_match = for_match.substr(0, next_slash);
String part_pattern = makeRegexpPatternFromGlobs(cur_item_for_match);
re2::RE2 matcher(part_pattern);
fs::directory_iterator end;
for (fs::directory_iterator it(path_for_ls); it != end; ++it)
{
std::string cur_path = it->path().string();
size_t last_slash = cur_path.rfind('/');
String cur_path_item = cur_path.substr(last_slash);
if ((!is_directory(it->path())) && (next_slash == std::string::npos))
{
if (re2::RE2::FullMatch(cur_path_item, matcher))
{
result.push_back(it->path().string());
}
}
else if ((is_directory(it->path())) && (next_slash != std::string::npos))
{
if (re2::RE2::FullMatch(cur_path_item, matcher))
{
Strings result_part = LSWithRegexpMatching(cur_path, for_match);
std::move(result_part.begin(), result_part.end(), std::back_inserter(result));
}
}
}
return result;
}
}
static std::string getTablePath(const std::string & db_dir_path, const std::string & table_name, const std::string & format_name)
{
@ -97,18 +135,9 @@ StorageFile::StorageFile(
if (path.find_first_of("*?{") != std::string::npos)
{
path_with_globs = true;
re2::RE2 matcher(makeRegexpPatternFromGlobs(path));
fs::path cur_dir(db_dir_path);
fs::directory_iterator end;
for (fs::directory_iterator it(cur_dir); it != end; ++it)
{
std::string file = it->path().string();
if (re2::RE2::FullMatch(file, matcher))
{
matched_paths.push_back(file);
checkCreationIsAllowed(context_global, db_dir_path, matched_paths.back(), table_fd);
}
}
matched_paths = LSWithRegexpMatching(db_dir_path, path);
for (const auto & cur_path : matched_paths)
checkCreationIsAllowed(context_global, db_dir_path, cur_path, table_fd);
} else
checkCreationIsAllowed(context_global, db_dir_path, path, table_fd);
is_db_table = false;

3
dbms/src/Storages/StorageHDFS.cpp
View File

@ -132,10 +132,11 @@ private:
BlockOutputStreamPtr writer;
};
static Strings LSWithRegexpMatching(const String & path_for_ls, const HDFSFSPtr & fs, const String & for_match)
Strings LSWithRegexpMatching(const String & path_for_ls, const HDFSFSPtr & fs, const String & for_match)
{
HDFSFileInfo ls;
ls.file_info = hdfsListDirectory(fs.get(), path_for_ls.data(), &ls.length);
Strings result;
size_t next_slash = for_match.find('/', 1);
String cur_item_for_match = for_match.substr(0, next_slash);