thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-23 08:02:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

use path to define compression type

Browse Source
This commit is contained in:
FArthur-cmd 2021-05-09 21:58:08 +03:00
parent cc583fde4f
commit 6363a5dd96
6 changed files with 9 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/Dictionaries/HTTPDictionarySource.cpp
View File

@ -88,9 +88,10 @@ HTTPDictionarySource::HTTPDictionarySource(const HTTPDictionarySource & other)
BlockInputStreamPtr HTTPDictionarySource::createWrappedBuffer(std::unique_ptr<ReadWriteBufferFromHTTP> http_buffer_ptr)
{
Poco::URI uri(url);
String http_request_compression_method_str = http_buffer_ptr->getCompressMethod();
auto in_ptr_wrapped
= wrapReadBufferWithCompressionMethod(std::move(http_buffer_ptr), chooseCompressionMethod(url, http_request_compression_method_str));
= wrapReadBufferWithCompressionMethod(std::move(http_buffer_ptr), chooseCompressionMethod(uri.getPath(), http_request_compression_method_str));
auto input_stream = context->getInputFormat(format, *in_ptr_wrapped, sample_block, max_block_size);
return std::make_shared<OwningBlockInputStream<ReadBuffer>>(input_stream, std::move(in_ptr_wrapped));
}

10
src/IO/CompressionMethod.cpp
View File

@ -46,13 +46,6 @@ std::string toContentEncodingName(CompressionMethod method)
__builtin_unreachable();
}
bool pathExtensionIsCorrect(const std::string& ending)
{
return ending == "gzip" || ending == "gz" || ending == "deflate" ||
ending == "brotli" || ending == "br" || ending == "lzma" ||
ending == "xz" || ending == "zstd" || ending == "zst";
}
CompressionMethod chooseCompressionMethod(const std::string & path, const std::string & hint)
{
std::string file_extension;
@ -63,9 +56,6 @@ CompressionMethod chooseCompressionMethod(const std::string & path, const std::s
file_extension = path.substr(pos + 1, std::string::npos);
}
if (!pathExtensionIsCorrect(file_extension))
file_extension.clear();
std::string method_str = file_extension.empty() ? hint : std::move(file_extension);
boost::algorithm::to_lower(method_str);

2
src/IO/CompressionMethod.h
View File

@ -37,8 +37,6 @@ enum class CompressionMethod
/// How the compression method is named in HTTP.
std::string toContentEncodingName(CompressionMethod method);
bool pathExtensionIsCorrect(const std::string& ending);
/** Choose compression method from path and hint.
* if hint is "auto" or empty string, then path is analyzed,
* otherwise path parameter is ignored and hint is used as compression method name.

2
src/IO/ReadWriteBufferFromHTTP.h
View File

@ -137,7 +137,7 @@ namespace detail
istr = receiveResponse(*sess, request, response, true);
response.getCookies(cookies);
content_encoding = response.get("Content-Encoding");
content_encoding = response.get("Content-Encoding", "");
return istr;
}

2
tests/queries/0_stateless/01854_HTTP_dict_decompression.python
View File

@ -44,7 +44,7 @@ CSV_DATA = "Hello, 1\nWorld, 2\nThis, 152\nis, 9283\ntesting, 2313213\ndata, 555
# (Will change during test, need to check standart data sending, to make sure that nothing broke)
COMPRESS_METHOD = 'none'
ADDING_ENDING = ''
ENDINGS = ['gz', 'xz']
ENDINGS = ['.gz', '.xz']
def get_ch_answer(query):
url = os.environ.get('CLICKHOUSE_URL', 'http://{host}:{port}'.format(host=CLICKHOUSE_HOST, port=CLICKHOUSE_PORT_HTTP))

5
tests/queries/0_stateless/01854_HTTP_dict_decompression.reference
View File

@ -1 +1,6 @@
0 none True
1 gzip True
2 lzma True
3 gzip .gz False
4 lzma .xz False
PASSED