thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 17:41:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix tests

Browse Source
This commit is contained in:
Anton Popov 2024-05-13 16:07:28 +00:00
parent 47d0eadb6f
commit 9ef86e948e
9 changed files with 11 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/Backups/BackupIO_AzureBlobStorage.cpp
View File

@ -9,7 +9,6 @@
#include <Storages/StorageAzureBlobCluster.h>
#include <Disks/IO/ReadBufferFromAzureBlobStorage.h>
#include <Disks/IO/WriteBufferFromAzureBlobStorage.h>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h>
#include <IO/AzureBlobStorage/copyAzureBlobStorageFile.h>
#include <Disks/IDisk.h>
#include <Disks/DiskType.h>

5
src/Backups/BackupIO_AzureBlobStorage.h
View File

@ -1,13 +1,10 @@
#pragma once
#include "Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h"
#include "config.h"
#if USE_AZURE_BLOB_STORAGE
#include <Backups/BackupIO_Default.h>
#include <Disks/DiskType.h>
#include <Storages/StorageAzureBlobCluster.h>
#include <Interpreters/Context_fwd.h>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h>
namespace DB

3
src/Backups/registerBackupEngineAzureBlobStorage.cpp
View File

@ -1,4 +1,3 @@
#include "Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h"
#include "config.h"
#include <Backups/BackupFactory.h>
@ -6,7 +5,7 @@
#if USE_AZURE_BLOB_STORAGE
#include <Backups/BackupIO_AzureBlobStorage.h>
#include <Storages/StorageAzureBlob.h>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h>
#include <Backups/BackupImpl.h>
#include <IO/Archives/hasRegisteredArchiveFileExtension.h>
#include <Interpreters/Context.h>

4
src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp
View File

@ -1,8 +1,4 @@
#include <memory>
#include <type_traits>
#include <variant>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h>
#include <Poco/URI.h>
#if USE_AZURE_BLOB_STORAGE

6
src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h
View File

@ -1,8 +1,4 @@
#pragma once
#include <memory>
#include <Poco/URI.h>
#include "base/strong_typedef.h"
#include "config.h"
#if USE_AZURE_BLOB_STORAGE
@ -15,10 +11,10 @@
#include <azure/identity/managed_identity_credential.hpp>
#include <azure/identity/workload_identity_credential.hpp>
#include <Core/Types.h>
#include <Core/Settings.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <Interpreters/Context_fwd.h>
#include <base/strong_typedef.h>
namespace DB
{

19
src/Storages/StorageAzureBlob.cpp
View File

@ -1,6 +1,4 @@
#include <optional>
#include <Storages/StorageAzureBlob.h>
#include "Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h"
#if USE_AZURE_BLOB_STORAGE
#include <Formats/FormatFactory.h>
@ -44,6 +42,7 @@
#include <Disks/IO/ReadBufferFromAzureBlobStorage.h>
#include <Disks/IO/WriteBufferFromAzureBlobStorage.h>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h>
#include <filesystem>
@ -1307,8 +1306,7 @@ namespace
if (!getContext()->getSettingsRef().schema_inference_use_cache_for_azure)
return;
const auto & params = configuration.connection_params;
String source = fs::path(params.getConnectionURL()) / params.getContainer() / current_path_with_metadata.relative_path;
String source = fs::path(configuration.connection_params.endpoint.getEndpoint()) / current_path_with_metadata.relative_path;
auto key = getKeyForSchemaCache(source, *format, format_settings, getContext());
StorageAzureBlob::getSchemaCache(getContext()).addNumRows(key, num_rows);
}
@ -1319,8 +1317,7 @@ namespace
|| getContext()->getSettingsRef().schema_inference_mode != SchemaInferenceMode::UNION)
return;
const auto & params = configuration.connection_params;
String source = fs::path(params.getConnectionURL()) / params.getContainer() / current_path_with_metadata.relative_path;
String source = fs::path(configuration.connection_params.endpoint.getEndpoint()) / current_path_with_metadata.relative_path;
auto key = getKeyForSchemaCache(source, *format, format_settings, getContext());
StorageAzureBlob::getSchemaCache(getContext()).addColumns(key, columns);
}
@ -1331,12 +1328,11 @@ namespace
|| getContext()->getSettingsRef().schema_inference_mode != SchemaInferenceMode::DEFAULT)
return;
const auto & params = configuration.connection_params;
auto host_and_bucket = params.getConnectionURL() + '/' + params.getContainer();
auto endpoint = fs::path(configuration.connection_params.endpoint.getEndpoint());
Strings sources;
sources.reserve(read_keys.size());
std::transform(read_keys.begin(), read_keys.end(), std::back_inserter(sources), [&](const auto & elem){ return host_and_bucket + '/' + elem.relative_path; });
std::transform(read_keys.begin(), read_keys.end(), std::back_inserter(sources), [&](const auto & elem) { return endpoint / elem.relative_path; });
auto cache_keys = getKeysForSchemaCache(sources, *format, format_settings, getContext());
StorageAzureBlob::getSchemaCache(getContext()).addManyColumns(cache_keys, columns);
}
@ -1376,9 +1372,8 @@ namespace
return std::nullopt;
};
const auto & params = configuration.connection_params;
auto host_and_bucket = params.getConnectionURL() + '/' + params.getContainer();
String source = host_and_bucket + '/' + it->relative_path;
auto endpoint = fs::path(configuration.connection_params.endpoint.getEndpoint());
String source = endpoint / it->relative_path;
if (format)
{

3
src/Storages/StorageAzureBlob.h
View File

@ -1,7 +1,4 @@
#pragma once
#include "Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h"
#include "Interpreters/Context_fwd.h"
#include "config.h"
#if USE_AZURE_BLOB_STORAGE

4
src/TableFunctions/TableFunctionAzureBlobStorageCluster.cpp
View File

@ -1,4 +1,3 @@
#include "Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h"
#include "config.h"
#if USE_AZURE_BLOB_STORAGE
@ -7,11 +6,8 @@
#include <TableFunctions/TableFunctionFactory.h>
#include <Interpreters/parseColumnsListForTableFunction.h>
#include <Storages/StorageAzureBlob.h>
#include "registerTableFunctions.h"
#include <memory>
namespace DB
{

2
tests/integration/test_storage_azure_blob_storage/test.py
View File

@ -789,7 +789,7 @@ def test_read_subcolumns(cluster):
def test_read_from_not_existing_container(cluster):
node = cluster.instances["node"]
query = (
f"select * from azureBlobStorage('{cluster.env_variables['AZURITE_STORAGE_ACCOUNT_URL']}', 'cont_not_exists', 'test_table.csv', "
f"select * from azureBlobStorage('{cluster.env_variables['AZURITE_STORAGE_ACCOUNT_URL']}', 'cont-not-exists', 'test_table.csv', "
f"'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV', 'auto')"
)
expected_err_msg = "container does not exist"