thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 00:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix typing in S3Helper.fast_parallel_upload_dir

Browse Source
This commit is contained in:
Mikhail f. Shiryaev 2023-05-10 22:56:13 +02:00
parent e79c1cdb1a
commit c375f948d0
No known key found for this signature in database
GPG Key ID: 4B02ED204C7D93F4
1 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
tests/ci/s3_helper.py
View File

@ -6,6 +6,8 @@ import re
import shutil
import time
from multiprocessing.dummy import Pool
from pathlib import Path
from typing import List, Union
import boto3 # type: ignore
import botocore # type: ignore
@ -128,7 +130,9 @@ class S3Helper:
else:
return S3Helper.copy_file_to_local(S3_BUILDS_BUCKET, file_path, s3_path)
def fast_parallel_upload_dir(self, dir_path, s3_dir_path, bucket_name):
def fast_parallel_upload_dir(
self, dir_path: Union[str, Path], s3_dir_path: str, bucket_name: str
) -> List[str]:
all_files = []
for root, _, files in os.walk(dir_path):
@ -141,12 +145,12 @@ class S3Helper:
t = time.time()
sum_time = 0
def upload_task(file_path):
def upload_task(file_path: str) -> str:
nonlocal counter
nonlocal t
nonlocal sum_time
try:
s3_path = file_path.replace(dir_path, s3_dir_path)
s3_path = file_path.replace(str(dir_path), s3_dir_path)
metadata = {}
if s3_path.endswith("html"):
metadata["ContentType"] = "text/html; charset=utf-8"