ClickHouse/tests/ci/compress_files.py

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

85 lines
2.6 KiB
Python
Raw Normal View History

2021-09-10 14:27:03 +00:00
#!/usr/bin/env python3
import subprocess
import logging
2023-08-10 20:41:41 +00:00
from pathlib import Path
2023-08-10 20:09:05 +00:00
from typing import Optional
2023-08-10 20:09:05 +00:00
2023-08-10 20:41:41 +00:00
PIGZ = Path("/usr/bin/pigz")
SUFFIX = ".zst"
2023-08-10 20:41:41 +00:00
def compress_file_fast(path: Path, archive_path: Path) -> None:
if archive_path.suffix == SUFFIX:
2023-02-15 21:37:21 +00:00
subprocess.check_call(f"zstd < {path} > {archive_path}", shell=True)
2023-08-10 20:41:41 +00:00
elif PIGZ.exists():
2023-02-15 21:37:21 +00:00
subprocess.check_call(f"pigz < {path} > {archive_path}", shell=True)
2021-09-10 14:27:03 +00:00
else:
2023-02-15 21:37:21 +00:00
subprocess.check_call(f"gzip < {path} > {archive_path}", shell=True)
2021-09-10 14:27:03 +00:00
2023-08-10 20:41:41 +00:00
def compress_fast(
path: Path, archive_path: Path, exclude: Optional[Path] = None
) -> None:
2023-01-01 20:17:43 +00:00
program_part = ""
if archive_path.suffix == SUFFIX:
2023-01-01 20:17:43 +00:00
logging.info("zstd will be used for compression")
program_part = "--use-compress-program='zstd --threads=0'"
2023-08-10 20:41:41 +00:00
elif PIGZ.exists():
2021-09-10 14:27:03 +00:00
logging.info("pigz found, will compress and decompress faster")
2023-01-01 20:17:43 +00:00
program_part = "--use-compress-program='pigz'"
2021-09-10 14:27:03 +00:00
else:
2023-01-01 20:17:43 +00:00
program_part = "-z"
2021-09-10 14:27:03 +00:00
logging.info("no pigz, compressing with default tar")
if exclude is None:
exclude_part = ""
elif isinstance(exclude, list):
2023-02-15 21:37:21 +00:00
exclude_part = " ".join([f"--exclude {x}" for x in exclude])
2021-09-10 14:27:03 +00:00
else:
2023-02-15 21:37:21 +00:00
exclude_part = f"--exclude {exclude}"
2021-09-10 14:27:03 +00:00
archive_path.parent.mkdir(parents=True, exist_ok=True)
2023-08-10 20:41:41 +00:00
fname = path.name
2023-01-01 20:17:43 +00:00
2023-08-10 20:41:41 +00:00
cmd = (
f"tar {program_part} {exclude_part} -cf {archive_path} -C {path.parent} {fname}"
)
logging.debug("compress_fast cmd: %s", cmd)
2021-09-10 14:27:03 +00:00
subprocess.check_call(cmd, shell=True)
2023-08-10 20:41:41 +00:00
def decompress_fast(archive_path: Path, result_path: Optional[Path] = None) -> None:
2023-01-01 20:17:43 +00:00
program_part = ""
if archive_path.suffix == SUFFIX:
2023-01-01 21:09:18 +00:00
logging.info(
"zstd will be used for decompression ('%s' -> '%s')",
2023-01-01 20:17:43 +00:00
archive_path,
result_path,
)
2024-07-16 13:11:17 +00:00
program_part = "--use-compress-program='zstd --threads=0 -d'"
2023-08-10 20:41:41 +00:00
elif PIGZ.exists():
logging.info(
"pigz found, will compress and decompress faster ('%s' -> '%s')",
archive_path,
result_path,
)
2024-07-16 13:11:17 +00:00
program_part = "--use-compress-program='pigz -d'"
2021-09-10 14:27:03 +00:00
else:
2023-01-01 20:17:43 +00:00
program_part = "-z"
logging.info(
"no pigz, decompressing with default tar ('%s' -> '%s')",
archive_path,
result_path,
)
2021-09-10 14:27:03 +00:00
if result_path is None:
2023-02-15 21:37:21 +00:00
subprocess.check_call(f"tar {program_part} -xf {archive_path}", shell=True)
2021-09-10 14:27:03 +00:00
else:
result_path.mkdir(parents=True, exist_ok=True)
2021-09-10 14:27:03 +00:00
subprocess.check_call(
2023-02-15 21:37:21 +00:00
f"tar {program_part} -xf {archive_path} -C {result_path}",
2021-09-10 14:27:03 +00:00
shell=True,
)