mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-12-15 10:52:30 +00:00
1443e490ea
It's impossible to have persistent pyc files
Each time they are built with different content, for example:
> cmp -bl lambda-package*/charset_normalizer/__pycache__/constant.cpython-310.pyc
15582 6 ^F 4 ^D
15583 164 t 155 m
15584 141 a 142 b
15586 164 t 163 s
15587 151 i 332 M-Z
15588 163 s 6 ^F
15589 332 M-Z 164 t
15590 4 ^D 141 a
15591 155 m 143 c
15592 142 b 164 t
15593 143 c 151 i
17425 74 < 75 =
17428 76 > 46 &
17429 332 M-Z 372 M-z
17431 173 { 55 -
17434 75 = 174 |
17437 57 / 72 :
17440 54 , 73 ;
17441 372 M-z 332 M-Z
17443 174 | 175 }
17446 55 - 54 ,
17447 372 M-z 332 M-Z
17449 46 & 173 {
17452 72 : 76 >
17455 42 " 74 <
17458 73 ; 133 [
17461 135 ] 42 "
17464 133 [ 135 ]
17465 332 M-Z 372 M-z
17467 175 } 57 /
17503 332 M-Z 162 r
17504 5 ^E 130 X
17505 152 j 0 ^@
17506 157 o 0 ^@
17507 150 h 0 ^@
17508 141 a 332 M-Z
17509 142 b 5 ^E
17510 162 r 152 j
17511 130 X 157 o
17512 0 ^@ 150 h
17513 0 ^@ 141 a
17514 0 ^@ 142 b
17536 5 ^E 2 ^B
17537 143 c 150 h
17538 160 p 172 z
17539 71 9 332 M-Z
17540 65 5 5 ^E
17541 60 0 143 c
17542 332 M-Z 160 p
17543 2 ^B 71 9
17544 150 h 65 5
17545 172 z 60 0
77 lines
2.2 KiB
Bash
77 lines
2.2 KiB
Bash
#!/usr/bin/env bash
|
|
set -xeo pipefail
|
|
|
|
WORKDIR=$(dirname "$0")
|
|
WORKDIR=$(readlink -f "${WORKDIR}")
|
|
DIR_NAME=$(basename "$WORKDIR")
|
|
cd "$WORKDIR"
|
|
|
|
# Do not deploy the lambda to AWS
|
|
DRY_RUN=${DRY_RUN:-}
|
|
# Python runtime to install dependencies
|
|
PY_VERSION=${PY_VERSION:-3.10}
|
|
PY_EXEC="python${PY_VERSION}"
|
|
# Image to build the lambda zip package
|
|
DOCKER_IMAGE="public.ecr.aws/lambda/python:${PY_VERSION}"
|
|
# Rename the_lambda_name directory to the-lambda-name lambda in AWS
|
|
LAMBDA_NAME=${DIR_NAME//_/-}
|
|
# The name of directory with lambda code
|
|
PACKAGE=lambda-package
|
|
|
|
# Do not rebuild and deploy the archive if it's newer than sources
|
|
if [ -e "$PACKAGE.zip" ] && [ -z "$FORCE" ]; then
|
|
REBUILD=""
|
|
for src in app.py build_and_deploy_archive.sh requirements.txt lambda_shared/*; do
|
|
if [ "$src" -nt "$PACKAGE.zip" ]; then
|
|
REBUILD=1
|
|
fi
|
|
done
|
|
[ -n "$REBUILD" ] || exit 0
|
|
fi
|
|
|
|
docker_cmd=(
|
|
docker run -i --net=host --rm --user="${UID}" -e HOME=/tmp --entrypoint=/bin/bash
|
|
--volume="${WORKDIR}/..:/ci" --workdir="/ci/${DIR_NAME}" "${DOCKER_IMAGE}"
|
|
)
|
|
rm -rf "$PACKAGE" "$PACKAGE".zip
|
|
mkdir "$PACKAGE"
|
|
cp app.py "$PACKAGE"
|
|
if [ -f requirements.txt ]; then
|
|
VENV=lambda-venv
|
|
rm -rf "$VENV"
|
|
"${docker_cmd[@]}" -ex <<EOF
|
|
'$PY_EXEC' -m venv '$VENV' &&
|
|
source '$VENV/bin/activate' &&
|
|
pip install -r requirements.txt &&
|
|
# To have consistent pyc files
|
|
find '$VENV/lib' -name '*.pyc' -delete
|
|
cp -rT '$VENV/lib/$PY_EXEC/site-packages/' '$PACKAGE'
|
|
rm -r '$PACKAGE'/{pip,pip-*,setuptools,setuptools-*}
|
|
chmod 0777 -R '$PACKAGE'
|
|
EOF
|
|
fi
|
|
# Create zip archive via python zipfile to have it cross-platform
|
|
"${docker_cmd[@]}" -ex <<EOF
|
|
cd '$PACKAGE'
|
|
find ! -type d -exec touch -t 201212121212 {} +
|
|
|
|
python <<'EOP'
|
|
import zipfile
|
|
import os
|
|
files_path = []
|
|
for root, _, files in os.walk('.'):
|
|
files_path.extend(os.path.join(root, file) for file in files)
|
|
# persistent file order
|
|
files_path.sort()
|
|
with zipfile.ZipFile('../$PACKAGE.zip', 'w') as zf:
|
|
for file in files_path:
|
|
zf.write(file)
|
|
EOP
|
|
EOF
|
|
|
|
ECHO=()
|
|
if [ -n "$DRY_RUN" ]; then
|
|
ECHO=(echo Run the following command to push the changes:)
|
|
fi
|
|
"${ECHO[@]}" aws lambda update-function-code --function-name "$LAMBDA_NAME" --zip-file fileb://"$WORKDIR/$PACKAGE".zip
|